diff --git a/modules/api/src/main/java/org/apache/ignite/tx/IgniteTransactions.java b/modules/api/src/main/java/org/apache/ignite/tx/IgniteTransactions.java index 0527e93e4d58..f8189ea3dc04 100644 --- a/modules/api/src/main/java/org/apache/ignite/tx/IgniteTransactions.java +++ b/modules/api/src/main/java/org/apache/ignite/tx/IgniteTransactions.java @@ -131,7 +131,7 @@ default CompletableFuture beginAsync() { * closure will be retried automatically within the transaction timeout, so it must be pure function. If the transaction timeout * expires before the closure completes successfully and the transaction has been committed, the transaction is rolled back instead. *
- * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * @param clo The closure. @@ -174,7 +174,7 @@ default void runInTransaction(Consumer clo) throws TransactionExcep * closure will be retried automatically within the transaction timeout, so it must be pure function. If the transaction timeout * expires before the closure completes successfully and the transaction has been committed, the transaction is rolled back instead. *
- * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * @param options Transaction options. @@ -223,7 +223,7 @@ default void runInTransaction(Consumer clo, @Nullable TransactionOp * closure will be retried automatically within the transaction timeout, so it must be pure function. If the transaction timeout * expires before the closure completes successfully and the transaction has been committed, the transaction is rolled back instead. *
- * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * @param clo Closure. @@ -268,7 +268,7 @@ default T runInTransaction(Function clo) throws TransactionE * closure will be retried automatically within the transaction timeout, so it must be pure function. If the transaction timeout * expires before the closure completes successfully and the transaction has been committed, the transaction is rolled back instead. *
- * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * @param clo The closure. @@ -304,7 +304,7 @@ default T runInTransaction(Function clo, @Nullable Transacti * closure will be retried automatically within the transaction timeout, so it must be pure function. If the transaction timeout * expires before the closure completes successfully and the transaction has been committed, the transaction is rolled back instead. *
- * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * @param clo The closure. @@ -333,7 +333,7 @@ default CompletableFuture runInTransactionAsync(Function - * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, {@code TimeoutException}, + * The closure is retried only in cases of "expected" exceptions, like {@code LockException}, * exceptions related to the primary replica change, etc. * * diff --git a/modules/api/src/main/java/org/apache/ignite/tx/RunInTransactionInternalImpl.java b/modules/api/src/main/java/org/apache/ignite/tx/RunInTransactionInternalImpl.java index c67797cc4ef8..60ac1ca8fcb8 100644 --- a/modules/api/src/main/java/org/apache/ignite/tx/RunInTransactionInternalImpl.java +++ b/modules/api/src/main/java/org/apache/ignite/tx/RunInTransactionInternalImpl.java @@ -30,7 +30,6 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Function; import org.jetbrains.annotations.Nullable; @@ -61,11 +60,14 @@ static T runInTransactionInternal( T ret; while (true) { + // TODO IGNITE-28448 Use tx restart counter to avoid starvation. tx = igniteTransactions.begin(txOptions); try { ret = clo.apply(tx); + tx.commit(); // Commit is retriable. + break; } catch (Exception ex) { addSuppressedToList(suppressed, ex); @@ -98,19 +100,6 @@ static T runInTransactionInternal( } } - try { - tx.commit(); - } catch (Exception e) { - try { - // Try to rollback tx in case if it's not finished. Retry is not needed here due to the durable finish. - tx.rollback(); - } catch (Exception re) { - e.addSuppressed(re); - } - - throw e; - } - return ret; } @@ -158,6 +147,7 @@ static CompletableFuture runInTransactionAsyncInternal( .thenCompose(tx -> { try { return clo.apply(tx) + .thenCompose(res -> tx.commitAsync().thenApply(ignored -> res)) .handle((res, e) -> { if (e != null) { return handleClosureException( @@ -173,30 +163,11 @@ static CompletableFuture runInTransactionAsyncInternal( } else { return completedFuture(res); } - }) - .thenCompose(identity()) - .thenApply(res -> new TxWithVal<>(tx, res)); + }).thenCompose(identity()); } catch (Exception e) { - return handleClosureException(igniteTransactions, tx, clo, txOptions, startTimestamp, initialTimeout, sup, e) - .thenApply(res -> new TxWithVal<>(tx, res)); + return handleClosureException(igniteTransactions, tx, clo, txOptions, startTimestamp, initialTimeout, sup, e); } - }) - // Transaction commit with rollback on failure, without retries. - // Transaction rollback on closure failure is implemented in closure retry logic. - .thenCompose(txWithVal -> - txWithVal.tx.commitAsync() - .handle((ignored, e) -> { - if (e == null) { - return completedFuture(null); - } else { - return txWithVal.tx.rollbackAsync() - // Rethrow commit exception. - .handle((ign, re) -> sneakyThrow(e)); - } - }) - .thenCompose(fut -> fut) - .thenApply(ignored -> txWithVal.val) - ); + }); } private static CompletableFuture handleClosureException( @@ -311,10 +282,7 @@ private static CompletableFuture throwExceptionWithSuppressedAsync(Throwab } private static boolean isRetriable(Throwable e) { - return hasCause(e, - TimeoutException.class, - RetriableTransactionException.class - ); + return hasCause(e, RetriableTransactionException.class); } private static boolean hasCause(Throwable e, Class... classes) { @@ -347,14 +315,4 @@ private static long calcRemainingTime(long initialTimeout, long startTimestamp) private static E sneakyThrow(Throwable e) throws E { throw (E) e; } - - private static class TxWithVal { - private final Transaction tx; - private final T val; - - private TxWithVal(Transaction tx, T val) { - this.tx = tx; - this.val = val; - } - } } diff --git a/modules/api/src/test/java/org/apache/ignite/tx/RunInTransactionRetryTest.java b/modules/api/src/test/java/org/apache/ignite/tx/RunInTransactionRetryTest.java index c629eba22fdc..2865a12535d9 100644 --- a/modules/api/src/test/java/org/apache/ignite/tx/RunInTransactionRetryTest.java +++ b/modules/api/src/test/java/org/apache/ignite/tx/RunInTransactionRetryTest.java @@ -88,8 +88,8 @@ public void testRetries( } boolean requiresEventualSuccess = closureFailureCount < Integer.MAX_VALUE - // Commit failure can't be retried. - && commitFailureCount == 0 + && commitFailureCount < Integer.MAX_VALUE + && (commitFailureCount == 0 || rollbackFailureCount < Integer.MAX_VALUE) // Rollbacks should be retried until success or timeout, so the rollback must succeed before closure retry. && (closureFailureCount == 0 || rollbackFailureCount < Integer.MAX_VALUE); diff --git a/modules/client/src/integrationTest/java/org/apache/ignite/internal/client/ItThinClientTransactionsTest.java b/modules/client/src/integrationTest/java/org/apache/ignite/internal/client/ItThinClientTransactionsTest.java index aedee88476be..b5176e9a5170 100644 --- a/modules/client/src/integrationTest/java/org/apache/ignite/internal/client/ItThinClientTransactionsTest.java +++ b/modules/client/src/integrationTest/java/org/apache/ignite/internal/client/ItThinClientTransactionsTest.java @@ -258,13 +258,19 @@ void testAccessLockedKeyTimesOut() throws Exception { // Lock the key in tx2. Transaction tx2 = client().transactions().begin(); + IgniteImpl server0 = unwrapIgniteImpl(server(0)); + boolean invertedWaitOrder = server0.txManager().lockManager().policy().invertedWaitOrder(); + + Transaction owner = invertedWaitOrder ? tx2 : tx1; + Transaction waiter = invertedWaitOrder ? tx1 : tx2; + try { - kvView.put(tx2, -100, "1"); + kvView.put(owner, -100, "1"); // Get the key in tx1 - time out. - assertThrows(TimeoutException.class, () -> kvView.getAsync(tx1, -100).get(1, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> kvView.getAsync(waiter, -100).get(1, TimeUnit.SECONDS)); } finally { - tx2.rollback(); + owner.rollback(); } } @@ -1374,11 +1380,15 @@ public void testRollbackDoesNotBlockOnLockConflict(KillTestContext ctx) { assertTrue(olderTx.txId().compareTo(youngerTx.txId()) < 0); - // Older is allowed to wait with wait-die. - CompletableFuture fut = ctx.put.apply(client(), olderTxProxy, key2); - assertFalse(fut.isDone()); - IgniteImpl ignite = unwrapIgniteImpl(server); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + + ClientLazyTransaction owner = invertedWaitOrder ? youngerTxProxy : olderTxProxy; + ClientLazyTransaction waiter = invertedWaitOrder ? olderTxProxy : youngerTxProxy; + + CompletableFuture fut = + invertedWaitOrder ? ctx.put.apply(client(), olderTxProxy, key2) : ctx.put.apply(client(), youngerTxProxy, key); + assertFalse(fut.isDone()); await().atMost(2, TimeUnit.SECONDS).until(() -> { Iterator locks = ignite.txManager().lockManager().locks(olderTx.txId()); @@ -1386,13 +1396,13 @@ public void testRollbackDoesNotBlockOnLockConflict(KillTestContext ctx) { return CollectionUtils.count(locks) == 2; }); - assertThat(olderTxProxy.rollbackAsync(), willSucceedFast()); + assertThat(waiter.rollbackAsync(), willSucceedFast()); // Operation future should be failed. assertThat(fut, willThrowWithCauseOrSuppressed(ctx.expectedErr)); // Ensure inflights cleanup. - assertThat(youngerTxProxy.rollbackAsync(), willSucceedFast()); + assertThat(owner.rollbackAsync(), willSucceedFast()); assertThat(kvView.removeAllAsync(null, Arrays.asList(key0, key, key2)), willSucceedFast()); } @@ -1480,10 +1490,18 @@ public void testRollbackDoesNotBlockOnLockConflictWithDirectMapping(KillTestCont // Should be directly mapped assertThat(ctx.put.apply(client(), youngerTxProxy, key3), willSucceedFast()); + assertThat(ctx.put.apply(client(), olderTxProxy, key4), willSucceedFast()); + + IgniteImpl server0 = unwrapIgniteImpl(server(0)); + boolean invertedWaitOrder = server0.txManager().lockManager().policy().invertedWaitOrder(); - // Younger is not allowed to wait with wait-die. - // Next operation should invalidate the transaction. - assertThat(ctx.put.apply(client(), youngerTxProxy, key), willThrowWithCauseOrSuppressed(ctx.expectedErr)); + // Force wrong order. + if (invertedWaitOrder) { + assertThat(ctx.put.apply(client(), youngerTxProxy, key), willThrowWithCauseOrSuppressed(ctx.expectedErr)); + } else { + assertThat(ctx.put.apply(client(), olderTxProxy, key2), willSucceedFast()); // Will invalidate younger tx. + assertThat(youngerTxProxy.commitAsync(), willThrowWithCauseOrSuppressed(TransactionException.class)); + } olderTxProxy.commit(); @@ -1493,7 +1511,7 @@ public void testRollbackDoesNotBlockOnLockConflictWithDirectMapping(KillTestCont @ParameterizedTest @MethodSource("killTestContextFactory") - public void testRollbackOnLocalError(KillTestContext ctx) throws Exception { + public void testRollbackOnLocalError(KillTestContext ctx) { ClientTable table = (ClientTable) table(); ClientSql sql = (ClientSql) client().sql(); KeyValueView kvView = table().keyValueView(); diff --git a/modules/client/src/integrationTest/java/org/apache/ignite/internal/streamer/ItClientDataStreamerLoadTest.java b/modules/client/src/integrationTest/java/org/apache/ignite/internal/streamer/ItClientDataStreamerLoadTest.java index 82b217d90f96..9988c7777d0e 100644 --- a/modules/client/src/integrationTest/java/org/apache/ignite/internal/streamer/ItClientDataStreamerLoadTest.java +++ b/modules/client/src/integrationTest/java/org/apache/ignite/internal/streamer/ItClientDataStreamerLoadTest.java @@ -20,6 +20,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.concurrent.CompletableFuture; import java.util.concurrent.SubmissionPublisher; @@ -27,6 +29,10 @@ import org.apache.ignite.client.IgniteClient; import org.apache.ignite.client.RetryLimitPolicy; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; +import org.apache.ignite.internal.TestWrappers; +import org.apache.ignite.internal.app.IgniteImpl; +import org.apache.ignite.internal.logger.IgniteLogger; +import org.apache.ignite.internal.logger.Loggers; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.table.DataStreamerItem; import org.apache.ignite.table.DataStreamerOptions; @@ -42,6 +48,8 @@ * Data streamer load test. */ public final class ItClientDataStreamerLoadTest extends ClusterPerClassIntegrationTest { + private static final IgniteLogger LOG = Loggers.forClass(ItClientDataStreamerLoadTest.class); + private static final String TABLE_NAME = "test_table"; private static final int CLIENT_COUNT = 2; @@ -90,6 +98,9 @@ public void clearTable() { @Test @Timeout(value = 20, unit = TimeUnit.MINUTES) public void testHighLoad() throws InterruptedException { + IgniteImpl ignite = TestWrappers.unwrapIgniteImpl(node(0)); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + Thread[] threads = new Thread[CLIENT_COUNT]; for (int i = 0; i < clients.length; i++) { @@ -106,8 +117,27 @@ public void testHighLoad() throws InterruptedException { RecordView view = clients[0].tables().table(TABLE_NAME).recordView(); + List keys = new ArrayList<>(ROW_COUNT); + for (int i = 0; i < ROW_COUNT; i++) { - Tuple res = view.get(null, tupleKey(i)); + Tuple key = tupleKey(i); + keys.add(key); + } + + List values = view.getAll(null, keys); + assertEquals(ROW_COUNT, values.size()); + + for (int i = 0; i < ROW_COUNT; i++) { + Tuple res = values.get(i); + + // TODO https://issues.apache.org/jira/browse/IGNITE-28365 + // A row might be missing in the following scenario (assuming 2 concurrent streamers): + // batch 1 is concurrently mapped to partition K, streamer 0 wins the conflict + // batch 2 is concurrently mapped to partition N, streamer 1 wins the conflict + // Both streamers become invalidated without proper implicit retries and stop. + if (res == null && !invertedWaitOrder) { + continue; + } assertNotNull(res, "Row not found: " + i); assertEquals("foo_" + i, res.value("name")); @@ -130,13 +160,20 @@ private static void streamData(IgniteClient client) { // Insert same data over and over again. for (int j = 0; j < LOOP_COUNT; j++) { + LOG.info("Loop " + j); for (int i = 0; i < ROW_COUNT; i++) { publisher.submit(DataStreamerItem.of(tuple(i, "foo_" + i))); } } } - streamerFut.orTimeout(10, TimeUnit.SECONDS).join(); + try { + streamerFut.orTimeout(10, TimeUnit.SECONDS).join(); + LOG.info("Done streaming"); + } catch (Exception e) { + // TODO IGNITE-28365 Don't expecting errors here with proper retries + LOG.warn("Done streaming with error", e); + } } private static Tuple tuple(int id, String name) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/lang/NodeStoppingException.java b/modules/core/src/main/java/org/apache/ignite/internal/lang/NodeStoppingException.java index 55a5ac34edd1..8983ab6e92ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/lang/NodeStoppingException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/lang/NodeStoppingException.java @@ -20,15 +20,12 @@ import static org.apache.ignite.lang.ErrorGroups.Common.NODE_STOPPING_ERR; import java.util.UUID; -import org.apache.ignite.tx.RetriableReplicaRequestException; -import org.apache.ignite.tx.RetriableTransactionException; import org.jetbrains.annotations.Nullable; /** * This exception is used to indicate that Ignite node is stopping (already stopped) for some reason. */ -public class NodeStoppingException extends IgniteInternalCheckedException implements RetriableTransactionException, - RetriableReplicaRequestException { +public class NodeStoppingException extends IgniteInternalCheckedException { /** Serial version UID. */ private static final long serialVersionUID = 0L; diff --git a/modules/core/src/testFixtures/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java b/modules/core/src/testFixtures/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java index e0ba820cbd47..3b07d0ad9899 100644 --- a/modules/core/src/testFixtures/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java +++ b/modules/core/src/testFixtures/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java @@ -24,6 +24,7 @@ import static org.apache.ignite.internal.util.IgniteUtils.deleteIfExists; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.fail; @@ -702,6 +703,19 @@ public static boolean waitForCondition(BooleanSupplier cond, long sleepMillis, l return false; } + /** + * Ensure the future is not completed for a duration. + * + * @param future The future. + * @param durationMillis Milliseconds to check for condition. + */ + public static void ensureFutureNotCompleted(CompletableFuture future, long durationMillis) { + Awaitility.await() + .pollInterval(10, TimeUnit.MILLISECONDS) + .during(durationMillis, TimeUnit.MILLISECONDS) + .until(future::isDone, is(false)); + } + /** * Returns random byte array. * diff --git a/modules/platforms/cpp/tests/odbc-test/transaction_test.cpp b/modules/platforms/cpp/tests/odbc-test/transaction_test.cpp index d5aa36499397..0a3abc55e840 100644 --- a/modules/platforms/cpp/tests/odbc-test/transaction_test.cpp +++ b/modules/platforms/cpp/tests/odbc-test/transaction_test.cpp @@ -567,7 +567,8 @@ TEST_F(transaction_test, transaction_environment_rollback_delete_2) { check_test_value(42, "Some"); } -TEST_F(transaction_test, transaction_error) { +// TODO https://issues.apache.org/jira/browse/IGNITE-28372 +TEST_F(transaction_test, DISABLED_transaction_error) { odbc_connect(get_basic_connection_string()); insert_test_value(1, "test_1"); diff --git a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicaUnavailableException.java b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicaUnavailableException.java index 67e2b45af267..3a6c75594534 100644 --- a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicaUnavailableException.java +++ b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicaUnavailableException.java @@ -21,11 +21,12 @@ import org.apache.ignite.internal.network.InternalClusterNode; import org.apache.ignite.internal.replicator.ReplicationGroupId; +import org.apache.ignite.tx.RetriableTransactionException; /** * The exception is thrown when a replica is not ready to handle a request. */ -public class ReplicaUnavailableException extends ReplicationException { +public class ReplicaUnavailableException extends ReplicationException implements RetriableTransactionException { private static final long serialVersionUID = 9142077461528136559L; /** diff --git a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicationException.java b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicationException.java index ea8360231dc0..ecf1884e0165 100644 --- a/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicationException.java +++ b/modules/replicator/src/main/java/org/apache/ignite/internal/replicator/exception/ReplicationException.java @@ -23,13 +23,11 @@ import org.apache.ignite.internal.lang.IgniteInternalException; import org.apache.ignite.internal.replicator.ReplicationGroupId; import org.apache.ignite.tx.RetriableReplicaRequestException; -import org.apache.ignite.tx.RetriableTransactionException; /** * The exception is thrown when some issue happened during a replication. */ -public class ReplicationException extends IgniteInternalException implements RetriableTransactionException, - RetriableReplicaRequestException { +public class ReplicationException extends IgniteInternalException implements RetriableReplicaRequestException { /** * Constructor. * diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/TpccBenchmarkNodeRunner.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/TpccBenchmarkNodeRunner.java new file mode 100644 index 000000000000..90f4263cfa84 --- /dev/null +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/benchmark/TpccBenchmarkNodeRunner.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmark; + +import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl; +import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.getAllResultSet; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteServer; +import org.apache.ignite.InitParameters; +import org.apache.ignite.internal.app.IgniteImpl; +import org.apache.ignite.internal.failure.handlers.configuration.StopNodeOrHaltFailureHandlerConfigurationSchema; +import org.apache.ignite.internal.lang.IgniteStringFormatter; +import org.apache.ignite.internal.testframework.TestIgnitionManager; +import org.apache.ignite.sql.IgniteSql; +import org.apache.ignite.sql.ResultSet; +import org.apache.ignite.sql.SqlRow; +import org.apache.ignite.sql.Statement; +import org.apache.ignite.sql.Statement.StatementBuilder; +import org.apache.ignite.tx.Transaction; +import org.intellij.lang.annotations.Language; +import org.jetbrains.annotations.Nullable; + +/** + * Extendable class to start a dedicated cluster node for TPC-C benchmark. + */ +public class TpccBenchmarkNodeRunner { + private static final int BASE_PORT = 3344; + private static final int BASE_CLIENT_PORT = 10942; + private static final int BASE_REST_PORT = 10300; + + private static final List igniteServers = new ArrayList<>(); + + protected static Ignite publicIgnite; + protected static IgniteImpl igniteImpl; + + public static void main(String[] args) throws Exception { + TpccBenchmarkNodeRunner runner = new TpccBenchmarkNodeRunner(); + runner.startCluster(args.length == 0 ? null : args[0]); + } + + public IgniteImpl node(int idx) { + return unwrapIgniteImpl(igniteServers.get(idx).api()); + } + + private void startCluster(@Nullable String pathToWorkDir) throws Exception { + Path workDir = workDir(pathToWorkDir); + + String connectNodeAddr = "\"localhost:" + BASE_PORT + '\"'; + + @Language("HOCON") + String configTemplate = "ignite {\n" + + " network: {\n" + + " port:{},\n" + + " nodeFinder:{\n" + + " netClusterNodes: [ {} ]\n" + + " }\n" + + " },\n" + + " storage.profiles: {" + + " " + DEFAULT_STORAGE_PROFILE + ".engine: aipersist, " + + " " + DEFAULT_STORAGE_PROFILE + ".sizeBytes: " + pageMemorySize() + " " + + " },\n" + + " clientConnector: { port:{} },\n" + + " clientConnector.sendServerExceptionStackTraceToClient: true\n" + + " rest.port: {},\n" + + " raft.fsync = " + fsync() + ",\n" + + " system.partitionsLogPath = \"" + logPath() + "\",\n" + + " failureHandler.handler: {\n" + + " type: \"" + StopNodeOrHaltFailureHandlerConfigurationSchema.TYPE + "\",\n" + + " tryStop: true,\n" + + " timeoutMillis: 60000,\n" // 1 minute for graceful shutdown + + " },\n" + + "}"; + + for (int i = 0; i < nodes(); i++) { + int port = BASE_PORT + i; + String nodeName = nodeName(port); + + String config = IgniteStringFormatter.format(configTemplate, port, connectNodeAddr, + BASE_CLIENT_PORT + i, BASE_REST_PORT + i); + + igniteServers.add(TestIgnitionManager.startWithProductionDefaults(nodeName, config, workDir.resolve(nodeName))); + } + + String metaStorageNodeName = nodeName(BASE_PORT); + + InitParameters initParameters = InitParameters.builder() + .metaStorageNodeNames(metaStorageNodeName) + .clusterName("cluster") + .clusterConfiguration(clusterConfiguration()) + .build(); + + TestIgnitionManager.init(igniteServers.get(0), initParameters); + + for (IgniteServer node : igniteServers) { + assertThat(node.waitForInitAsync(), willCompleteSuccessfully()); + + if (publicIgnite == null) { + publicIgnite = node.api(); + igniteImpl = unwrapIgniteImpl(publicIgnite); + } + } + } + + @Nullable + protected String clusterConfiguration() { + return "ignite {}"; + } + + protected static String nodeName(int port) { + return "node_" + port; + } + + protected static Path workDir(@Nullable String pathToWorkDir) throws Exception { + if (pathToWorkDir == null) { + return Files.createTempDirectory("tmpDirPrefix").toFile().toPath(); + } else { + return new File(pathToWorkDir).toPath(); + } + } + + protected int pageMemorySize() { + return 2073741824; + } + + protected String logPath() { + return ""; + } + + protected boolean fsync() { + return false; + } + + protected int nodes() { + return 1; + } + + protected void dumpWarehouse() { + final String query = "select * from warehouse"; + System.out.println("Executing the query: "); + List> rows = sql(publicIgnite, null, null, null, query); + for (List row : rows) { + System.out.println("Row: " + row); + } + } + + protected static List> sql(Ignite node, @Nullable Transaction tx, @Nullable String schema, @Nullable ZoneId zoneId, + String query, Object... args) { + IgniteSql sql = node.sql(); + StatementBuilder builder = sql.statementBuilder() + .query(query); + + if (zoneId != null) { + builder.timeZoneId(zoneId); + } + + if (schema != null) { + builder.defaultSchema(schema); + } + + Statement statement = builder.build(); + try (ResultSet rs = sql.execute(tx, statement, args)) { + return getAllResultSet(rs); + } + } +} diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDataConsistencyTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDataConsistencyTest.java index 9b5ed3530fd4..f4f1d06d3db3 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDataConsistencyTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItDataConsistencyTest.java @@ -17,10 +17,13 @@ package org.apache.ignite.internal.table; +import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl; import static org.apache.ignite.internal.catalog.CatalogService.DEFAULT_STORAGE_PROFILE; import static org.apache.ignite.internal.sql.engine.util.SqlTestUtils.executeUpdate; import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; @@ -28,14 +31,19 @@ import java.util.List; import java.util.Random; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import org.apache.ignite.Ignite; import org.apache.ignite.internal.ClusterPerClassIntegrationTest; +import org.apache.ignite.internal.app.IgniteImpl; +import org.apache.ignite.internal.util.ExceptionUtils; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.table.Table; import org.apache.ignite.table.Tuple; +import org.apache.ignite.tx.RetriableTransactionException; import org.apache.ignite.tx.Transaction; import org.apache.ignite.tx.TransactionException; import org.junit.jupiter.api.AfterAll; @@ -50,19 +58,20 @@ public class ItDataConsistencyTest extends ClusterPerClassIntegrationTest { private static final String ZONE_NAME = "test_zone"; private static final String TABLE_NAME = "accounts"; private static final int WRITE_PARALLELISM = Runtime.getRuntime().availableProcessors(); - private static final int READ_PARALLELISM = 1; + private static final int READ_PARALLELISM = 0; private static final int ACCOUNTS_COUNT = WRITE_PARALLELISM * 10; private static final double INITIAL = 1000; private static final double TOTAL = ACCOUNTS_COUNT * INITIAL; - private static final int DURATION_MILLIS = 10000; + private static final int DURATION_MILLIS = 30000; private CyclicBarrier startBar = new CyclicBarrier(WRITE_PARALLELISM + READ_PARALLELISM, () -> log.info("Before test")); private LongAdder ops = new LongAdder(); private LongAdder fails = new LongAdder(); + private LongAdder restarts = new LongAdder(); private LongAdder readOps = new LongAdder(); private LongAdder readFails = new LongAdder(); private AtomicBoolean stop = new AtomicBoolean(); - private Random rng = new Random(); + private Random rng = new Random(0); private AtomicReference firstErr = new AtomicReference<>(); @BeforeAll @@ -92,6 +101,7 @@ public void dropTables() { @Test public void testDataConsistency() throws InterruptedException { + stop.set(false); Thread[] threads = new Thread[WRITE_PARALLELISM]; for (int i = 0; i < threads.length; i++) { @@ -112,18 +122,29 @@ public void testDataConsistency() throws InterruptedException { readThreads[i].start(); } + log.info("Started {} writers", WRITE_PARALLELISM); + log.info("Started {} readers", READ_PARALLELISM); + long cur = System.currentTimeMillis(); + long curOps = ops.sum(); + while (cur + DURATION_MILLIS > System.currentTimeMillis()) { Thread.sleep(1000); - log.info("Waiting..."); + long tmp = ops.sum(); + assertNotEquals(tmp, curOps, "Test doesn't make progress"); + log.info("Running... ops={} restarts={} fails={} readOps={} readFails={}", + tmp, restarts.sum(), fails.sum(), readOps.sum(), readFails.sum()); + curOps = tmp; if (firstErr.get() != null) { throw new IgniteException(INTERNAL_ERR, firstErr.get()); } } + log.info("Stop running"); + stop.set(true); for (Thread thread : threads) { @@ -133,6 +154,7 @@ public void testDataConsistency() throws InterruptedException { readThread.join(3_000); } + // TODO IGNITE-28464 unregister from expiration tracker. validate(); } @@ -163,7 +185,8 @@ private void validate() { Ignite node = node(0); Table accounts = node.tables().table("accounts"); - log.info("After test ops={} fails={} readOps={} readFails={}", ops.sum(), fails.sum(), readOps.sum(), readFails.sum()); + log.info("After test ops={} restarts={} fails={} readOps={} readFails={}", ops.sum(), restarts.sum(), fails.sum(), readOps.sum(), + readFails.sum()); double total0 = 0; @@ -174,6 +197,12 @@ private void validate() { } assertEquals(TOTAL, total0, "Total amount invariant is not preserved"); + + for (int i = 0; i < initialNodes(); i++) { + IgniteImpl ignite = unwrapIgniteImpl(node(i)); + await("node " + i + " should release all locks").atMost(3, TimeUnit.SECONDS) + .until(() -> ignite.txManager().lockManager().isEmpty()); + } } private Runnable createWriter(int workerId) { @@ -191,8 +220,8 @@ private Runnable createWriter(int workerId) { Transaction tx = node.transactions().begin(); var view = node.tables().table("accounts").recordView(); - try { + long acc1 = rng.nextInt(ACCOUNTS_COUNT); double amount = 100 + rng.nextInt(500); @@ -215,13 +244,23 @@ private Runnable createWriter(int workerId) { ops.increment(); } catch (TransactionException e) { - // Don't need to rollback manually if got IgniteException. - fails.increment(); + if (isRetriable(e)) { + restarts.increment(); + } else { + fails.increment(); + } } } }; } + private static boolean isRetriable(Throwable e) { + return ExceptionUtils.hasCause(e, + TimeoutException.class, + RetriableTransactionException.class + ); + } + private Runnable createReader(int workerId) { return () -> { try { @@ -277,4 +316,9 @@ private static Tuple makeKey(long id) { private static Tuple makeValue(long id, double balance) { return Tuple.create().set("accountNumber", id).set("balance", balance); } + + @Override + protected int initialNodes() { + return 1; + } } diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItTableScanTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItTableScanTest.java index a3165cc0acd5..996166003461 100644 --- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItTableScanTest.java +++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/table/ItTableScanTest.java @@ -221,25 +221,36 @@ private int getIndexId(IgniteImpl ignite, String idxName) { public void testInsertWaitScanComplete() throws Exception { IgniteTransactions transactions = igniteTx(); - InternalTransaction tx0 = (InternalTransaction) transactions.begin(); - InternalTransaction tx1 = startTxWithEnlistedPartition(PART_ID, false); + IgniteImpl ignite = unwrapIgniteImpl(CLUSTER.aliveNode()); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + + InternalTransaction waiterTx; + InternalTransaction lockerTx; + + if (invertedWaitOrder) { + waiterTx = (InternalTransaction) transactions.begin(); + lockerTx = startTxWithEnlistedPartition(PART_ID, false); + } else { + lockerTx = startTxWithEnlistedPartition(PART_ID, false); + waiterTx = (InternalTransaction) transactions.begin(); + } int sortedIndexId = getSortedIndexId(); List scannedRows = new ArrayList<>(); ZonePartitionId replicationGroupId = replicationGroup(PART_ID); - PendingTxPartitionEnlistment enlistment = tx1.enlistedPartition(replicationGroupId); + PendingTxPartitionEnlistment enlistment = lockerTx.enlistedPartition(replicationGroupId); InternalClusterNode recipient = getNodeByConsistentId(enlistment.primaryNodeConsistentId()); Publisher publisher = new RollbackTxOnErrorPublisher<>( - tx1, + lockerTx, internalTable.scan( PART_ID, recipient, sortedIndexId, IndexScanCriteria.unbounded(), - OperationContext.create(TxContext.readWrite(tx1, enlistment.consistencyToken())) + OperationContext.create(TxContext.readWrite(lockerTx, enlistment.consistencyToken())) ) ); @@ -254,7 +265,7 @@ public void testInsertWaitScanComplete() throws Exception { assertFalse(scanned.isDone()); CompletableFuture updateKey2Fut = table.keyValueView() - .putAsync(tx0, Tuple.create().set("key", 2), Tuple.create().set("valInt", 2).set("valStr", "New_2")); + .putAsync(waiterTx, Tuple.create().set("key", 2), Tuple.create().set("valInt", 2).set("valStr", "New_2")); assertFalse(updateKey2Fut.isDone()); @@ -263,7 +274,7 @@ public void testInsertWaitScanComplete() throws Exception { assertThat(scanned, willCompleteSuccessfully()); CompletableFuture insertKey99Fut = table.keyValueView() - .putAsync(tx0, Tuple.create().set("key", 99), Tuple.create().set("valInt", 99).set("valStr", "New_99")); + .putAsync(waiterTx, Tuple.create().set("key", 99), Tuple.create().set("valInt", 99).set("valStr", "New_99")); assertFalse(insertKey99Fut.isDone()); @@ -271,12 +282,12 @@ public void testInsertWaitScanComplete() throws Exception { assertEquals(ROW_IDS.size(), scannedRows.size()); - tx1.commit(); + lockerTx.commit(); assertThat(updateKey2Fut, willCompleteSuccessfully()); assertThat(insertKey99Fut, willCompleteSuccessfully()); - tx0.commit(); + waiterTx.commit(); } @Test @@ -537,9 +548,14 @@ public void testTwiceScanInTransaction() throws Exception { assertFalse(scanned.isDone()); - assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( - () -> kvView.put(null, Tuple.create().set("key", 3), Tuple.create().set("valInt", 3).set("valStr", "New_3")) - ); + IgniteImpl ignite = unwrapIgniteImpl(CLUSTER.aliveNode()); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + + if (invertedWaitOrder) { + assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( + () -> kvView.put(null, Tuple.create().set("key", 3), Tuple.create().set("valInt", 3).set("valStr", "New_3")) + ); + } kvView.put(null, Tuple.create().set("key", 8), Tuple.create().set("valInt", 8).set("valStr", "New_8")); @@ -608,13 +624,18 @@ public void testScanWithUpperBound() throws Exception { assertEquals(3, scannedRows.size()); - assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( - () -> kvView.put(null, Tuple.create().set("key", 8), Tuple.create().set("valInt", 8).set("valStr", "New_8")) - ); + IgniteImpl ignite = unwrapIgniteImpl(CLUSTER.aliveNode()); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); - assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( - () -> kvView.put(null, Tuple.create().set("key", 9), Tuple.create().set("valInt", 9).set("valStr", "New_9")) - ); + if (invertedWaitOrder) { + assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( + () -> kvView.put(null, Tuple.create().set("key", 8), Tuple.create().set("valInt", 8).set("valStr", "New_8")) + ); + + assertPossibleDeadLockExceptionOnReadWriteSingleRowOperation( + () -> kvView.put(null, Tuple.create().set("key", 9), Tuple.create().set("valInt", 9).set("valStr", "New_9")) + ); + } Publisher publisher1 = new RollbackTxOnErrorPublisher<>( tx, diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItDmlTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItDmlTest.java index f2011008192c..1ce5c94349d6 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItDmlTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/ItDmlTest.java @@ -565,22 +565,22 @@ public void scanExecutedWithinGivenTransaction() { sql("CREATE TABLE test (id int primary key, val int)"); Transaction olderTx = CLUSTER.aliveNode().transactions().begin(); - Transaction tx = CLUSTER.aliveNode().transactions().begin(); + Transaction newerTx = CLUSTER.aliveNode().transactions().begin(); - sql(tx, "INSERT INTO test VALUES (0, 0)"); + sql(olderTx, "INSERT INTO test VALUES (0, 0)"); // just inserted row should be visible within the same transaction - assertEquals(1, sql(tx, "select * from test").size()); + assertEquals(1, sql(olderTx, "select * from test").size()); // just inserted row should not be visible until related transaction is committed assertEquals(0, sql(CLUSTER.aliveNode().transactions().begin(new TransactionOptions().readOnly(true)), "select * from test").size()); - CompletableFuture selectFut = runAsync(() -> sql(olderTx, "select * from test").size()); + CompletableFuture selectFut = runAsync(() -> sql(newerTx, "select * from test").size()); assertFalse(selectFut.isDone()); - tx.commit(); + olderTx.commit(); assertThat(selectFut, willCompleteSuccessfully()); @@ -589,7 +589,7 @@ public void scanExecutedWithinGivenTransaction() { assertEquals(1, sql(CLUSTER.aliveNode().transactions().begin(new TransactionOptions().readOnly(true)), "select * from test").size()); - olderTx.commit(); + newerTx.commit(); } @Test diff --git a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/systemviews/ItLocksSystemViewTest.java b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/systemviews/ItLocksSystemViewTest.java index 0d965acca314..5e6ede131a7a 100644 --- a/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/systemviews/ItLocksSystemViewTest.java +++ b/modules/sql-engine/src/integrationTest/java/org/apache/ignite/internal/sql/engine/systemviews/ItLocksSystemViewTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.sql.engine.systemviews; +import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl; import static org.apache.ignite.internal.TestWrappers.unwrapInternalTransaction; import static org.apache.ignite.internal.catalog.commands.CatalogUtils.DEFAULT_VARLEN_LENGTH; import static org.hamcrest.CoreMatchers.hasItem; @@ -34,6 +35,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.ignite.Ignite; +import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.sql.engine.util.MetadataMatcher; import org.apache.ignite.internal.tx.InternalTransaction; import org.apache.ignite.internal.tx.LockMode; @@ -123,7 +125,8 @@ public void testData() { @Test void testLocksViewWorksCorrectlyWhenTxConflict() { - Ignite ignite = CLUSTER.aliveNode(); + IgniteImpl ignite = unwrapIgniteImpl(CLUSTER.aliveNode()); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); ignite.sql().executeScript("CREATE TABLE testTable (accountNumber INT PRIMARY KEY, balance DOUBLE)"); @@ -133,14 +136,20 @@ void testLocksViewWorksCorrectlyWhenTxConflict() { IgniteTransactions igniteTransactions = igniteTx(); - InternalTransaction tx1 = unwrapInternalTransaction(igniteTransactions.begin()); - InternalTransaction tx2 = unwrapInternalTransaction(igniteTransactions.begin()); + InternalTransaction owner = unwrapInternalTransaction(igniteTransactions.begin()); + InternalTransaction waiter = unwrapInternalTransaction(igniteTransactions.begin()); + + if (invertedWaitOrder) { + InternalTransaction tmp = owner; + owner = waiter; + waiter = tmp; + } var table = test.recordView(); - table.upsert(tx2, makeValue(1, 1.0)); + table.upsert(owner, makeValue(1, 1.0)); - var fut = table.upsertAsync(tx1, makeValue(1, 2.0)); + var fut = table.upsertAsync(waiter, makeValue(1, 2.0)); assertFalse(fut.isDone()); @@ -149,19 +158,19 @@ void testLocksViewWorksCorrectlyWhenTxConflict() { // pk lock, row lock, partition lock assertThat(rows.size(), is(3)); - verifyTxIdAndLockMode(rows, tx2.id().toString(), LockMode.X.name()); - verifyTxIdAndLockMode(rows, tx2.id().toString(), LockMode.IX.name()); + verifyTxIdAndLockMode(rows, owner.id().toString(), LockMode.X.name()); + verifyTxIdAndLockMode(rows, owner.id().toString(), LockMode.IX.name()); - tx2.commit(); + owner.commit(); rows = sql("SELECT * FROM SYSTEM.LOCKS"); assertThat(rows.size(), is(3)); - verifyTxIdAndLockMode(rows, tx1.id().toString(), LockMode.X.name()); - verifyTxIdAndLockMode(rows, tx1.id().toString(), LockMode.IX.name()); + verifyTxIdAndLockMode(rows, waiter.id().toString(), LockMode.X.name()); + verifyTxIdAndLockMode(rows, waiter.id().toString(), LockMode.IX.name()); - tx1.commit(); + waiter.commit(); } /** diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxAbstractDistributedTestSingleNode.java b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxAbstractDistributedTestSingleNode.java index c2c57e55ca8a..a5de447c11e3 100644 --- a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxAbstractDistributedTestSingleNode.java +++ b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxAbstractDistributedTestSingleNode.java @@ -203,6 +203,10 @@ public void testImplicitTransactionRetry() { @Test public void testImplicitTransactionTimeout() { + if (!txManager(accounts).lockManager().policy().invertedWaitOrder()) { + return; // Not compatible with inverted wait order. + } + var rv = accounts.recordView(); // Default tx timeout is 30 sec, default implicit transaction retry timeout is also 30 sec. diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxStateLocalMapTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxStateLocalMapTest.java index 4d6487a1bdfe..da6cef2629f2 100644 --- a/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxStateLocalMapTest.java +++ b/modules/table/src/integrationTest/java/org/apache/ignite/distributed/ItTxStateLocalMapTest.java @@ -165,7 +165,8 @@ private void testTransaction(Consumer touchOp, boolean checkAfterTo tx.rollback(); } - if (read) { + if (read && commit) { + // Unlock only optimization path. checkLocalTxStateOnNodes(tx.id(), null); } else { checkLocalTxStateOnNodes( @@ -179,7 +180,7 @@ private void testTransaction(Consumer touchOp, boolean checkAfterTo } } - private void checkLocalTxStateOnNodes(UUID txId, TxStateMeta expected) { + private void checkLocalTxStateOnNodes(UUID txId, @Nullable TxStateMeta expected) { checkLocalTxStateOnNodes(txId, expected, IntStream.range(0, NODES).boxed().collect(toList())); } diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/ItOperationRetryTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/ItOperationRetryTest.java index fee0ee906e2a..68857b65b748 100644 --- a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/ItOperationRetryTest.java +++ b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/ItOperationRetryTest.java @@ -105,6 +105,11 @@ protected void customizeInitParameters(InitParametersBuilder builder) { @Test public void testLockExceptionRetry() { + IgniteImpl ignite = node0(); + if (!ignite.txManager().lockManager().policy().invertedWaitOrder()) { + return; // Not compatible with inverted wait order. + } + IgniteImpl leaseholderNode = findLeaseholderNode(testPartitionGroupId()); IgniteImpl otherNode = findNonLeaseholderNode(leaseholderNode.name()); @@ -227,6 +232,11 @@ public void retryImplicitTransactionsDueToReplicaMissTest() { @Test public void retryAfterLockFailureInSameTransaction() { + IgniteImpl ignite = node0(); + if (!ignite.txManager().lockManager().policy().invertedWaitOrder()) { + return; // Not compatible with inverted wait order. + } + Transaction tx1 = node(0).transactions().begin(); Transaction tx2 = node(0).transactions().begin(); diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionInflights.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionInflights.java new file mode 100644 index 000000000000..cd00cddd1c45 --- /dev/null +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionInflights.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.table.distributed.replicator; + +import static java.util.concurrent.atomic.AtomicLongFieldUpdater.newUpdater; +import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLongFieldUpdater; +import java.util.function.Predicate; +import org.apache.ignite.internal.partition.replicator.network.replication.RequestType; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.TestOnly; + +/** + * The class is responsible to track partition enlistment operations in a thread safe way. + * Its main purpose is to ensure absence of data races in case of concurrent transaction rollback and partition enlistment operation. + * + *

Partition operations register itself using {@link #addInflight(UUID, Predicate, RequestType)} method. + * + *

Before transaction cleanup {@link #lockForCleanup(UUID)} is called, which prevents enlistment of new operations and ensures all + * current operations are completed. + */ +public class PartitionInflights { + /** Hint for maximum concurrent txns. */ + private static final int MAX_CONCURRENT_TXNS_HINT = 1024; + + /** Field updater for inflights. */ + private static final AtomicLongFieldUpdater INFLIGHTS_UPDATER = newUpdater(CleanupContext.class, "inflights"); + + /** Txn contexts. */ + private final ConcurrentHashMap txCtxMap = new ConcurrentHashMap<>(MAX_CONCURRENT_TXNS_HINT); + + /** + * Registers the inflight for a transaction. + * + * @param txId The transaction id. + * @param enlistPred A predicate to test enlistment possibility under a transaction lock. + * @param requestType Request type. + * + * @return Cleanup context. + */ + @Nullable CleanupContext addInflight(UUID txId, Predicate enlistPred, RequestType requestType) { + boolean[] res = {true}; + + CleanupContext ctx0 = txCtxMap.compute(txId, (uuid, ctx) -> { + if (ctx == null) { + ctx = new CleanupContext(); + } + + if (ctx.finishFut != null || enlistPred.test(txId)) { + res[0] = false; + } else { + INFLIGHTS_UPDATER.incrementAndGet(ctx); + if (requestType.isWrite()) { + ctx.hasWrites = true; + } + } + + return ctx; + }); + + return res[0] ? ctx0 : null; + } + + /** + * Runs a closure under a transaction lock. + * + * @param txId Transaction id. + * @param r Runnable. + */ + void runClosure(UUID txId, Runnable r) { + txCtxMap.compute(txId, (uuid, ctx) -> { + r.run(); + + return ctx; + }); + } + + /** + * Unregisters the inflight for a transaction. + * + * @param ctx Cleanup context. + */ + static void removeInflight(CleanupContext ctx) { + long val = INFLIGHTS_UPDATER.decrementAndGet(ctx); + + if (ctx.finishFut != null && val == 0) { + // If finishFut is null, counter can only go down. + ctx.finishFut.complete(null); + } + } + + /** + * Locks a transaction for cleanup. This prevents new enlistments into the transaction. + * + * @param txId Transaction id. + * @return The context. + */ + @Nullable CleanupContext lockForCleanup(UUID txId) { + return txCtxMap.compute(txId, (uuid, ctx) -> { + if (ctx == null) { + return null; + } + + if (ctx.finishFut == null) { + ctx.finishFut = INFLIGHTS_UPDATER.get(ctx) == 0 ? nullCompletedFuture() : new CompletableFuture<>(); + + // Avoiding a data race with a concurrent decrementing thread, which might not see finishFut publication. + if (INFLIGHTS_UPDATER.get(ctx) == 0 && !ctx.finishFut.isDone()) { + ctx.finishFut = nullCompletedFuture(); + } + } + + return ctx; + }); + } + + /** + * Cleanup inflights context for this transaction. + * + * @param uuid Tx id. + */ + void erase(UUID uuid) { + txCtxMap.remove(uuid); + } + + /** + * Check if the inflights map contains a given transaction. + * + * @param txId Tx id. + * @return {@code True} if contains. + */ + public boolean contains(UUID txId) { + return txCtxMap.containsKey(txId); + } + + /** + * Shared cleanup context. + */ + public static class CleanupContext { + /** An enlistment guard. Not null value means enlistments are not allowed any more. */ + volatile CompletableFuture finishFut; + /** Inflights counter. */ + volatile long inflights = 0; + /** Flag to test if a transaction has writes. If no writes, cleanup message will be skipped. */ + volatile boolean hasWrites = false; + } + + @TestOnly + public ConcurrentHashMap map() { + return txCtxMap; + } +} diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java index 17420561b210..a6b7be17a3db 100644 --- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/replicator/PartitionReplicaListener.java @@ -33,6 +33,7 @@ import static org.apache.ignite.internal.partition.replicator.network.replication.RequestType.RW_REPLACE; import static org.apache.ignite.internal.partition.replicator.network.replication.RequestType.RW_SCAN; import static org.apache.ignite.internal.replicator.message.ReplicaMessageUtils.toZonePartitionIdMessage; +import static org.apache.ignite.internal.table.distributed.replicator.PartitionInflights.removeInflight; import static org.apache.ignite.internal.table.distributed.replicator.RemoteResourceIds.cursorId; import static org.apache.ignite.internal.tx.TransactionErrors.finishedTransactionErrorCode; import static org.apache.ignite.internal.tx.TransactionErrors.finishedTransactionErrorMessage; @@ -82,10 +83,8 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Predicate; @@ -183,10 +182,12 @@ import org.apache.ignite.internal.table.distributed.TableSchemaAwareIndexStorage; import org.apache.ignite.internal.table.distributed.TableUtils; import org.apache.ignite.internal.table.distributed.index.IndexMetaStorage; +import org.apache.ignite.internal.table.distributed.replicator.PartitionInflights.CleanupContext; import org.apache.ignite.internal.table.distributed.replicator.handlers.BuildIndexReplicaRequestHandler; import org.apache.ignite.internal.table.distributed.replicator.handlers.ReadOnlyReplicaRequestHandler; import org.apache.ignite.internal.table.distributed.replicator.handlers.ReplicaRequestHandler; import org.apache.ignite.internal.table.distributed.replicator.handlers.ReplicaRequestHandlers; +import org.apache.ignite.internal.table.distributed.replicator.handlers.ReplicaRequestHandlers.Builder; import org.apache.ignite.internal.table.distributed.replicator.handlers.ScanCloseRequestHandler; import org.apache.ignite.internal.table.metrics.ReadWriteMetricSource; import org.apache.ignite.internal.tx.DelayedAckException; @@ -304,7 +305,8 @@ public class PartitionReplicaListener implements ReplicaTableProcessor { private final Supplier> indexesLockers; - private final ConcurrentMap txCleanupReadyFutures = new ConcurrentHashMap<>(); + /** Used to handle race between concurrent rollback and enlist. */ + private final PartitionInflights partitionInflights = new PartitionInflights(); /** Cleanup futures. */ private final ConcurrentHashMap> rowCleanupMap = new ConcurrentHashMap<>(); @@ -428,7 +430,7 @@ public PartitionReplicaListener( reliableCatalogVersions = new ReliableCatalogVersions(schemaSyncService, catalogService); raftCommandApplicator = new ReplicationRaftCommandApplicator(raftCommandRunner, replicationGroupId); - ReplicaRequestHandlers.Builder handlersBuilder = new ReplicaRequestHandlers.Builder(); + Builder handlersBuilder = new Builder(); handlersBuilder.addHandler( PartitionReplicationMessageGroup.GROUP_TYPE, @@ -552,10 +554,8 @@ private CompletableFuture processRequest(ReplicaRequest request, ReplicaPrima if (request instanceof ReadWriteReplicaRequest) { var req = (ReadWriteReplicaRequest) request; - // Saving state is not needed for full transactions. - if (!req.full()) { - replicaTouch(req.transactionId(), req.coordinatorId(), req.commitPartitionId().asZonePartitionId(), req.txLabel()); - } + // Saving state for full transactions. This is required for implicit kill to work properly. + replicaTouch(req.transactionId(), req.coordinatorId(), req.commitPartitionId().asZonePartitionId(), req.txLabel()); } if (request instanceof GetEstimatedSizeRequest) { @@ -1529,33 +1529,6 @@ private CompletableFuture continueIndexLookup( } private CompletableFuture processTableWriteIntentSwitchAction(TableWriteIntentSwitchReplicaRequest request) { - TxStateMeta txStateMeta = txManager.stateMeta(request.txId()); - - if (txStateMeta != null && txStateMeta.txState() == ABORTED) { - Throwable cause = txStateMeta.lastException(); - boolean isFinishedDueToTimeout = txStateMeta.isFinishedDueToTimeoutOrFalse(); - boolean isFinishedDueToError = !isFinishedDueToTimeout - && txStateMeta.lastExceptionErrorCode() != null; - Throwable publicCause = isFinishedDueToError ? cause : null; - Integer causeErrorCode = txStateMeta.lastExceptionErrorCode(); - - // At this point the transaction is marked as finished by ReplicaTxFinishMarker#markFinished, preventing new locks to appear. - // Safe to invalidate waiters, which otherwise will block the cleanup process. - // Using non-retriable exception intentionally to prevent unnecessary retries. - lockManager.failAllWaiters(request.txId(), new TransactionException( - finishedTransactionErrorCode(isFinishedDueToTimeout, isFinishedDueToError), - format("Can't acquire a lock because {} [{}].", - finishedTransactionErrorMessage( - isFinishedDueToTimeout, - isFinishedDueToError, - causeErrorCode, - publicCause != null - ).toLowerCase(Locale.ROOT), - formatTxInfo(request.txId(), txManager)), - publicCause - )); - } - return awaitCleanupReadyFutures(request.txId()) .thenApply(res -> { if (res.shouldApplyWriteIntent()) { @@ -1567,35 +1540,49 @@ private CompletableFuture processTableWriteIntentSwitchAction(Tab } private CompletableFuture awaitCleanupReadyFutures(UUID txId) { - AtomicBoolean cleanupNeeded = new AtomicBoolean(true); - AtomicReference> cleanupReadyFutureRef = new AtomicReference<>(nullCompletedFuture()); + CleanupContext cleanupContext = partitionInflights.lockForCleanup(txId); - txCleanupReadyFutures.compute(txId, (id, txCleanupState) -> { - // Cleanup operations (both read and update) aren't registered in two cases: - // - there were no actions in the transaction - // - write intent switch is being executed on the new primary (the primary has changed after write intent appeared) - // Both cases are expected to happen extremely rarely so we are fine to force the write intent switch. - - // The reason for the forced switch is that otherwise write intents would not be switched (if there is no volatile state and - // txCleanupState.hadWrites() returns false). - boolean forceCleanup = txCleanupState == null || !txCleanupState.hadAnyOperations(); - - if (txCleanupState == null) { - return null; - } - - cleanupNeeded.set(txCleanupState.hadWrites() || forceCleanup); + TxStateMeta txStateMeta = txManager.stateMeta(txId); - CompletableFuture fut = txCleanupState.lockAndAwaitInflights(); - cleanupReadyFutureRef.set(fut); + // Perform waiters fail after inflights barrier. + if (txStateMeta != null && txStateMeta.txState() == ABORTED) { + Throwable cause = txStateMeta.lastException(); + boolean isFinishedDueToTimeout = txStateMeta.isFinishedDueToTimeoutOrFalse(); + boolean isFinishedDueToError = !isFinishedDueToTimeout + && txStateMeta.lastExceptionErrorCode() != null; + Throwable publicCause = isFinishedDueToError ? cause : null; + Integer causeErrorCode = txStateMeta.lastExceptionErrorCode(); - return txCleanupState; - }); + // At this point the transaction is marked as finished by ReplicaTxFinishMarker#markFinished, preventing new locks to appear. + // Safe to invalidate waiters, which otherwise will block the cleanup process. + // Using non-retriable exception intentionally to prevent unnecessary retries. + // Killed state will be propagated in the cause. + partitionInflights.runClosure(txId, () -> { + lockManager.failAllWaiters(txId, new TransactionException( + finishedTransactionErrorCode(isFinishedDueToTimeout, isFinishedDueToError), + format("Can't acquire a lock because {} [{}].", + finishedTransactionErrorMessage( + isFinishedDueToTimeout, + isFinishedDueToError, + causeErrorCode, + publicCause != null + ).toLowerCase(Locale.ROOT), + formatTxInfo(txId, txManager)), + publicCause + )); + }); + } - return cleanupReadyFutureRef.get() - .thenApplyAsync(v -> new FuturesCleanupResult(cleanupNeeded.get()), txManager.writeIntentSwitchExecutor()) - // TODO https://issues.apache.org/jira/browse/IGNITE-27904 proper cleanup. - .whenComplete((v, e) -> txCleanupReadyFutures.remove(txId)); + if (cleanupContext == null) { + return CompletableFuture.supplyAsync(() -> new FuturesCleanupResult(true), txManager.writeIntentSwitchExecutor()); + } else { + return cleanupContext.finishFut + .thenApplyAsync(v -> new FuturesCleanupResult(cleanupContext.hasWrites), txManager.writeIntentSwitchExecutor()) + // TODO https://issues.apache.org/jira/browse/IGNITE-27904 proper cleanup. + .whenComplete((v, e) -> { + partitionInflights.erase(txId); + }); + } } private void applyWriteIntentSwitchCommandLocally(WriteIntentSwitchReplicaRequestBase request) { @@ -1695,36 +1682,25 @@ private CompletableFuture appendTxCommand( Supplier> op ) { if (full) { - return op.get().whenComplete((v, th) -> { + AtomicReference> futRef = new AtomicReference<>(); + + partitionInflights.runClosure(txId, () -> futRef.set(op.get())); + + return futRef.get().whenComplete((v, th) -> { // Fast unlock. releaseTxLocks(txId); + // Drop volatile state. + txManager.updateTxMeta(txId, ignored -> null); }); } - AtomicBoolean inflightStarted = new AtomicBoolean(false); - - TxCleanupReadyState txCleanupReadyState = txCleanupReadyFutures.compute(txId, (id, txCleanupState) -> { - // First check whether the transaction has already been finished. - // And complete cleanupReadyFut with exception if it is the case. + // It's important to test partition state under txn cleanup lock to avoid a data race. + @Nullable CleanupContext ctx = partitionInflights.addInflight(txId, uuid -> { TxStateMeta txStateMeta = txManager.stateMeta(txId); + return txStateMeta == null || isFinalState(txStateMeta.txState()) || txStateMeta.txState() == FINISHING; + }, requestType); - if (txStateMeta == null || isFinalState(txStateMeta.txState()) || txStateMeta.txState() == FINISHING) { - // Don't start inflight. - return txCleanupState; - } - - // Otherwise start new inflight in txCleanupState. - if (txCleanupState == null) { - txCleanupState = new TxCleanupReadyState(); - } - - boolean started = txCleanupState.startInflight(requestType); - inflightStarted.set(started); - - return txCleanupState; - }); - - if (!inflightStarted.get()) { + if (ctx == null) { TxStateMeta txStateMeta = txManager.stateMeta(txId); TxState txState = txStateMeta == null ? null : txStateMeta.txState(); @@ -1752,32 +1728,36 @@ private CompletableFuture appendTxCommand( )); } - CompletableFuture fut = op.get(); + try { + AtomicReference> futRef = new AtomicReference<>(); - // If inflightStarted then txCleanupReadyState is not null. - requireNonNull(txCleanupReadyState, "txCleanupReadyState cannot be null here."); + partitionInflights.runClosure(txId, () -> futRef.set(op.get())); - fut.whenComplete((v, th) -> { - if (th != null) { - txCleanupReadyState.completeInflight(txId); - } else { - if (v instanceof ReplicaResult) { - ReplicaResult res = (ReplicaResult) v; + futRef.get().whenComplete((v, th) -> { + if (th != null) { + removeInflight(ctx); + } else { + if (v instanceof ReplicaResult) { + ReplicaResult res = (ReplicaResult) v; - if (res.applyResult().replicationFuture() != null) { - res.applyResult().replicationFuture().whenComplete((v0, th0) -> { - txCleanupReadyState.completeInflight(txId); - }); + if (res.applyResult().replicationFuture() != null) { + res.applyResult().replicationFuture().whenComplete((v0, th0) -> { + removeInflight(ctx); + }); + } else { + removeInflight(ctx); + } } else { - txCleanupReadyState.completeInflight(txId); + removeInflight(ctx); } - } else { - txCleanupReadyState.completeInflight(txId); } - } - }); + }); - return fut; + return futRef.get(); + } catch (Throwable err) { + removeInflight(ctx); + throw err; + } } /** @@ -2981,6 +2961,7 @@ private CompletableFuture processSingleEntryAction(ReadWriteSingl }); } case RW_UPSERT: { + // TODO IGNITE-28450 Acquire an X lock for PK. return resolveRowByPk(extractPk(searchRow), txId, (rowId, row, lastCommitTime) -> { boolean insert = rowId == null; @@ -3842,107 +3823,6 @@ private static ZonePartitionIdMessage replicationGroupIdMessage(ZonePartitionId return toZonePartitionIdMessage(REPLICA_MESSAGES_FACTORY, groupId); } - /** - * Class that stores a counter of inflight operations for a transaction. - * - *

Synchronization model: - *

    - *
  • {@code hadAnyOperations}, {@code hadWrites} — plain fields, only accessed inside {@code compute()} critical section.
    • - *
  • {@code inflightOperationsCount} — {@link AtomicInteger}, cross-thread safe.
    • - *
  • {@code completionFuture} — volatile, written from {@code compute()}, read cross-thread. - * Non-null value also serves as the "locked" indicator (no new inflights accepted).
    • - *
- */ - private static class TxCleanupReadyState { - // Only accessed inside compute() critical section. - boolean hadAnyOperations = false; - boolean hadWrites = false; - - final AtomicInteger inflightOperationsCount = new AtomicInteger(0); - - // Non-null means locked (no new inflights accepted). Written from compute(), read cross-thread. - volatile CompletableFuture completionFuture = null; - - // Should be called inside critical section on transaction. - boolean hadAnyOperations() { - return hadAnyOperations; - } - - // Should be called inside critical section on transaction. - boolean hadWrites() { - return hadWrites; - } - - // Should be called inside critical section on transaction. - CompletableFuture lockAndAwaitInflights() { - CompletableFuture f = completionFuture; - - if (f != null) { - return f; // Already locked. - } - - if (inflightOperationsCount.get() == 0) { - f = nullCompletedFuture(); - completionFuture = f; - return f; - } - - f = new CompletableFuture<>(); - completionFuture = f; - - // Recheck: a cross-thread completeInflight() may have decremented to 0 - // before seeing completionFuture != null. - if (inflightOperationsCount.get() == 0) { - f.complete(null); - } - - return f; - } - - // Should be called inside critical section on transaction. - boolean startInflight(RequestType requestType) { - if (completionFuture != null) { - return false; - } - - hadAnyOperations = true; - - if (requestType.isWrite()) { - hadWrites = true; - } - - inflightOperationsCount.incrementAndGet(); - - return true; - } - - // Cross-thread. - void completeInflight(UUID txId) { - int remaining = inflightOperationsCount.decrementAndGet(); - - if (remaining < 0) { - LOG.error("Removed inflight when there were no inflights [txId={}]", txId); - } - - if (remaining == 0) { - completeFutureIfAny(); - } - } - - private void completeFutureIfAny() { - CompletableFuture f = completionFuture; - - if (f == null || f.isDone()) { - return; - } - - // Double check inflightOperationsCount after locked, because we are outside of critical section. - if (inflightOperationsCount.get() == 0) { - f.complete(null); - } - } - } - @Override public void onShutdown() { if (!stopGuard.compareAndSet(false, true)) { diff --git a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/storage/InternalTableImpl.java b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/storage/InternalTableImpl.java index 0108cc1531b3..a0f190f93d6f 100644 --- a/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/storage/InternalTableImpl.java +++ b/modules/table/src/main/java/org/apache/ignite/internal/table/distributed/storage/InternalTableImpl.java @@ -56,10 +56,6 @@ import static org.apache.ignite.internal.util.ExceptionUtils.withCause; import static org.apache.ignite.internal.util.FastTimestamps.coarseCurrentTimeMillis; import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR; -import static org.apache.ignite.lang.ErrorGroups.PlacementDriver.PRIMARY_REPLICA_AWAIT_ERR; -import static org.apache.ignite.lang.ErrorGroups.PlacementDriver.PRIMARY_REPLICA_AWAIT_TIMEOUT_ERR; -import static org.apache.ignite.lang.ErrorGroups.Replicator.GROUP_OVERLOADED_ERR; -import static org.apache.ignite.lang.ErrorGroups.Replicator.REPLICA_ABSENT_ERR; import static org.apache.ignite.lang.ErrorGroups.Replicator.REPLICA_MISS_ERR; import static org.apache.ignite.lang.ErrorGroups.Replicator.REPLICA_UNAVAILABLE_ERR; import static org.apache.ignite.lang.ErrorGroups.Transactions.ACQUIRE_LOCK_ERR; @@ -130,11 +126,13 @@ import org.apache.ignite.internal.tx.TxStateMeta; import org.apache.ignite.internal.tx.impl.TransactionInflights; import org.apache.ignite.internal.util.CollectionUtils; +import org.apache.ignite.internal.util.ExceptionUtils; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.lang.IgniteException; import org.apache.ignite.table.QualifiedName; import org.apache.ignite.table.QualifiedNameHelper; import org.apache.ignite.tx.RetriableReplicaRequestException; +import org.apache.ignite.tx.RetriableTransactionException; import org.apache.ignite.tx.TransactionException; import org.jetbrains.annotations.Nullable; @@ -651,32 +649,8 @@ private CompletableFuture trackingInvoke( if (req.isWrite()) { // Track only write requests from explicit transactions. if (!tx.remote() && !transactionInflights.addInflight(tx.id())) { - TxStateMeta txStateMeta = txManager.stateMeta(tx.id()); - Throwable cause = txStateMeta == null ? null : txStateMeta.lastException(); - boolean isFinishedDueToTimeout = txStateMeta == null - ? tx.isRolledBackWithTimeoutExceeded() - : txStateMeta.isFinishedDueToTimeoutOrFalse(); - boolean isFinishedDueToError = !isFinishedDueToTimeout - && txStateMeta != null - && txStateMeta.lastExceptionErrorCode() != null; - Throwable publicCause = isFinishedDueToError ? cause : null; - Integer causeErrorCode = txStateMeta == null ? null : txStateMeta.lastExceptionErrorCode(); - int code = finishedTransactionErrorCode(isFinishedDueToTimeout, isFinishedDueToError); - - return failedFuture( - new TransactionException(code, format( - finishedTransactionErrorMessage( - isFinishedDueToTimeout, - isFinishedDueToError, - causeErrorCode, - publicCause != null - ) - + " [tableName={}, partId={}, txState={}, timeoutExceeded={}].", - tableName, - partId, - tx.state(), - isFinishedDueToTimeout - ), publicCause)); + // TODO IGNITE-28461 fail fast if TxContext.err != null. + return failedFuture(tx.enlistFailedException()); } return replicaSvc.invoke(enlistment.primaryNodeConsistentId(), request).thenApply(res -> { @@ -1194,6 +1168,7 @@ private CompletableFuture updateAllWithRetry( @Nullable Long txStartTs ) { InternalTransaction tx = txManager.beginImplicitRw(observableTimestampTracker); + ZonePartitionId replicationGroupId = targetReplicationGroupId(partition); assert rows.stream().allMatch(row -> partitionId(row) == partition) : "Invalid batch for partition " + partition; @@ -2326,16 +2301,7 @@ private ReplicaRequest upsertAllInternal( * @return True if retrying is possible, false otherwise. */ private static boolean exceptionAllowsImplicitTxRetry(Throwable e) { - return matchAny( - unwrapCause(e), - ACQUIRE_LOCK_ERR, - GROUP_OVERLOADED_ERR, - REPLICA_MISS_ERR, - REPLICA_UNAVAILABLE_ERR, - REPLICA_ABSENT_ERR, - PRIMARY_REPLICA_AWAIT_ERR, - PRIMARY_REPLICA_AWAIT_TIMEOUT_ERR - ); + return ExceptionUtils.hasCause(e, RetriableTransactionException.class); } private CompletableFuture awaitPrimaryReplica(ZonePartitionId replicationGroupId, HybridTimestamp timestamp) { diff --git a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/TxAbstractTest.java b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/TxAbstractTest.java index dee1239e313d..7510fb565ace 100644 --- a/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/TxAbstractTest.java +++ b/modules/table/src/testFixtures/java/org/apache/ignite/internal/table/TxAbstractTest.java @@ -21,7 +21,7 @@ import static java.util.concurrent.CompletableFuture.completedFuture; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; -import static org.apache.ignite.internal.testframework.IgniteTestUtils.assertThrowsWithCause; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.ensureFutureNotCompleted; import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willBe; @@ -52,17 +52,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.Flow; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; @@ -88,7 +84,6 @@ import org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener; import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.internal.tx.InternalTransaction; -import org.apache.ignite.internal.tx.InternalTxOptions; import org.apache.ignite.internal.tx.Lock; import org.apache.ignite.internal.tx.LockException; import org.apache.ignite.internal.tx.LockManager; @@ -417,28 +412,32 @@ public void testLockOrdering() throws InterruptedException { assertTrue(tx2.id().compareTo(tx3.id()) < 0); assertTrue(tx1.id().compareTo(tx2.id()) < 0); + boolean invertedWaitOrder = txManager(accounts).lockManager().policy().invertedWaitOrder(); + if (invertedWaitOrder) { + InternalTransaction tmp = tx1; + tx1 = tx4; + tx4 = tmp; + + tmp = tx2; + tx2 = tx3; + tx3 = tmp; + } + RecordView acc0 = accounts.recordView(); RecordView acc2 = accounts.recordView(); RecordView acc3 = accounts.recordView(); RecordView acc4 = accounts.recordView(); - acc0.upsert(tx4, makeValue(1, 100.)); + acc0.upsert(tx1, makeValue(1, 100.)); CompletableFuture fut = acc3.upsertAsync(tx2, makeValue(1, 300.)); + ensureFutureNotCompleted(fut, 100); - Thread.sleep(100); - - assertFalse(fut.isDone()); - - CompletableFuture fut2 = acc4.upsertAsync(tx2, makeValue(1, 400.)); - - Thread.sleep(100); - - assertFalse(fut2.isDone()); - - CompletableFuture fut3 = acc2.upsertAsync(tx3, makeValue(1, 200.)); + CompletableFuture fut2 = acc4.upsertAsync(tx3, makeValue(1, 400.)); + ensureFutureNotCompleted(fut2, 100); - assertFalse(fut3.isDone()); + CompletableFuture fut3 = acc2.upsertAsync(tx4, makeValue(1, 200.)); + ensureFutureNotCompleted(fut3, 100); } /** @@ -587,6 +586,11 @@ public void testBatchPutConcurrently() { Transaction tx1 = igniteTransactions.begin(); Transaction tx2 = igniteTransactions.begin(); + boolean invertedWaitOrder = txManager(accounts).lockManager().policy().invertedWaitOrder(); + + Transaction owner = invertedWaitOrder ? tx2 : tx1; + Transaction waiter = invertedWaitOrder ? tx1 : tx2; + log.info("Tx " + tx2); log.info("Tx2 " + tx1); @@ -601,13 +605,13 @@ public void testBatchPutConcurrently() { var table = accounts.recordView(); var table2 = accounts.recordView(); - table2.upsertAll(tx1, rows2); + table2.upsertAll(owner, rows2); - Exception err = assertThrows(Exception.class, () -> table.upsertAll(tx2, rows)); + CompletableFuture fut = table.upsertAllAsync(waiter, rows); + ensureFutureNotCompleted(fut, 100); - assertTransactionLockException(err); - - tx1.commit(); + owner.commit(); + waiter.commit(); } @Test @@ -615,6 +619,11 @@ public void testBatchReadPutConcurrently() throws InterruptedException { InternalTransaction tx1 = (InternalTransaction) igniteTransactions.begin(); InternalTransaction tx2 = (InternalTransaction) igniteTransactions.begin(); + boolean invertedWaitOrder = txManager(accounts).lockManager().policy().invertedWaitOrder(); + + InternalTransaction owner = invertedWaitOrder ? tx2 : tx1; + InternalTransaction waiter = invertedWaitOrder ? tx1 : tx2; + log.info("Tx1 " + tx1); log.info("Tx2 " + tx2); @@ -629,8 +638,8 @@ public void testBatchReadPutConcurrently() throws InterruptedException { keys2.add(makeKey(i)); } - table2.getAll(tx1, keys); - table2.getAll(tx2, keys2); + table2.getAll(owner, keys); + table2.getAll(waiter, keys2); ArrayList rows = new ArrayList<>(); ArrayList rows2 = new ArrayList<>(); @@ -640,15 +649,20 @@ public void testBatchReadPutConcurrently() throws InterruptedException { rows2.add(makeValue(i, 2 * i * 100.)); } - var futUpd2 = table2.upsertAllAsync(tx1, rows2); + var futUpd2 = table2.upsertAllAsync(waiter, rows2); assertTrue(waitForCondition(() -> { boolean lockUpgraded = false; - for (Iterator it = txManager(accounts).lockManager().locks(tx1.id()); it.hasNext(); ) { + for (Iterator it = txManager(accounts).lockManager().locks(waiter.id()); it.hasNext(); ) { Lock lock = it.next(); - lockUpgraded = txManager(accounts).lockManager().waiter(lock.lockKey(), tx1.id()).intendedLockMode() == LockMode.X; + if (lock.lockKey().contextId() == null) { + // Ignore coarse locks. + continue; + } + + lockUpgraded = txManager(accounts).lockManager().waiter(lock.lockKey(), waiter.id()).intendedLockMode() == LockMode.X; if (lockUpgraded) { break; @@ -658,9 +672,7 @@ public void testBatchReadPutConcurrently() throws InterruptedException { return lockUpgraded; }, 3000)); - assertFalse(futUpd2.isDone()); - - assertThrowsWithCause(() -> table.upsertAll(tx2, rows), LockException.class); + ensureFutureNotCompleted(futUpd2, 100); } /** @@ -713,36 +725,6 @@ tx, makeKey(2), makeValue(pair.getSecond().doubleValue("balance") + DELTA)) assertEquals(BALANCE_2 + DELTA, accounts.recordView().get(null, makeKey(2)).doubleValue("balance")); } - @Test - public void testSimpleConflict() { - accounts.recordView().upsert(null, makeValue(1, 100.)); - - Transaction tx1 = igniteTransactions.begin(); - Transaction tx2 = igniteTransactions.begin(); - - var table = accounts.recordView(); - var table2 = accounts.recordView(); - - double val = table.get(tx2, makeKey(1)).doubleValue("balance"); - table2.get(tx1, makeKey(1)).doubleValue("balance"); - - try { - table.upsert(tx2, makeValue(1, val + 1)); - - fail(); - } catch (Exception e) { - // Expected. - } - - table2.upsert(tx1, makeValue(1, val + 1)); - - tx1.commit(); - - tx2.commit(); - - assertEquals(101., accounts.recordView().get(null, makeKey(1)).doubleValue("balance")); - } - @Test public void testCommit() throws TransactionException { InternalTransaction tx = (InternalTransaction) igniteTransactions.begin(); @@ -824,6 +806,11 @@ public void testIncrement() throws TransactionException { Transaction tx1 = igniteTransactions.begin(); Transaction tx2 = igniteTransactions.begin(); + boolean invertedWaitOrder = txManager(accounts).lockManager().policy().invertedWaitOrder(); + + Transaction owner = invertedWaitOrder ? tx2 : tx1; + Transaction waiter = invertedWaitOrder ? tx1 : tx2; + Tuple key = makeKey(1); Tuple val = makeValue(1, 100.); @@ -833,21 +820,26 @@ public void testIncrement() throws TransactionException { var table2 = accounts.recordView(); // Read in tx2 - double valTx = table.get(tx2, key).doubleValue("balance"); + double valTx = table.get(owner, key).doubleValue("balance"); // Read in tx1 - double valTx2 = table2.get(tx1, key).doubleValue("balance"); + double valTx2 = table2.get(waiter, key).doubleValue("balance"); - // Write in tx2 (out of order) - // TODO asch IGNITE-15937 fix exception model. - Exception err = assertThrows(Exception.class, () -> table.upsert(tx2, makeValue(1, valTx + 1))); + CompletableFuture fut = table.upsertAsync(waiter, makeValue(1, valTx + 1)); + ensureFutureNotCompleted(fut, 100); - assertTransactionLockException(err); - - // Write in tx1 - table2.upsert(tx1, makeValue(1, valTx2 + 1)); + try { + table2.upsert(owner, makeValue(1, valTx2 + 1)); + } catch (TransactionException e) { + // Expected. + } - tx1.commit(); + owner.commit(); + try { + waiter.commit(); + } catch (TransactionException e) { + // Expected. + } assertEquals(101., accounts.recordView().get(null, key).doubleValue("balance")); } @@ -986,24 +978,25 @@ public void testGetAllAbort() throws TransactionException { public void testGetAllConflict() throws Exception { accounts.recordView().upsertAll(null, List.of(makeValue(1, 100.), makeValue(2, 200.))); - InternalTransaction tx1 = (InternalTransaction) igniteTransactions.begin(); - InternalTransaction tx2 = (InternalTransaction) igniteTransactions.begin(); + InternalTransaction older = (InternalTransaction) igniteTransactions.begin(); + InternalTransaction younger = (InternalTransaction) igniteTransactions.begin(); + + boolean invertedWaitOrder = txManager(accounts).lockManager().policy().invertedWaitOrder(); RecordView txAcc = accounts.recordView(); RecordView txAcc2 = accounts.recordView(); - txAcc2.upsert(tx1, makeValue(1, 300.)); - txAcc.upsert(tx2, makeValue(2, 400.)); + txAcc2.upsert(older, makeValue(1, 300.)); + txAcc.upsert(younger, makeValue(2, 400.)); - Exception err = assertThrows(Exception.class, () -> txAcc.getAll(tx2, List.of(makeKey(2), makeKey(1)))); - assertTransactionLockException(err); + // Triggers a conflict, which invalidates younger transaction. + txAcc.getAllAsync(invertedWaitOrder ? younger : older, List.of(makeKey(2), makeKey(1))); + assertTrue(waitForCondition(() -> TxState.ABORTED == younger.state(), 5_000), younger.state().toString()); - validateBalance(txAcc2.getAll(tx1, List.of(makeKey(2), makeKey(1))), 200., 300.); - validateBalance(txAcc2.getAll(tx1, List.of(makeKey(1), makeKey(2))), 300., 200.); + validateBalance(txAcc2.getAll(older, List.of(makeKey(2), makeKey(1))), 200., 300.); + validateBalance(txAcc2.getAll(older, List.of(makeKey(1), makeKey(2))), 300., 200.); - assertTrue(waitForCondition(() -> TxState.ABORTED == tx2.state(), 5_000), tx2.state().toString()); - - tx1.commit(); + older.commit(); validateBalance(accounts.recordView().getAll(null, List.of(makeKey(2), makeKey(1))), 200., 300.); } @@ -1401,16 +1394,6 @@ public void testCrossTableAsyncKeyValueViewRollback() throws Exception { assertTrue(waitForCondition(() -> lockManager(accounts).isEmpty(), 10_000)); } - @Test - public void testBalance() throws InterruptedException { - doTestSingleKeyMultithreaded(5_000, false); - } - - @Test - public void testLockedTooLong() { - // TODO asch IGNITE-15936 if lock can't be acquired until timeout tx should be rolled back. - } - @Test public void testScan() { doTestScan(null); @@ -1686,147 +1669,6 @@ public void doTestComplexKeyValue(KeyValueView view, @Nullable Tra } } - /** - * Performs a test. - * - * @param duration The duration. - * @param verbose Verbose mode. - * @throws InterruptedException If interrupted while waiting. - */ - private void doTestSingleKeyMultithreaded(long duration, boolean verbose) throws InterruptedException { - int threadsCnt = Runtime.getRuntime().availableProcessors() * 2; - - Thread[] threads = new Thread[threadsCnt]; - - final int accountsCount = threads.length * 10; - - final double initial = 1000; - final double total = accountsCount * initial; - - for (int i = 0; i < accountsCount; i++) { - accounts.recordView().upsert(null, makeValue(i, 1000)); - } - - double total0 = 0; - - for (long i = 0; i < accountsCount; i++) { - double balance = accounts.recordView().get(null, makeKey(i)).doubleValue("balance"); - - total0 += balance; - } - - assertEquals(total, total0, "Total amount invariant is not preserved"); - - CyclicBarrier startBar = new CyclicBarrier(threads.length, () -> log.info("Before test")); - - LongAdder ops = new LongAdder(); - LongAdder fails = new LongAdder(); - - AtomicBoolean stop = new AtomicBoolean(); - - Random r = new Random(); - - AtomicReference firstErr = new AtomicReference<>(); - - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new Runnable() { - @Override - public void run() { - try { - startBar.await(); - } catch (Exception e) { - fail(); - } - - while (!stop.get() && firstErr.get() == null) { - InternalTransaction tx = clientTxManager().beginExplicitRw(timestampTracker, InternalTxOptions.defaults()); - - var table = accounts.recordView(); - - try { - long acc1 = r.nextInt(accountsCount); - - double amount = 100 + r.nextInt(500); - - if (verbose) { - log.info("op=tryGet ts={} id={}", tx.id(), acc1); - } - - double val0 = table.get(tx, makeKey(acc1)).doubleValue("balance"); - - long acc2 = acc1; - - while (acc1 == acc2) { - acc2 = r.nextInt(accountsCount); - } - - if (verbose) { - log.info("op=tryGet ts={} id={}", tx.id(), acc2); - } - - double val1 = table.get(tx, makeKey(acc2)).doubleValue("balance"); - - if (verbose) { - log.info("op=tryPut ts={} id={}", tx.id(), acc1); - } - - table.upsert(tx, makeValue(acc1, val0 - amount)); - - if (verbose) { - log.info("op=tryPut ts={} id={}", tx.id(), acc2); - } - - table.upsert(tx, makeValue(acc2, val1 + amount)); - - tx.commit(); - - ops.increment(); - } catch (Exception e) { - assertTransactionLockException(e); - - tx.rollback(); - - fails.increment(); - } - } - } - }); - - threads[i].setName("Worker-" + i); - threads[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - firstErr.compareAndExchange(null, e); - } - }); - threads[i].start(); - } - - Thread.sleep(duration); - - stop.set(true); - - for (Thread thread : threads) { - thread.join(3_000); - } - - if (firstErr.get() != null) { - throw new IgniteException(INTERNAL_ERR, firstErr.get()); - } - - log.info("After test ops={} fails={}", ops.sum(), fails.sum()); - - total0 = 0; - - for (long i = 0; i < accountsCount; i++) { - double balance = accounts.recordView().get(null, makeKey(i)).doubleValue("balance"); - - total0 += balance; - } - - assertEquals(total, total0, "Total amount invariant is not preserved"); - } - /** * Get a lock manager on a partition leader. * @@ -2188,12 +2030,9 @@ public void testSingleGet() { accountRecordsView.upsert(null, makeValue(1, 100.)); Transaction tx1 = igniteTransactions.begin(); - Transaction tx2 = igniteTransactions.begin(); accountRecordsView.upsert(tx1, makeValue(1, 200.)); - assertThrows(TransactionException.class, () -> accountRecordsView.get(tx2, makeKey(1))); - assertEquals(100., accountRecordsView.get(null, makeKey(1)).doubleValue("balance")); tx1.commit(); @@ -2226,13 +2065,9 @@ public void testBatchSinglePartitionGet() { accountRecordsView.upsertAll(null, keys.stream().map(k -> makeValue(k, 100.)).collect(toList())); Transaction tx1 = igniteTransactions.begin(); - Transaction tx2 = igniteTransactions.begin(); accountRecordsView.upsertAll(tx1, keys.stream().map(k -> makeValue(k, 200.)).collect(toList())); - assertThrows(TransactionException.class, - () -> accountRecordsView.getAll(tx2, keys.stream().map(k -> makeKey(k)).collect(toList()))); - for (Tuple tuple : accountRecordsView.getAll(null, keys.stream().map(k -> makeKey(k)).collect(toList()))) { assertEquals(100., tuple.doubleValue("balance")); } @@ -2283,6 +2118,10 @@ public void testYoungerTransactionWithHigherPriorityWaitsForOlderTransactionComm @ParameterizedTest @EnumSource(TxPriority.class) public void testYoungerTransactionThrowsExceptionIfKeyLockedByOlderTransactionWithSamePriority(TxPriority priority) { + if (!txManager(accounts).lockManager().policy().invertedWaitOrder()) { + return; // Not compatible with inverted wait order. + } + IgniteTransactionsImpl igniteTransactionsImpl = (IgniteTransactionsImpl) igniteTransactions; KeyValueView keyValueView = customers.keyValueView(Long.class, String.class); diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java index 0f08ecd06fb1..949f9e02360c 100644 --- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java +++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/disaster/ItDisasterRecoveryReconfigurationTest.java @@ -317,6 +317,14 @@ void testManualRebalanceIfMajorityIsLostSpecifyPartitions() throws Exception { int anotherPartId = 0; IgniteImpl node0 = igniteImpl(0); + + if (!node0.txManager().lockManager().policy().invertedWaitOrder()) { + // Not compatible with inverted wait order. + // An older transaction can attempt to request a lock after partition reset, because it's not dead, and disrupts test logic. + // TODO https://issues.apache.org/jira/browse/IGNITE-28365 + return; + } + Table table = node0.tables().table(TABLE_NAME); awaitPrimaryReplica(node0, anotherPartId); diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/tx/ItRunInTransactionTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/tx/ItRunInTransactionTest.java index 3390f82d51c8..271382da0e59 100644 --- a/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/tx/ItRunInTransactionTest.java +++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/internal/tx/ItRunInTransactionTest.java @@ -18,7 +18,9 @@ package org.apache.ignite.internal.tx; import static java.lang.String.format; +import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrowWithCauseOrSuppressed; +import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,10 +28,12 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import org.apache.ignite.Ignite; import org.apache.ignite.internal.ClusterPerTestIntegrationTest; +import org.apache.ignite.internal.app.IgniteImpl; import org.apache.ignite.internal.lang.IgniteTriFunction; import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.table.Table; @@ -89,6 +93,11 @@ public void testSync(SyncTestContext ctx) { AtomicInteger cnt = new AtomicInteger(); + IgniteImpl ignite = unwrapIgniteImpl(node(0)); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + + Phaser phaser = new Phaser(2); + CompletableFuture fut = IgniteTestUtils.runAsync(() -> { ignite().transactions().runInTransaction(youngerTx -> { if (cnt.incrementAndGet() == 2) { @@ -97,11 +106,26 @@ public void testSync(SyncTestContext ctx) { ctx.put.apply(ignite(), youngerTx, key2); assertTrue(txId(olderTx).compareTo(txId(youngerTx)) < 0); - // Younger is not allowed to wait for older. - ctx.put.apply(ignite(), youngerTx, key); + + phaser.arriveAndAwaitAdvance(); + + if (invertedWaitOrder) { + // Younger is not allowed to wait for older. + ctx.put.apply(ignite(), youngerTx, key); + } else { + phaser.arriveAndAwaitAdvance(); + } }); }); + phaser.arriveAndAwaitAdvance(); + + if (!invertedWaitOrder) { + // Older will invalidate younger, so commit fails. + ctx.put.apply(ignite(), olderTx, key2); + phaser.arriveAndAwaitAdvance(); + } + assertThat(fut, willThrowWithCauseOrSuppressed(Exception.class, "retry")); assertEquals(2, cnt.get(), "Should retry at least once"); @@ -122,6 +146,11 @@ public void testAsync(AsyncTestContext ctx) { AtomicInteger cnt = new AtomicInteger(); + IgniteImpl ignite = unwrapIgniteImpl(node(0)); + boolean invertedWaitOrder = ignite.txManager().lockManager().policy().invertedWaitOrder(); + + Phaser phaser = new Phaser(2); + CompletableFuture fut = ignite().transactions().runInTransactionAsync(youngerTx -> { if (cnt.incrementAndGet() == 2) { throw new RuntimeException("retry"); @@ -131,10 +160,26 @@ public void testAsync(AsyncTestContext ctx) { assertTrue(txId(olderTx).compareTo(txId(youngerTx)) < 0, "Wrong ordering: old=" + olderTx.toString() + ", new=" + youngerTx.toString()); // Younger is not allowed to wait for older. - return ctx.put.apply(ignite(), youngerTx, key); + phaser.arriveAndAwaitAdvance(); + + if (invertedWaitOrder) { + // Younger is not allowed to wait for older. + return ctx.put.apply(ignite(), youngerTx, key); + } else { + phaser.arriveAndAwaitAdvance(); + return nullCompletedFuture(); + } }); }); + phaser.arriveAndAwaitAdvance(); + + if (!invertedWaitOrder) { + // Older will invalidate younger, so commit fails. + ctx.put.apply(ignite(), olderTx, key2).join(); + phaser.arriveAndAwaitAdvance(); + } + assertThat(fut, willThrowWithCauseOrSuppressed(Exception.class, "retry")); assertEquals(2, cnt.get(), "Should retry at least once"); diff --git a/modules/transactions/src/integrationTest/java/org/apache/ignite/tx/distributed/ItTransactionRecoveryTest.java b/modules/transactions/src/integrationTest/java/org/apache/ignite/tx/distributed/ItTransactionRecoveryTest.java index 0de4de47a59e..b5a156ceab5d 100644 --- a/modules/transactions/src/integrationTest/java/org/apache/ignite/tx/distributed/ItTransactionRecoveryTest.java +++ b/modules/transactions/src/integrationTest/java/org/apache/ignite/tx/distributed/ItTransactionRecoveryTest.java @@ -153,6 +153,8 @@ protected void customizeInitParameters(InitParametersBuilder builder) { public void testMultipleAbandonedTxsAreAborted() throws Exception { TableImpl tbl = unwrapTableImpl(node(0).tables().table(TABLE_NAME)); + boolean invertedWaitOrder = unwrapIgniteImpl(node(0)).txManager().lockManager().policy().invertedWaitOrder(); + var partitionGroupId = new ZonePartitionId(tbl.zoneId(), PART_ID); String leaseholder = waitAndGetLeaseholder(node(0), partitionGroupId); @@ -170,6 +172,8 @@ public void testMultipleAbandonedTxsAreAborted() throws Exception { List txns = new ArrayList<>(); + Transaction waiterTx = invertedWaitOrder ? node(0).transactions().begin() : null; // Older is allowed to wait in WD. + for (int i = 0; i < 10; i++) { InternalTransaction tx = (InternalTransaction) txCrdNode.transactions().begin(); Tuple ignored = view.get(tx, Tuple.create().set("key", 42)); @@ -189,8 +193,11 @@ public void testMultipleAbandonedTxsAreAborted() throws Exception { () -> node(0).cluster().nodes().stream().filter(n -> txCrdNode.id().equals(n.id())).count() == 0, 10_000)); - InternalTransaction conflictTx = (InternalTransaction) node(0).transactions().begin(); - runConflictingTransaction(node(0), conflictTx); + if (!invertedWaitOrder) { + waiterTx = node(0).transactions().begin(); // Younger allowed to wait in WW. + } + + runConflictingTransaction(node(0), waiterTx); // Test if all abandoned transactions are aborted. for (InternalTransaction txn : txns) { @@ -255,6 +262,8 @@ public void testMultipleRecoveryRequestsIssued() throws Exception { public void testAbandonedTxIsAborted() throws Exception { TableImpl tbl = unwrapTableImpl(node(0).tables().table(TABLE_NAME)); + boolean invertedWaitOrder = unwrapIgniteImpl(node(0)).txManager().lockManager().policy().invertedWaitOrder(); + var partitionGroupId = new ZonePartitionId(tbl.zoneId(), PART_ID); String leaseholder = waitAndGetLeaseholder(node(0), partitionGroupId); @@ -267,6 +276,8 @@ public void testAbandonedTxIsAborted() throws Exception { log.info("Transaction coordinator is chosen [node={}].", txCrdNode.name()); + Transaction waiterTx = invertedWaitOrder ? node(0).transactions().begin() : null; // Older is allowed to wait in WD. + UUID orphanTxId = startTransactionAndStopNode(txCrdNode); CompletableFuture recoveryTxMsgCaptureFut = new CompletableFuture<>(); @@ -284,7 +295,10 @@ public void testAbandonedTxIsAborted() throws Exception { return false; }); - runConflictingTransaction(node(0), node(0).transactions().begin()); + if (!invertedWaitOrder) { + waiterTx = node(0).transactions().begin(); // Younger allowed to wait in WW. + } + runConflictingTransaction(node(0), waiterTx); assertThat(recoveryTxMsgCaptureFut, willCompleteSuccessfully()); @@ -557,6 +571,8 @@ public void testSendCommitAndDie() throws Exception { public void testCommitAndDieRecoveryFirst() throws Exception { TableImpl tbl = unwrapTableImpl(node(0).tables().table(TABLE_NAME)); + boolean invertedWaitOrder = unwrapIgniteImpl(node(0)).txManager().lockManager().policy().invertedWaitOrder(); + var partitionGroupId = new ZonePartitionId(tbl.zoneId(), PART_ID); String leaseholder = waitAndGetLeaseholder(node(0), partitionGroupId); @@ -569,6 +585,8 @@ public void testCommitAndDieRecoveryFirst() throws Exception { log.info("Transaction coordinator is chosen [node={}].", txCrdNode.name()); + Transaction waiterTx = invertedWaitOrder ? node(0).transactions().begin() : null; // Older is allowed to wait in WD. + InternalTransaction orphanTx = (InternalTransaction) createRwTransaction(txCrdNode); CompletableFuture finishRequestCaptureFut = new CompletableFuture<>(); @@ -606,9 +624,11 @@ public void testCommitAndDieRecoveryFirst() throws Exception { // The state on the commit partition is still PENDING. assertEquals(TxState.PENDING, txVolatileState(commitPartNode, orphanTx.id())); - Ignite newTxCoord = node(0); + if (!invertedWaitOrder) { + waiterTx = commitPartNode.transactions().begin(); + } - runRwTransactionNoError(newTxCoord, newTxCoord.transactions().begin()); + runConflictingTransaction(commitPartNode, waiterTx); // Trigger resolution on conflicting wait. assertTrue(waitForCondition(() -> txStoredState(commitPartNode, orphanTx.id()) == TxState.ABORTED, 10_000)); @@ -632,6 +652,8 @@ public void testCommitAndDieRecoveryFirst() throws Exception { public void testRecoveryIsTriggeredOnce() throws Exception { TableImpl tbl = unwrapTableImpl(node(0).tables().table(TABLE_NAME)); + boolean invertedWaitOrder = unwrapIgniteImpl(node(0)).txManager().lockManager().policy().invertedWaitOrder(); + var partitionGroupId = new ZonePartitionId(tbl.zoneId(), PART_ID); String leaseholder = waitAndGetLeaseholder(node(0), partitionGroupId); @@ -644,6 +666,8 @@ public void testRecoveryIsTriggeredOnce() throws Exception { log.info("Transaction coordinator is chosen [node={}].", txCrdNode.name()); + Transaction rwTx1 = invertedWaitOrder ? commitPartNode.transactions().begin() : null; + UUID orphanTxId = startTransactionAndStopNode(txCrdNode); log.info("Orphan tx [id={}]", orphanTxId); @@ -668,7 +692,9 @@ public void testRecoveryIsTriggeredOnce() throws Exception { log.info("New transaction coordinator is chosen [node={}].", newCoordNode.name()); // Run RW transaction. - Transaction rwTx1 = commitPartNode.transactions().begin(); + if (!invertedWaitOrder) { + rwTx1 = commitPartNode.transactions().begin(); + } UUID rwTx1Id = ((InternalTransaction) rwTx1).id(); diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/DeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/DeadlockPreventionPolicy.java index 7b5b02ce3163..3ca7fa61a0c5 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/DeadlockPreventionPolicy.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/DeadlockPreventionPolicy.java @@ -19,6 +19,7 @@ import java.util.Comparator; import java.util.UUID; +import org.apache.ignite.internal.tx.impl.TxIdPriorityComparator; import org.jetbrains.annotations.Nullable; /** @@ -27,27 +28,30 @@ * See also {@link org.apache.ignite.internal.tx.impl.HeapLockManager}. */ public interface DeadlockPreventionPolicy { + /** Default comparator. */ + TxIdPriorityComparator TX_ID_PRIORITY_COMPARATOR = new TxIdPriorityComparator(); + /** * No-op policy which does nothing to prevent deadlocks. */ DeadlockPreventionPolicy NO_OP = new DeadlockPreventionPolicy() {}; /** - * Comparator for transaction ids that allows to set transaction priority, if deadlock prevention policy requires this priority. - * The transaction with higher id has lower priority. If this comparator is {@code null} then behavior of any transaction - * in case of conflict depends only on whether this transaction holds a lock or makes a request for lock acquisition. + * A comparator for transaction ids that orders transactions according to their priority. Transactions with higher priority + * will acquire locks first. Also, the priority is used to prevent deadlocks, if a policy supports deadlock prevention. * * @return Transaction id comparator. */ - @Nullable default Comparator txIdComparator() { - return null; + default Comparator txIdComparator() { + return TX_ID_PRIORITY_COMPARATOR; } /** * Timeout (in milliseconds) to wait before aborting a lock attempt that is made by a transaction in case of a conflict - * of this transaction with another one on certain key. If transaction priority is applicable (see {@link #txIdComparator()}) - * then this timeout is applied only for transaction with lower priority. If this method returns {@code 0} this means that + * of this transaction with another one on certain key. If a policy allows deadlock prevention, + * then this timeout is applied only to a waiting transaction. If this method returns {@code 0} this means that * the lock attempt is aborted instantly (timeout is zero). If lesser that {@code 0}, it means that the wait time is infinite. + * TODO IGNITE-28507 make configurable. * * @return Timeout, in milliseconds. */ @@ -56,11 +60,36 @@ default long waitTimeout() { } /** - * Whether transaction priority if used for conflict resolution. + * Invokes fail action on the owner. + * + * @param owner The owner. + */ + default void failAction(UUID owner) { + // No-op. + } + + /** + * Tests if waiter is allowed to wait for owner. + * + * @param waiter The waiter. + * @param owner The owner. + * + * @return Waiter to fail or {@code null} if waiting is allowed. + */ + default @Nullable Waiter allowWait(Waiter waiter, Waiter owner) { + return null; + } + + /** + * Returns {@code true}, if wait order is inverted: high priority (older) transactions are allowed to wait for low priority (younger) + * transactions. + * + *

Must be consistent with {@code allowWait} implementation: for example, if higher priority is allowed to wait for lower priority, + * a search should start from low priority first, to have a chance of finding a valid waiter-owner pair. * - * @return Whether priority is used. + * @return If {@code true} for inverted wait order. */ - default boolean usePriority() { - return txIdComparator() != null; + default boolean invertedWaitOrder() { + return false; } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/InternalTransaction.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/InternalTransaction.java index d821da262869..479f97e1e500 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/InternalTransaction.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/InternalTransaction.java @@ -190,4 +190,8 @@ CompletableFuture finish( default void processDelayedAck(Object val, @Nullable Throwable err) { // No-op. } + + default RuntimeException enlistFailedException() { + return new IllegalStateException("Unexpected call"); + } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/Lock.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/Lock.java index 008363102525..c6144966a858 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/Lock.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/Lock.java @@ -17,12 +17,12 @@ package org.apache.ignite.internal.tx; +import java.util.Objects; import java.util.UUID; import org.apache.ignite.internal.tostring.S; /** Lock. */ public class Lock { - /** Lock key. */ private final LockKey lockKey; @@ -72,6 +72,23 @@ public UUID txId() { return txId; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Lock lock = (Lock) o; + return Objects.equals(lockKey, lock.lockKey) && lockMode == lock.lockMode && Objects.equals(txId, lock.txId); + } + + @Override + public int hashCode() { + return Objects.hash(lockKey, lockMode, txId); + } + @Override public String toString() { return S.toString(this); diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockKey.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockKey.java index bae2005914f8..9bf44e6747f3 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockKey.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockKey.java @@ -17,7 +17,10 @@ package org.apache.ignite.internal.tx; +import static org.apache.ignite.internal.util.IgniteUtils.byteBufferToByteArray; + import java.nio.ByteBuffer; +import java.util.Arrays; import org.apache.ignite.internal.tostring.S; import org.apache.ignite.internal.util.HashUtils; @@ -88,6 +91,14 @@ public int hashCode() { @Override public String toString() { - return S.toString(LockKey.class, this, "ctx", contextId, "key", key); + return S.toString(LockKey.class, this, "ctx", contextId, "key", dump(key)); + } + + private static String dump(Object key) { + if (key instanceof ByteBuffer) { + return Arrays.toString(byteBufferToByteArray((ByteBuffer) key)); + } + + return key.toString(); } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockManager.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockManager.java index 04b404a41afe..a957678f2821 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockManager.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/LockManager.java @@ -35,6 +35,8 @@ public interface LockManager extends EventProducer>> 16)) & 0x7fffffff; + } + + /** + * Compares transaction IDs based on their associated priorities and the IDs themselves. + * The IDs with higher priorities are sorted first. + * If the priorities are equal, the IDs are sorted by their natural order, based on tx begin timestamp, + * which implies older transactions having more priority than younger. + * + * @param id1 id1. + * @param id2 id2. + * + * @return The result. + */ + public static int compare(UUID id1, UUID id2) { + TxPriority priority1 = priority(id1); + TxPriority priority2 = priority(id2); + + int priorityComparison = priority1.compareTo(priority2); + + if (priorityComparison == 0) { + return id1.compareTo(id2); + } else { + return priorityComparison * -1; // Reverse order. + } + } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/DeadlockPreventionPolicyImpl.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/DeadlockPreventionPolicyImpl.java deleted file mode 100644 index 1e017b1e7f50..000000000000 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/DeadlockPreventionPolicyImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.tx.impl; - -import java.util.Comparator; -import java.util.UUID; -import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; -import org.jetbrains.annotations.Nullable; - -/** - * Implementation of {@link DeadlockPreventionPolicy} that allows to set its parameters directly in the constructor, so it is able to - * provide different policies' behaviour such as wait-die, reversed wait-die, timeout wait, no-wait, etc. - */ -public class DeadlockPreventionPolicyImpl implements DeadlockPreventionPolicy { - private final Comparator txIdComparator; - - private final long waitTimeout; - - /** - * Constructor. - * - * @param txIdComparator Comparator name as {@link TxIdComparators} element. - * @param waitTimeout Wait timeout. - */ - public DeadlockPreventionPolicyImpl(TxIdComparators txIdComparator, long waitTimeout) { - switch (txIdComparator) { - case NATURAL: { - this.txIdComparator = new TxIdPriorityComparator(); - break; - } - case REVERSED: { - this.txIdComparator = new TxIdPriorityComparator().reversed(); - break; - } - case NONE: { - this.txIdComparator = null; - break; - } - default: { - throw new IllegalArgumentException("Unknown comparator type: " + txIdComparator); - } - } - - this.waitTimeout = waitTimeout; - } - - @Override - public @Nullable Comparator txIdComparator() { - return txIdComparator; - } - - @Override - public long waitTimeout() { - return waitTimeout; - } - - /** - * Enum of names of transaction ID comparators. - */ - public enum TxIdComparators { - NATURAL, - REVERSED, - NONE - } -} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/HeapLockManager.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/HeapLockManager.java index b9ae7002c0f9..ee29f57c8eba 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/HeapLockManager.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/HeapLockManager.java @@ -20,6 +20,10 @@ import static java.util.Collections.emptyList; import static java.util.concurrent.CompletableFuture.completedFuture; import static java.util.concurrent.CompletableFuture.failedFuture; +import static org.apache.ignite.internal.lang.IgniteStringFormatter.format; +import static org.apache.ignite.internal.tx.TransactionErrors.finishedTransactionErrorCode; +import static org.apache.ignite.internal.tx.TransactionErrors.finishedTransactionErrorMessage; +import static org.apache.ignite.internal.tx.TransactionLogUtils.formatTxInfo; import static org.apache.ignite.internal.tx.event.LockEvent.LOCK_CONFLICT; import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture; @@ -29,7 +33,6 @@ import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -43,6 +46,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; +import java.util.function.Predicate; import org.apache.ignite.internal.configuration.SystemLocalConfiguration; import org.apache.ignite.internal.configuration.SystemPropertyView; import org.apache.ignite.internal.event.AbstractEventProducer; @@ -57,25 +61,29 @@ import org.apache.ignite.internal.tx.LockMode; import org.apache.ignite.internal.tx.LockTableOverflowException; import org.apache.ignite.internal.tx.PossibleDeadlockOnLockAcquireException; +import org.apache.ignite.internal.tx.TransactionIds; +import org.apache.ignite.internal.tx.TxStateMeta; import org.apache.ignite.internal.tx.Waiter; import org.apache.ignite.internal.tx.event.LockEvent; import org.apache.ignite.internal.tx.event.LockEventParameters; import org.apache.ignite.internal.util.CollectionUtils; import org.apache.ignite.internal.util.IgniteStripedReadWriteLock; +import org.apache.ignite.tx.TransactionException; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; /** * A {@link LockManager} implementation which stores lock queues in the heap. * - *

Lock waiters are placed in the queue, ordered according to comparator provided by {@link HeapLockManager#deadlockPreventionPolicy}. - * When a new waiter is placed in the queue, it's validated against current lock owner: if there is an owner with a higher priority (as - * defined by comparator) lock request is denied. + *

Lock waiters are placed in the queue, ordered according to transaction priority: higher priority transactions go first. + * When a new waiter is placed in the queue, it's validated against current lock owners: if a waiter is not allowed to wait, + * according to the {@link HeapLockManager#deadlockPreventionPolicy}, lock request is denied or current owner is invalidated. * - *

Read lock can be upgraded to write lock (only available for the lowest read-locked entry of - * the queue). + *

When an owner is removed from the queue (when a lock is released), the lock queue is processed twice. + * In the first iteration, we attempt to acquire all possible locks. + * In the second iteration, any existing lock conflicts are resolved. * - *

Additionally limits the lock map size. + *

Lock table size is limited and implicitly defines the maximum size of a transaction. */ public class HeapLockManager extends AbstractEventProducer implements LockManager { /** Table size. */ @@ -104,7 +112,7 @@ public class HeapLockManager extends AbstractEventProducer> txMap = new ConcurrentHashMap<>(1024); + private final ConcurrentHashMap txMap = new ConcurrentHashMap<>(1024); /** Coarse locks. */ private final ConcurrentHashMap coarseMap = new ConcurrentHashMap<>(); @@ -112,6 +120,15 @@ public class HeapLockManager extends AbstractEventProducer { + /** When {@code true}, prevent enlisting a lock in a transaction. */ + volatile boolean sealed; + } + /** * Creates an instance of {@link HeapLockManager} with a few slots eligible for tests which don't stress the lock manager too much. * Such a small instance is started way faster than a full-blown production ready instance with a lot of slots. @@ -144,6 +161,29 @@ private static int intProperty(SystemLocalConfiguration systemProperties, String return property == null ? defaultValue : Integer.parseInt(property.propertyValue()); } + private Exception resolveTransactionSealedException(UUID txId) { + // TODO IGNITE-28506 reduce copy paste. + TxStateMeta meta = txStateVolatileStorage.state(txId); + Throwable cause = meta == null ? null : meta.lastException(); + boolean isFinishedDueToTimeout = meta != null && meta.isFinishedDueToTimeoutOrFalse(); + boolean isFinishedDueToError = meta != null && !isFinishedDueToTimeout && meta.lastExceptionErrorCode() != null; + Throwable publicCause = isFinishedDueToError ? cause : null; + Integer causeErrorCode = meta == null ? null : meta.lastExceptionErrorCode(); + + return new TransactionException( + finishedTransactionErrorCode(isFinishedDueToTimeout, isFinishedDueToError), + format("{} [{}, txState={}].", + finishedTransactionErrorMessage( + isFinishedDueToTimeout, + isFinishedDueToError, + causeErrorCode, + publicCause != null + ), + formatTxInfo(txId, txStateVolatileStorage, false), + meta), + publicCause); + } + @Override public void start(DeadlockPreventionPolicy deadlockPreventionPolicy) { this.deadlockPreventionPolicy = deadlockPreventionPolicy; @@ -156,6 +196,11 @@ public void start(DeadlockPreventionPolicy deadlockPreventionPolicy) { locks = new ConcurrentHashMap<>(lockMapSize); } + @Override + public DeadlockPreventionPolicy policy() { + return deadlockPreventionPolicy; + } + @Override public CompletableFuture acquire(UUID txId, LockKey lockKey, LockMode lockMode) { assert lockMode != null : "Lock mode is null"; @@ -248,6 +293,9 @@ public void releaseAll(UUID txId) { @Override public void failAllWaiters(UUID txId, Exception cause) { + seal(txId); + + // After sealing txMap is protected from concurrent updates. ConcurrentLinkedQueue states = this.txMap.get(txId); if (states != null) { @@ -357,16 +405,40 @@ private LockState adjustLockState(LockState state, LockState v) { } } - private void track(UUID txId, Releasable val) { + private void seal(UUID txId) { + txMap.compute(txId, (k, v) -> { + if (v == null) { + return null; + } + + v.sealed = true; + + return v; + }); + } + + private boolean sealed(UUID txId) { + SealableQueue queue = txMap.get(txId); + return queue != null && queue.sealed; + } + + private boolean track(UUID txId, Releasable val) { + boolean[] ret = {true}; txMap.compute(txId, (k, v) -> { if (v == null) { - v = new ConcurrentLinkedQueue<>(); + v = new SealableQueue(); } - v.add(val); + if (v.sealed) { + ret[0] = false; + } else { + v.add(val); + } return v; }); + + return ret[0]; } private static List collectLocksFromStates(UUID txId, ConcurrentLinkedQueue lockStates) { @@ -437,12 +509,9 @@ public class CoarseLockState implements Releasable { private final Map>> slockWaiters = new HashMap<>(); private final ConcurrentHashMap slockOwners = new ConcurrentHashMap<>(); private final LockKey lockKey; - private final Comparator txComparator; CoarseLockState(LockKey lockKey) { this.lockKey = lockKey; - txComparator = - deadlockPreventionPolicy.txIdComparator() != null ? deadlockPreventionPolicy.txIdComparator() : UUID::compareTo; } @Override @@ -467,7 +536,7 @@ public Lock lock(UUID txId) { return lock; } - int idx = Math.floorMod(spread(txId.hashCode()), CONCURRENCY); + int idx = TransactionIds.hash(txId, CONCURRENCY); stripedLock.readLock(idx).lock(); @@ -497,7 +566,7 @@ public boolean coarse() { @Override public void tryFail(UUID txId, Exception cause) { - int idx = Math.floorMod(spread(txId.hashCode()), CONCURRENCY); + int idx = TransactionIds.hash(txId, CONCURRENCY); IgniteBiTuple> waiter0 = null; @@ -536,7 +605,9 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { if (ixlockOwners.containsKey(txId)) { if (ixlockOwners.size() == 1) { // Safe to upgrade. - track(txId, this); // Double track. + if (!track(txId, this)) { // Double track. + return failedFuture(resolveTransactionSealedException(txId)); + } Lock lock = new Lock(lockKey, lockMode, txId); slockOwners.putIfAbsent(txId, lock); return completedFuture(lock); @@ -552,17 +623,16 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { assert false : "Should not reach here"; } - // Validate reordering with IX locks if prevention is enabled. - if (deadlockPreventionPolicy.usePriority()) { - for (Lock lock : ixlockOwners.values()) { - // Allow only high priority transactions to wait. - if (txComparator.compare(lock.txId(), txId) < 0) { - return notifyAndFail(txId, lock.txId(), lockMode, lock.lockMode()); - } + // Prevent deadlocks by allowing only younger transactions to wait. + for (Lock lock : ixlockOwners.values()) { + if (TransactionIds.compare(txId, lock.txId()) < 0) { + return notifyAndFail(txId, lock.txId(), lockMode, lock.lockMode()); } } - track(txId, this); + if (!track(txId, this)) { + return failedFuture(resolveTransactionSealedException(txId)); + } CompletableFuture fut = new CompletableFuture<>(); IgniteBiTuple> prev = slockWaiters.putIfAbsent(txId, @@ -572,8 +642,11 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { Lock lock = new Lock(lockKey, lockMode, txId); Lock prev = slockOwners.putIfAbsent(txId, lock); + // Do not track on reenter. if (prev == null) { - track(txId, this); // Do not track on reenter. + if (!track(txId, this)) { + return failedFuture(resolveTransactionSealedException(txId)); + } } return completedFuture(lock); @@ -583,7 +656,7 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { } case IX: - int idx = Math.floorMod(spread(txId.hashCode()), CONCURRENCY); + int idx = TransactionIds.hash(txId, CONCURRENCY); stripedLock.readLock(idx).lock(); @@ -593,7 +666,9 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { if (slockOwners.containsKey(txId)) { if (slockOwners.size() == 1) { // Safe to upgrade. - track(txId, this); // Double track. + if (!track(txId, this)) { // Double track. + return failedFuture(resolveTransactionSealedException(txId)); + } Lock lock = new Lock(lockKey, lockMode, txId); ixlockOwners.putIfAbsent(txId, lock); return completedFuture(lock); @@ -616,8 +691,12 @@ public CompletableFuture acquire(UUID txId, LockMode lockMode) { Lock lock = new Lock(lockKey, lockMode, txId); Lock prev = ixlockOwners.putIfAbsent(txId, lock); // Avoid overwrite existing lock. + // Do not track on reenter. if (prev == null) { - track(txId, this); // Do not track on reenter. + if (!track(txId, this)) { + ixlockOwners.remove(txId); + return failedFuture(resolveTransactionSealedException(txId)); + } } return completedFuture(lock); @@ -708,7 +787,7 @@ public void release(@Nullable Lock lock) { break; case IX: - int idx = Math.floorMod(spread(lock.txId().hashCode()), CONCURRENCY); + int idx = TransactionIds.hash(lock.txId(), CONCURRENCY); Map>> wakeups; @@ -758,17 +837,20 @@ public void release(@Nullable Lock lock) { * Key lock. */ public class LockState implements Releasable { - /** Waiters. */ - private final TreeMap waiters; + private final Map waiters; + private final Map conflictsView; /** Lock key. */ private volatile LockKey key; LockState() { - Comparator txComparator = - deadlockPreventionPolicy.txIdComparator() != null ? deadlockPreventionPolicy.txIdComparator() : UUID::compareTo; + Comparator txComparator = deadlockPreventionPolicy.txIdComparator(); - this.waiters = new TreeMap<>(txComparator); + // Keep ordered event store for non-priority based policies to avoid starvation: higher priority transactions will acquire + // locks sooner. + var waitersStore = new TreeMap(txComparator); + this.waiters = waitersStore; + this.conflictsView = deadlockPreventionPolicy.invertedWaitOrder() ? waitersStore.descendingMap() : waitersStore; } /** @@ -832,6 +914,7 @@ IgniteBiTuple, LockMode> tryAcquire(UUID txId, LockMode assert lockMode != null : "Lock mode is null"; WaiterImpl waiter = new WaiterImpl(txId, lockMode); + List notifications; // Called after exiting the waiters monitor. synchronized (waiters) { if (!isUsed()) { @@ -861,35 +944,114 @@ IgniteBiTuple, LockMode> tryAcquire(UUID txId, LockMode } } - if (!isWaiterReadyToNotify(waiter, false)) { - if (deadlockPreventionPolicy.waitTimeout() > 0) { + notifications = tryAcquireInternal(waiter, prev == null, false); + } + + // Callback outside the monitor. + for (Notification r : notifications) { + r.run(); + } + + return new IgniteBiTuple<>(waiter.fut, waiter.lockMode()); + } + + private void failWaiter(WaiterImpl waiter, List notifications, Exception exception) { + if (!waiter.locked()) { + waiters.remove(waiter.txId()); + } else if (waiter.hasLockIntent()) { + waiter.refuseIntent(); // Reset lock intention. + } + waiter.fail(exception); + notifications.add(waiter::notifyLocked); + } + + private List tryAcquireInternal(WaiterImpl waiter, boolean track, boolean unlock) { + List notifications = new ArrayList<>(); + + if (sealed(waiter.txId)) { + failWaiter(waiter, notifications, resolveTransactionSealedException(waiter.txId)); + return notifications; + } + + boolean[] needWait = {false}; + boolean[] notified = {false}; + + findConflicts(waiter, owner -> { + assert !waiter.txId.equals(owner.txId); + @Nullable Waiter toFail = deadlockPreventionPolicy.allowWait(waiter, owner); + + if (!notified[0]) { + // Notify once on first found conflict. + notified[0] = true; + if (notifyListeners(waiter.txId())) { + // If there is an abandoned owner, fail waiter. TODO IGNITE-28458 wait instead of failing. + failWaiter(waiter, notifications, createLockException(waiter, owner, true)); + + return true; + } + } + + // Waiting is allowed. + if (toFail == null) { + // Set upper wait bound. + if (deadlockPreventionPolicy.waitTimeout() > 0 && !unlock) { + // Do not add wait timeout again on unlock. setWaiterTimeout(waiter); } // Put to wait queue, track. - if (prev == null) { - track(waiter.txId, this); + if (track) { + if (!track(waiter.txId, this)) { + failWaiter(waiter, notifications, resolveTransactionSealedException(waiter.txId)); + return true; + } } - return new IgniteBiTuple<>(waiter.fut, waiter.lockMode()); - } + needWait[0] = true; - if (!waiter.locked()) { - waiters.remove(waiter.txId()); - } else if (waiter.hasLockIntent()) { - waiter.refuseIntent(); // Restore old lock. + return true; // Stop iteration on found first eligible for waiting owner. } else { - // Lock granted, track. - if (prev == null) { - track(waiter.txId, this); + // Wait is not allowed, fail one of lockers according to policy. + if (toFail == waiter) { + failWaiter(waiter, notifications, createLockException(waiter, owner, false)); + + return true; + } else { + // Track waiter. + if (track) { + if (!track(waiter.txId, this)) { + failWaiter(waiter, notifications, resolveTransactionSealedException(waiter.txId)); + return true; + } + } + + // We need to fail the owner. Call fail action outside the lock. + notifications.add(() -> deadlockPreventionPolicy.failAction(toFail.txId())); + + // Iterate all owners in search of conflict. + return false; } } + }); + + if (!notifications.isEmpty() || needWait[0]) { + // Grant is not allowed. + return notifications; } - // Notify outside the monitor. - waiter.notifyLocked(); + // Lock granted, track if possible, otherwise fail the lock attempt. + if (track) { + if (!track(waiter.txId, this)) { + failWaiter(waiter, notifications, resolveTransactionSealedException(waiter.txId)); + return notifications; + } + } - return new IgniteBiTuple<>(waiter.fut, waiter.lockMode()); + waiter.lock(); + + notifications.add(waiter::notifyLocked); + + return notifications; } /** @@ -903,89 +1065,37 @@ public int waitersCount() { } } - /** - * Checks current waiter. It can change the internal state of the waiter. - * - * @param waiter Checked waiter. - * @return True if current waiter ready to notify, false otherwise. - */ - private boolean isWaiterReadyToNotify(WaiterImpl waiter, boolean skipFail) { + private void findConflicts(WaiterImpl waiter, Predicate callback) { LockMode intendedLockMode = waiter.intendedLockMode(); - assert intendedLockMode != null : "Intended lock mode is null"; - for (Entry entry : waiters.tailMap(waiter.txId(), false).entrySet()) { + for (Entry entry : conflictsView.entrySet()) { WaiterImpl tmp = entry.getValue(); - LockMode currentlyAcquiredLockMode = tmp.lockMode; - - if (currentlyAcquiredLockMode != null && !currentlyAcquiredLockMode.isCompatible(intendedLockMode)) { - if (conflictFound(waiter.txId())) { - // We treat the current lock as the abandoned one. - waiter.fail(new PossibleDeadlockOnLockAcquireException( - waiter.txId, - tmp.txId, - intendedLockMode, - currentlyAcquiredLockMode, - true, - txStateVolatileStorage - )); - - return true; - } else if (!deadlockPreventionPolicy.usePriority() && deadlockPreventionPolicy.waitTimeout() == 0) { - waiter.fail(new PossibleDeadlockOnLockAcquireException( - waiter.txId, - tmp.txId, - intendedLockMode, - currentlyAcquiredLockMode, - false, - txStateVolatileStorage - )); - - return true; - } - return false; + if (tmp.equals(waiter)) { + continue; } - } - for (Entry entry : waiters.headMap(waiter.txId()).entrySet()) { - WaiterImpl tmp = entry.getValue(); LockMode currentlyAcquiredLockMode = tmp.lockMode; if (currentlyAcquiredLockMode != null && !currentlyAcquiredLockMode.isCompatible(intendedLockMode)) { - if (skipFail) { - return false; - } else if (conflictFound(waiter.txId())) { - // We treat the current lock as the abandoned one. - waiter.fail(new PossibleDeadlockOnLockAcquireException( - waiter.txId, - tmp.txId, - intendedLockMode, - currentlyAcquiredLockMode, - true, - txStateVolatileStorage - )); - return true; - } else if (deadlockPreventionPolicy.waitTimeout() == 0) { - waiter.fail(new PossibleDeadlockOnLockAcquireException( - waiter.txId, - tmp.txId, - intendedLockMode, - currentlyAcquiredLockMode, - false, - txStateVolatileStorage - )); - - return true; - } else { - return false; + boolean stop = callback.test(tmp); + if (stop) { + break; } } } + } - waiter.lock(); - - return true; + private Exception createLockException(WaiterImpl waiter, WaiterImpl owner, boolean abandoned) { + return new PossibleDeadlockOnLockAcquireException( + waiter.txId, + owner.txId, + waiter.intendedLockMode, + owner.lockMode, + abandoned, + txStateVolatileStorage + ); } /** @@ -996,15 +1106,15 @@ private boolean isWaiterReadyToNotify(WaiterImpl waiter, boolean skipFail) { */ @Override public boolean tryRelease(UUID txId) { - Collection toNotify; + Collection toNotify; synchronized (waiters) { toNotify = release(txId); } // Notify outside the monitor. - for (WaiterImpl waiter : toNotify) { - waiter.notifyLocked(); + for (Notification runnable : toNotify) { + runnable.run(); } return key != null && waitersCount() == 0; @@ -1020,7 +1130,7 @@ public boolean tryRelease(UUID txId) { boolean tryRelease(UUID txId, LockMode lockMode) { assert lockMode != null : "Lock mode is null"; - List toNotify = emptyList(); + List toNotify = emptyList(); synchronized (waiters) { WaiterImpl waiter = waiters.get(txId); @@ -1043,8 +1153,8 @@ boolean tryRelease(UUID txId, LockMode lockMode) { } // Notify outside the monitor. - for (WaiterImpl waiter : toNotify) { - waiter.notifyLocked(); + for (Notification waiter : toNotify) { + waiter.run(); } return key != null && waitersCount() == 0; @@ -1056,7 +1166,7 @@ boolean tryRelease(UUID txId, LockMode lockMode) { * @param txId Transaction id. * @return List of waiters to notify. */ - private List release(UUID txId) { + private List release(UUID txId) { WaiterImpl removed = waiters.remove(txId); // Removing incomplete waiter doesn't affect lock state. @@ -1072,45 +1182,44 @@ private List release(UUID txId) { * * @return List of waiters to notify. */ - private List unlockCompatibleWaiters() { - if (!deadlockPreventionPolicy.usePriority() && deadlockPreventionPolicy.waitTimeout() == 0) { + private List unlockCompatibleWaiters() { + if (waiters.isEmpty()) { return emptyList(); } - ArrayList toNotify = new ArrayList<>(); - Set toFail = new HashSet<>(); + List toNotify = new ArrayList<>(); - for (Entry entry : waiters.entrySet()) { - WaiterImpl tmp = entry.getValue(); - - if (tmp.hasLockIntent() && isWaiterReadyToNotify(tmp, true)) { - assert !tmp.hasLockIntent() : "This waiter in not locked for notification [waiter=" + tmp + ']'; + // Current implementation involves copying and quadratic iteration complexity. + // Can try to avoid it by splitting waiters and owners in two separate collections. + // TODO IGNITE-23028 + Collection values = new ArrayList<>(waiters.values()); - toNotify.add(tmp); + // Try to lock anything that possible. + for (WaiterImpl tmp : values) { + if (!tmp.hasLockIntent()) { + continue; } - } - if (deadlockPreventionPolicy.usePriority() && deadlockPreventionPolicy.waitTimeout() >= 0) { - for (Entry entry : waiters.entrySet()) { - WaiterImpl tmp = entry.getValue(); + boolean[] hasConflicts = {false}; - if (tmp.hasLockIntent() && isWaiterReadyToNotify(tmp, false)) { - assert tmp.hasLockIntent() : "Only failed waiter can be notified here [waiter=" + tmp + ']'; + findConflicts(tmp, owner -> { + hasConflicts[0] = true; + return true; + }); - toNotify.add(tmp); - toFail.add(tmp.txId()); - } + if (!hasConflicts[0]) { + tmp.lock(); + toNotify.add(tmp::notifyLocked); } + } - for (UUID failTx : toFail) { - var w = waiters.get(failTx); - - if (w.locked()) { - w.refuseIntent(); - } else { - waiters.remove(failTx); - } + // Re-test waiters to handle possible order violations. After previous step new owners can appear which allow waiting. + for (WaiterImpl tmp : values) { + if (!tmp.hasLockIntent()) { + continue; // Ignore waiters which become owners on previous iteration. } + List notifications = tryAcquireInternal(tmp, false, true); + toNotify.addAll(notifications); } return toNotify; @@ -1156,11 +1265,11 @@ public Waiter waiter(UUID txId) { /** * Notifies about the lock conflict found between transactions. * - * @param acquirerTx Transaction which tries to acquire the lock. + * @param waiter Transaction which tries to acquire the lock. * @return True if the conflict connected with an abandoned transaction, false in the other case. */ - private boolean conflictFound(UUID acquirerTx) { - CompletableFuture eventResult = fireEvent(LOCK_CONFLICT, new LockEventParameters(acquirerTx, allLockHolderTxs())); + private boolean notifyListeners(UUID waiter) { + CompletableFuture eventResult = fireEvent(LOCK_CONFLICT, new LockEventParameters(waiter, allLockHolderTxs())); // No async handling is expected. // TODO: https://issues.apache.org/jira/browse/IGNITE-21153 assert eventResult.isDone() : "Async lock conflict handling is not supported"; @@ -1368,7 +1477,7 @@ public LockMode lockMode() { } /** Grant a lock. */ - private void lock() { + void lock() { assert intendedLockMode != null : "Intended lock mode is null"; lockMode = intendedLockMode; @@ -1408,12 +1517,13 @@ public int hashCode() { /** {@inheritDoc} */ @Override public String toString() { - return S.toString(WaiterImpl.class, this, "granted", fut.isDone() && !fut.isCompletedExceptionally()); + return S.toString(WaiterImpl.class, this, "notified", fut.isDone(), "failed", fut.isDone() && fut.isCompletedExceptionally()); } } - private static int spread(int h) { - return (h ^ (h >>> 16)) & 0x7fffffff; + @FunctionalInterface + private interface Notification { + void run(); } @TestOnly diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/NoWaitDeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/NoWaitDeadlockPreventionPolicy.java new file mode 100644 index 000000000000..12bd6e0c1da9 --- /dev/null +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/NoWaitDeadlockPreventionPolicy.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.impl; + +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.Waiter; + +/** + * Nowait deadlock prevention policy. + */ +public class NoWaitDeadlockPreventionPolicy implements DeadlockPreventionPolicy { + /** {@inheritDoc} */ + @Override + public long waitTimeout() { + return 0; + } + + /** {@inheritDoc} */ + @Override + public Waiter allowWait(Waiter waiter, Waiter owner) { + return waiter; + } +} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/PublicApiThreadingTransaction.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/PublicApiThreadingTransaction.java index b738db73f7ab..c1f65239c261 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/PublicApiThreadingTransaction.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/PublicApiThreadingTransaction.java @@ -173,4 +173,9 @@ public CompletableFuture kill() { public boolean isRolledBackWithTimeoutExceeded() { return transaction.isRolledBackWithTimeoutExceeded(); } + + @Override + public RuntimeException enlistFailedException() { + return transaction.enlistFailedException(); + } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReadWriteTransactionImpl.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReadWriteTransactionImpl.java index 0c29cf328231..277da8a2a924 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReadWriteTransactionImpl.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReadWriteTransactionImpl.java @@ -71,7 +71,7 @@ public class ReadWriteTransactionImpl extends IgniteAbstractTransactionImpl { /** * {@code True} if a transaction is externally killed. */ - private boolean killed; + private volatile boolean killed; /** * {@code True} if a remote(directly mapped) part of this transaction has no writes. @@ -154,9 +154,12 @@ public void enlist( } /** - * Fails the operation. + * Return the exception depending on a transaction state. + * + * @return The exception. */ - private RuntimeException enlistFailedException() { + @Override + public RuntimeException enlistFailedException() { TxStateMeta meta = txManager.stateMeta(id()); Throwable cause = meta == null ? null : meta.lastException(); boolean isFinishedDueToTimeout = meta != null && meta.isFinishedDueToTimeoutOrFalse(); @@ -275,6 +278,8 @@ private CompletableFuture finishInternal( return finishFutureInternal; } else { + killed = !isComplete; + CompletableFuture finishFutureInternal = txManager.finish( observableTsTracker, commitPart, @@ -290,8 +295,6 @@ private CompletableFuture finishInternal( finishFuture = finishFutureInternal.handle((unused, throwable) -> null); this.timeoutExceeded = isFinishedDueToTimeout(finishReason); } else { - killed = true; - return finishFutureInternal.handle((unused, throwable) -> { // TODO https://issues.apache.org/jira/browse/IGNITE-25825 move before finish after async cleanup if (killClosure != null) { @@ -354,7 +357,7 @@ public boolean isRolledBackWithTimeoutExceeded() { } /** - * Fail the transaction with exception so finishing it is not possible. + * Fails the transaction with exception so finishing it is not possible. * * @param e Fail reason. */ @@ -364,7 +367,7 @@ public void fail(TransactionException e) { } /** - * Set no remote writes flag. + * Sets no remote writes flag. * * @param noRemoteWrites The value. */ diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReversedWaitDieDeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReversedWaitDieDeadlockPreventionPolicy.java new file mode 100644 index 000000000000..e4dea3e4041d --- /dev/null +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/ReversedWaitDieDeadlockPreventionPolicy.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.impl; + +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.TransactionIds; +import org.apache.ignite.internal.tx.Waiter; + +/** + * Reversed wait die implementation. Same as wait die, but reverses the wait order: younger is allowed to wait for older, older is rejected + * if conflicts with younger. + */ +public class ReversedWaitDieDeadlockPreventionPolicy implements DeadlockPreventionPolicy { + @Override + public Waiter allowWait(Waiter waiter, Waiter owner) { + int res = TransactionIds.compare(waiter.txId(), owner.txId()); + assert res != 0; + + // Waiter is allowed to wait for owner if it's younger. + // Otherwise we have to fail waiter. + return res > 0 ? null : waiter; + } +} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TimeoutDeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TimeoutDeadlockPreventionPolicy.java new file mode 100644 index 000000000000..1ddd0bf22195 --- /dev/null +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TimeoutDeadlockPreventionPolicy.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.impl; + +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.Waiter; + +/** + * Timeout deadlock prevention policy. + */ +public class TimeoutDeadlockPreventionPolicy implements DeadlockPreventionPolicy { + /** {@inheritDoc} */ + @Override + public long waitTimeout() { + return 0; + } + + /** {@inheritDoc} */ + @Override + public Waiter allowWait(Waiter waiter, Waiter owner) { + return null; + } +} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxIdPriorityComparator.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxIdPriorityComparator.java index 1f5fd5ce9e58..0cda9bc44282 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxIdPriorityComparator.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxIdPriorityComparator.java @@ -20,26 +20,14 @@ import java.util.Comparator; import java.util.UUID; import org.apache.ignite.internal.tx.TransactionIds; -import org.apache.ignite.internal.tx.TxPriority; /** * Comparator for transaction IDs based on their associated priorities and the IDs themselves. The IDs with higher priorities are sorted * first. If the priorities are equal, the IDs are sorted by their natural order. */ public class TxIdPriorityComparator implements Comparator { - private static final Comparator TX_PRIORITY_COMPARATOR = TxPriority::compareTo; - @Override public int compare(UUID o1, UUID o2) { - TxPriority priority1 = TransactionIds.priority(o1); - TxPriority priority2 = TransactionIds.priority(o2); - - int priorityComparison = TX_PRIORITY_COMPARATOR.compare(priority1, priority2); - - if (priorityComparison == 0) { - return o1.compareTo(o2); - } else { - return priorityComparison * -1; // Reverse order. - } + return TransactionIds.compare(o1, o2); } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java index b8052a6f997b..2c8cc25e878d 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxManagerImpl.java @@ -83,7 +83,6 @@ import org.apache.ignite.internal.network.InternalClusterNode; import org.apache.ignite.internal.network.MessagingService; import org.apache.ignite.internal.network.NetworkMessage; -import org.apache.ignite.internal.network.NetworkMessageHandler; import org.apache.ignite.internal.network.TopologyService; import org.apache.ignite.internal.placementdriver.PlacementDriver; import org.apache.ignite.internal.placementdriver.event.PrimaryReplicaEvent; @@ -113,8 +112,9 @@ import org.apache.ignite.internal.tx.TxStateMeta; import org.apache.ignite.internal.tx.TxStateMetaFinishing; import org.apache.ignite.internal.tx.configuration.TransactionConfiguration; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl.TxIdComparators; import org.apache.ignite.internal.tx.impl.TransactionInflights.ReadWriteTxContext; +import org.apache.ignite.internal.tx.message.TxKillMessage; +import org.apache.ignite.internal.tx.message.TxMessageGroup; import org.apache.ignite.internal.tx.message.WriteIntentSwitchReplicatedInfo; import org.apache.ignite.internal.tx.metrics.ResourceVacuumMetrics; import org.apache.ignite.internal.tx.metrics.TransactionMetricsSource; @@ -131,7 +131,7 @@ * *

Uses 2PC for atomic commitment and 2PL for concurrency control. */ -public class TxManagerImpl implements TxManager, NetworkMessageHandler, SystemViewProvider { +public class TxManagerImpl implements TxManager, SystemViewProvider { private static final String ABANDONED_CHECK_TS_PROP = "txnAbandonedCheckTs"; private static final long ABANDONED_CHECK_TS_PROP_DEFAULT_VALUE = 5_000; @@ -144,8 +144,6 @@ public class TxManagerImpl implements TxManager, NetworkMessageHandler, SystemVi private static final int RESOURCE_TTL_PROP_DEFAULT_VALUE = 30 * 1000; - private static final TxIdComparators DEFAULT_TX_ID_COMPARATOR = TxIdComparators.NATURAL; - private static final long DEFAULT_LOCK_TIMEOUT = 0; /** Expiration trigger frequency. */ @@ -189,6 +187,9 @@ public class TxManagerImpl implements TxManager, NetworkMessageHandler, SystemVi /** Detector of transactions that lost the coordinator. */ private final OrphanDetector orphanDetector; + /** Topology service. */ + private final TopologyService topologyService; + /** Local node. */ private final InternalClusterNode localNode; @@ -354,6 +355,7 @@ public TxManagerImpl( this.transactionIdGenerator = transactionIdGenerator; this.placementDriver = placementDriver; this.idleSafeTimePropagationPeriodMsSupplier = idleSafeTimePropagationPeriodMsSupplier; + this.topologyService = topologyService; this.localNode = localNode; this.messagingService = messagingService; this.primaryReplicaExpiredListener = this::primaryReplicaExpiredListener; @@ -773,7 +775,8 @@ private CompletableFuture prepareFinish( Map groups = enlistedGroups.entrySet().stream() .collect(toMap(Entry::getKey, Entry::getValue)); - if (unlockOnly) { + if (unlockOnly && commit) { + // Go with waitCleanupFuture path to avoid a race with inflight operations. return txCleanupRequestSender.cleanup(null, groups, verifiedCommit, commitTimestamp, txId) .thenAccept(ignored -> { // Don't keep useless state. @@ -1031,13 +1034,35 @@ public void processDelayedAck(Object ignored, @Nullable Throwable err) { @Override public CompletableFuture startAsync(ComponentContext componentContext) { - var deadlockPreventionPolicy = new DeadlockPreventionPolicyImpl(DEFAULT_TX_ID_COMPARATOR, DEFAULT_LOCK_TIMEOUT); + var deadlockPreventionPolicy = new WoundWaitDeadlockPreventionPolicy() { + @Override + public long waitTimeout() { + return DEFAULT_LOCK_TIMEOUT; + } + + @Override + public void failAction(UUID owner) { + // TODO IGNITE-28447 sendTxRecoveryMessage and delete locks. + TxStateMeta state = txStateVolatileStorage.state(owner); + if (state == null || state.txCoordinatorId() == null) { + return; // Tx state is invalid. Locks will be cleaned up by tx recovery process. + } + + InternalClusterNode coordinator = topologyService.getById(state.txCoordinatorId()); + if (coordinator == null) { + return; // Tx is abandoned. Locks will be cleaned up by tx recovery process. + } + + txMessageSender.kill(coordinator, owner); + } + }; + txStateVolatileStorage.start(); // TODO https://issues.apache.org/jira/browse/IGNITE-23539 lockManager.start(deadlockPreventionPolicy); - messagingService.addMessageHandler(ReplicaMessageGroup.class, this); + messagingService.addMessageHandler(ReplicaMessageGroup.class, this::handleReplicaAsyncResponse); persistentTxStateVacuumizer = new PersistentTxStateVacuumizer( replicaService, @@ -1067,6 +1092,8 @@ public CompletableFuture startAsync(ComponentContext componentContext) { metricsManager.registerSource(txMetrics); metricsManager.enable(txMetrics); + messagingService.addMessageHandler(TxMessageGroup.class, this::handleTxKillMessage); + return nullCompletedFuture(); } @@ -1187,11 +1214,6 @@ public CompletableFuture kill(UUID txId) { TxStateMeta state = txStateVolatileStorage.state(txId); if (state != null && state.tx() != null) { - // TODO: IGNITE-24382 Kill implicit read-write transaction. - if (!state.tx().isReadOnly() && state.tx().implicit()) { - return falseCompletedFuture(); - } - return state.tx().kill().thenApply(unused -> true); } @@ -1237,9 +1259,17 @@ void onCompleteReadOnlyTransaction(boolean commitIntent, TxIdAndTimestamp txIdAn transactionInflights.markReadOnlyTxFinished(txId); } - @Override - public void onReceived(NetworkMessage message, InternalClusterNode sender, @Nullable Long correlationId) { - if (!(message instanceof ReplicaResponse) || correlationId != null) { + private void handleTxKillMessage(NetworkMessage message, InternalClusterNode sender, @Nullable Long correlationId) { // NOPMD + if (!(message instanceof TxKillMessage)) { + return; + } + + TxKillMessage killMessage = (TxKillMessage) message; + kill(killMessage.txId()); + } + + private void handleReplicaAsyncResponse(NetworkMessage message, InternalClusterNode sender, @Nullable Long correlationId) { // NOPMD + if (!(message instanceof ReplicaResponse)) { return; } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxMessageSender.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxMessageSender.java index 5435ca76bb0e..1e67aa4dc124 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxMessageSender.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/TxMessageSender.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.tx.message.EnlistedPartitionGroupMessage; import org.apache.ignite.internal.tx.message.PartitionEnlistmentMessage; import org.apache.ignite.internal.tx.message.RowIdMessage; +import org.apache.ignite.internal.tx.message.TxKillMessage; import org.apache.ignite.internal.tx.message.TxMessagesFactory; import org.apache.ignite.internal.tx.message.TxStateResponse; import org.apache.ignite.internal.tx.message.WriteIntentSwitchReplicatedInfo; @@ -67,6 +68,7 @@ public class TxMessageSender { /** Replica service. */ private final ReplicaService replicaService; + /** Clock service. */ private final ClockService clockService; /** @@ -331,4 +333,16 @@ private static Map toEnliste return messages; } + + /** + * Sends a message to kill a transaction to its coordinator. + * + * @param coordinator The coordinator. + * @param txId The id. + */ + public void kill(InternalClusterNode coordinator, UUID txId) { + TxKillMessage message = TX_MESSAGES_FACTORY.txKillMessage().txId(txId).build(); + + messagingService.send(coordinator, message); + } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WaitDieDeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WaitDieDeadlockPreventionPolicy.java index 062e2d1303ba..99d3692d53c9 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WaitDieDeadlockPreventionPolicy.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WaitDieDeadlockPreventionPolicy.java @@ -17,9 +17,9 @@ package org.apache.ignite.internal.tx.impl; -import java.util.Comparator; -import java.util.UUID; import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.TransactionIds; +import org.apache.ignite.internal.tx.Waiter; /** * Implements a deadlock prevention policy that resolves conflicts between two transactions (tx1 and tx2) contending for the same key. When @@ -28,22 +28,27 @@ *

    *
  • tx2 is older than tx1.
    • *
  • tx2 is younger than tx1 but has a higher {@link org.apache.ignite.internal.tx.TxPriority}.
    • - *
  • The wait timeout is greater than 0.
    • - *
+ * * If none of these conditions are met, tx2 is aborted to prevent deadlock. */ public class WaitDieDeadlockPreventionPolicy implements DeadlockPreventionPolicy { - private static final TxIdPriorityComparator TX_ID_PRIORITY_COMPARATOR = new TxIdPriorityComparator(); - /** {@inheritDoc} */ @Override - public Comparator txIdComparator() { - return TX_ID_PRIORITY_COMPARATOR; + public long waitTimeout() { + return 0; } - /** {@inheritDoc} */ @Override - public long waitTimeout() { - return 0; + public final Waiter allowWait(Waiter waiter, Waiter owner) { + int res = TransactionIds.compare(waiter.txId(), owner.txId()); + assert res != 0; + + // Waiter is allowed to wait for owner if it has higher priority. + return res < 0 ? null : waiter; + } + + @Override + public final boolean invertedWaitOrder() { + return true; } } diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WoundWaitDeadlockPreventionPolicy.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WoundWaitDeadlockPreventionPolicy.java new file mode 100644 index 000000000000..aa07f92e512a --- /dev/null +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/impl/WoundWaitDeadlockPreventionPolicy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.impl; + +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.TransactionIds; +import org.apache.ignite.internal.tx.Waiter; + +/** + * Implements a deadlock prevention policy that resolves conflicts between two transactions (tx1 and tx2) contending for the same key. When + * tx1 holds a lock and tx2 attempts to acquire it, the policy allows tx2 to wait for the lock if any of the following conditions are + * met: + *
    + *
  • tx2 is younger than tx1.
    • + *
  • tx2 is older than tx1 but has a lower {@link org.apache.ignite.internal.tx.TxPriority}.
    • + *
+ * If none of these conditions are met, tx1 is killed to prevent deadlock. + */ +public class WoundWaitDeadlockPreventionPolicy implements DeadlockPreventionPolicy { + @Override + public Waiter allowWait(Waiter waiter, Waiter owner) { + int res = TransactionIds.compare(waiter.txId(), owner.txId()); + assert res != 0; + + // Waiter is allowed to wait for owner if it has lower priority. + // Otherwise we have to fail owner. + return res > 0 ? null : owner; + } +} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxKillMessage.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxKillMessage.java new file mode 100644 index 000000000000..660bc2a8ad5c --- /dev/null +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxKillMessage.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.message; + +import static org.apache.ignite.internal.tx.message.TxMessageGroup.TX_KILL_MESSAGE; + +import java.util.UUID; +import org.apache.ignite.internal.network.NetworkMessage; +import org.apache.ignite.internal.network.annotations.Transferable; + +/** + * Message for transferring a tx kill request. Doesn't imply response. + */ +@Transferable(TX_KILL_MESSAGE) +public interface TxKillMessage extends NetworkMessage { + /** + * Returns a transaction Id. + * + * @return Transaction id. + */ + UUID txId(); +} diff --git a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxMessageGroup.java b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxMessageGroup.java index 095f67dd6ef6..b00737bc761b 100644 --- a/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxMessageGroup.java +++ b/modules/transactions/src/main/java/org/apache/ignite/internal/tx/message/TxMessageGroup.java @@ -135,4 +135,9 @@ public class TxMessageGroup { /** Message type for {@link TxStateMetaUnknownMessage}. */ public static final short TX_STATE_META_UNKNOWN_MESSAGE = 24; + + /** + * Message type for {@link TxKillMessage}. + */ + public static final short TX_KILL_MESSAGE = 25; } diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractDeadlockPreventionTest.java index 4a613e36c436..3da36189405b 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractDeadlockPreventionTest.java @@ -17,52 +17,62 @@ package org.apache.ignite.internal.tx; +import static org.apache.ignite.internal.testframework.IgniteTestUtils.ensureFutureNotCompleted; import static org.apache.ignite.internal.testframework.IgniteTestUtils.hasCause; -import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.LockMode.X; +import static org.apache.ignite.internal.tx.test.LockFutureMatcher.isGranted; +import static org.apache.ignite.internal.tx.test.LockWaiterMatcher.awaits; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import org.hamcrest.Matcher; import org.junit.jupiter.api.Test; /** * Abstract class containing some tests for deadlock prevention that check common scenarios for different policies. */ public abstract class AbstractDeadlockPreventionTest extends AbstractLockingTest { - protected abstract DeadlockPreventionPolicy deadlockPreventionPolicy(); - - @Override - protected LockManager lockManager() { - return lockManager(deadlockPreventionPolicy()); - } + protected abstract Matcher> conflictMatcher(UUID txId); @Test public void testSimpleConflict0() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + + var key = lockKey("test"); + + assertThat(xlock(tx1, key), isGranted(key, X, tx1)); + assertThat(xlock(tx2, key), conflictMatcher(tx1)); + } - var key1 = key("test"); + @Test + public void testSimpleWait0() { + var tx1 = tx1(); + var tx2 = tx2(); - assertThat(xlock(tx1, key1), willCompleteSuccessfully()); + var key = lockKey("test"); - assertFutureFailsOrWaitsForTimeout(() -> xlock(tx2, key1)); + assertThat(xlock(tx2, key), isGranted(key, X, tx2)); + assertThat(xlock(tx1, key), awaits()); } @Test public void testSimpleConflict1() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(tx2, key1), willSucceedFast()); - CompletableFuture xlockFutTx1 = xlock(tx1, key1); - assertFalse(xlockFutTx1.isDone()); + var xlockFutTx1 = xlock(tx1, key1); + assertThat(xlockFutTx1, awaits()); commitTx(tx2); assertThat(xlockFutTx1, willSucceedFast()); @@ -70,33 +80,32 @@ public void testSimpleConflict1() { @Test public void testSimpleConflictSlocks1() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var key1 = key("test"); + var key = lockKey("test"); - assertThat(slock(tx1, key1), willSucceedFast()); - assertThat(slock(tx2, key1), willSucceedFast()); + assertThat(slock(tx1, key), willSucceedFast()); + assertThat(slock(tx2, key), willSucceedFast()); - assertFutureFailsOrWaitsForTimeout(() -> xlock(tx2, key1)); + assertThat(xlock(tx2, key), conflictMatcher(tx1)); } @Test public void testSimpleConflictSlocks2() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(slock(tx1, key1), willSucceedFast()); assertThat(slock(tx2, key1), willSucceedFast()); - CompletableFuture xlockTx1 = xlock(tx1, key1); - assertFalse(xlockTx1.isDone()); - - CompletableFuture xlockTx2 = xlock(tx2, key1); + var xlockTx1 = xlock(tx1, key1); + assertThat(xlockTx1, awaits()); - assertFutureFailsOrWaitsForTimeout(() -> xlockTx2); + var xlockTx2 = xlock(tx2, key1); + assertThat(xlockTx2, conflictMatcher(tx1)); if (xlockTx2.isDone()) { rollbackTx(tx2); @@ -107,39 +116,44 @@ public void testSimpleConflictSlocks2() { @Test public void testNonFair() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx3, k), willSucceedFast()); - CompletableFuture futTx2 = xlock(tx2, k); - assertFalse(futTx2.isDone()); + var futTx2 = xlock(tx2, k); + assertThat(futTx2, awaits()); - CompletableFuture futTx1 = xlock(tx1, k); - assertFalse(futTx1.isDone()); + var futTx1 = xlock(tx1, k); + assertThat(futTx1, awaits()); commitTx(tx3); - assertThat(futTx1, willSucceedFast()); - - assertFutureFailsOrWaitsForTimeout(() -> futTx2); + // An oldest txn should be locked first. + if (tx2.compareTo(tx1) < 0) { + assertThat(futTx2, willSucceedFast()); + assertFutureFailsOrWaitsForTimeout(() -> futTx1); + } else { + assertThat(futTx1, willSucceedFast()); + assertFutureFailsOrWaitsForTimeout(() -> futTx2); + } } @Test public void testReenterWithConflict() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx2, k), willSucceedFast()); assertThat(slock(tx1, k), willSucceedFast()); - CompletableFuture futTx1 = xlock(tx1, k); - assertFalse(futTx1.isDone()); + var futTx1 = xlock(tx1, k); + assertThat(futTx1, awaits()); commitTx(tx2); @@ -148,22 +162,22 @@ public void testReenterWithConflict() { @Test public void testReenterWithConflictAndAbort() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx2, k), willSucceedFast()); assertThat(slock(tx1, k), willSucceedFast()); - assertFutureFailsOrWaitsForTimeout(() -> xlock(tx2, k)); + assertThat(xlock(tx2, k), conflictMatcher(tx1)); } @Test public void testReenterAllowed() { - var tx1 = beginTx(); + var tx1 = tx1(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx1, k), willSucceedFast()); assertThat(xlock(tx1, k), willSucceedFast()); @@ -171,16 +185,16 @@ public void testReenterAllowed() { @Test public void testNonFairConflictWithAlreadyWaiting() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx2, k), willSucceedFast()); - CompletableFuture futTx1 = xlock(tx1, k); - assertFalse(futTx1.isDone()); + var futTx1 = xlock(tx1, k); + assertThat(futTx1, awaits()); assertThat(slock(tx3, k), willSucceedFast()); @@ -189,75 +203,75 @@ public void testNonFairConflictWithAlreadyWaiting() { @Test public void testNonFairConflictWithAlreadyWaitingWithAbort() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx3, k), willSucceedFast()); - CompletableFuture futTx2 = xlock(tx2, k); - assertFalse(futTx2.isDone()); + var futTx2 = xlock(tx2, k); + assertThat(futTx2, awaits()); assertThat(slock(tx1, k), willSucceedFast()); commitTx(tx3); - assertFutureFailsOrWaitsForTimeout(() -> futTx2); + assertThat(futTx2, conflictMatcher(tx1)); } @Test public void testNonFairTakeFirstCompatible() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); - var tx4 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); + var tx4 = tx4(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx4, k), willSucceedFast()); - CompletableFuture futTx2 = xlock(tx2, k); - assertFalse(futTx2.isDone()); + var futTx2 = xlock(tx2, k); + assertThat(futTx2, awaits()); assertThat(slock(tx1, k), willSucceedFast()); assertThat(slock(tx3, k), willSucceedFast()); - assertFalse(futTx2.isDone()); + assertThat(futTx2, awaits()); commitTx(tx1); commitTx(tx3); commitTx(tx4); - futTx2.join(); + assertThat(futTx2, willSucceedFast()); } @Test public void testLockOrderAfterRelease() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); - var tx4 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); + var tx4 = tx4(); - var k = key("test"); + var k = lockKey("test"); assertThat(xlock(tx4, k), willSucceedFast()); - CompletableFuture futTx3 = slock(tx3, k); - assertFalse(futTx3.isDone()); + var futTx3 = slock(tx3, k); + assertThat(futTx3, awaits()); - CompletableFuture futTx2 = xlock(tx2, k); - assertFalse(futTx2.isDone()); + var futTx2 = xlock(tx2, k); + assertThat(futTx2, awaits()); - CompletableFuture futTx1 = slock(tx1, k); - assertFalse(futTx1.isDone()); + var futTx1 = slock(tx1, k); + assertThat(futTx1, awaits()); commitTx(tx4); assertThat(futTx3, willSucceedFast()); assertThat(futTx1, willSucceedFast()); - assertFalse(futTx2.isDone()); + assertThat(futTx2, awaits()); commitTx(tx1); commitTx(tx3); @@ -267,19 +281,19 @@ public void testLockOrderAfterRelease() { @Test public void testMultipleCompatibleLocksAcquiredAfterIncompatibleReleased() { - var tx1 = beginTx(); - var tx2 = beginTx(); - var tx3 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); + var tx3 = tx3(); - var k = key("test"); + var k = lockKey("test"); assertThat(xlock(tx3, k), willSucceedFast()); - CompletableFuture futTx2 = slock(tx2, k); - assertFalse(futTx2.isDone()); + var futTx2 = slock(tx2, k); + assertThat(futTx2, awaits()); - CompletableFuture futTx1 = slock(tx1, k); - assertFalse(futTx1.isDone()); + var futTx1 = slock(tx1, k); + assertThat(futTx1, awaits()); commitTx(tx3); @@ -289,15 +303,14 @@ public void testMultipleCompatibleLocksAcquiredAfterIncompatibleReleased() { @Test public void testIncompatibleLockRetry() { - var tx1 = beginTx(); - var tx2 = beginTx(); + var tx1 = tx1(); + var tx2 = tx2(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx1, k), willSucceedFast()); assertThat(slock(tx2, k), willSucceedFast()); - - assertFutureFailsOrWaitsForTimeout(() -> xlock(tx2, k)); + assertThat(xlock(tx2, k), conflictMatcher(tx1)); commitTx(tx1); @@ -309,7 +322,7 @@ public void testDeadlockRecovery() { var tx1 = beginTx(); var tx2 = beginTx(); - var k = key("test"); + var k = lockKey("test"); assertThat(slock(tx2, k), willSucceedFast()); assertThat(slock(tx1, k), willSucceedFast()); @@ -340,7 +353,7 @@ protected void assertFutureFailsOrWaitsForTimeout(Supplier> fail(); } else { - assertFalse(f.isDone()); + ensureFutureNotCompleted(f, 25); } } catch (Exception e) { if (!hasCause(e, LockException.class, null)) { diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerEventsTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerEventsTest.java index c7f5303f4fa8..f28924065b85 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerEventsTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerEventsTest.java @@ -40,7 +40,7 @@ public abstract class AbstractLockManagerEventsTest extends AbstractLockingTest private final EventListener lockEventListener = this::lockEventListener; - private final LockKey key = key(new Object()); + private final LockKey key = lockKey(new Object()); private boolean listenerAdded; @@ -76,7 +76,7 @@ public void simpleConflictTest() { UUID tx0 = beginTx(); UUID tx1 = beginTx(); - LockKey key = key(new Object()); + LockKey key = lockKey(new Object()); xlock(tx0, key); xlock(tx1, key); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerTest.java deleted file mode 100644 index 85eb9bd881a0..000000000000 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockManagerTest.java +++ /dev/null @@ -1,1358 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.tx; - -import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow; -import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrowWithCauseOrSuppressed; -import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; -import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; -import static org.apache.ignite.internal.tx.LockMode.IS; -import static org.apache.ignite.internal.tx.LockMode.IX; -import static org.apache.ignite.internal.tx.LockMode.S; -import static org.apache.ignite.internal.tx.LockMode.SIX; -import static org.apache.ignite.internal.tx.LockMode.X; -import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.LongAdder; -import org.apache.ignite.internal.configuration.SystemLocalConfiguration; -import org.apache.ignite.internal.configuration.testframework.ConfigurationExtension; -import org.apache.ignite.internal.configuration.testframework.InjectConfiguration; -import org.apache.ignite.internal.lang.IgniteBiTuple; -import org.apache.ignite.internal.testframework.IgniteAbstractTest; -import org.apache.ignite.internal.testframework.IgniteTestUtils; -import org.apache.ignite.internal.tx.test.TestTransactionIds; -import org.apache.ignite.lang.IgniteException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * Tests a LockManager implementation. - */ -@ExtendWith(ConfigurationExtension.class) -public abstract class AbstractLockManagerTest extends IgniteAbstractTest { - @InjectConfiguration - private SystemLocalConfiguration systemLocalConfiguration; - - protected LockManager lockManager; - - @BeforeEach - public void before() { - lockManager = newInstance(systemLocalConfiguration); - } - - protected abstract LockManager newInstance(SystemLocalConfiguration systemLocalConfiguration); - - protected abstract LockKey lockKey(); - - @Test - public void testSingleKeyWrite() { - UUID txId1 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - CompletableFuture fut0 = lockManager.acquire(txId1, key, X); - - assertTrue(fut0.isDone()); - - Collection queue = lockManager.queue(key); - - assertTrue(queue.size() == 1 && queue.iterator().next().equals(txId1)); - - Waiter waiter = lockManager.waiter(key, txId1); - - assertTrue(waiter.locked()); - - lockManager.release(fut0.join()); - } - - @Test - public void testSingleKeyWriteLock() { - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - CompletableFuture fut0 = lockManager.acquire(txId2, key, X); - - assertTrue(fut0.isDone()); - - assertTrue(txId2.compareTo(txId1) > 0); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, X); - - assertFalse(fut1.isDone()); - - assertTrue(lockManager.waiter(key, txId2).locked()); - assertFalse(lockManager.waiter(key, txId1).locked()); - - lockManager.release(fut0.join()); - - assertTrue(fut1.isDone()); - - assertNull(lockManager.waiter(key, txId2)); - assertTrue(lockManager.waiter(key, txId1).locked()); - - lockManager.release(fut1.join()); - - assertNull(lockManager.waiter(key, txId2)); - assertNull(lockManager.waiter(key, txId1)); - } - - @Test - public void downgradeLockOutOfTurnTest() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - lockManager.acquire(txId0, key, S).join(); - Lock lock = lockManager.acquire(txId2, key, S).join(); - - CompletableFuture fut0 = lockManager.acquire(txId0, key, X); - assertFalse(fut0.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId2, key, X); - expectConflict(fut2); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, S); - fut1.join(); - - assertFalse(fut0.isDone()); - - lockManager.release(lock); - fut0.thenAccept(l -> lockManager.release(l)); - } - - @Test - public void upgradeLockImmediatelyTest() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - CompletableFuture fut = lockManager.acquire(txId0, key, IS); - assertTrue(fut.isDone()); - - CompletableFuture fut0 = lockManager.acquire(txId1, key, IS); - assertTrue(fut0.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId2, key, IS); - assertTrue(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId1, key, IX); - assertTrue(fut2.isDone()); - - lockManager.release(fut1.join()); - } - - @Test - public void testSingleKeyReadWriteLock() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - UUID txId3 = TestTransactionIds.newTransactionId(); - assertTrue(txId3.compareTo(txId2) > 0); - assertTrue(txId2.compareTo(txId1) > 0); - assertTrue(txId1.compareTo(txId0) > 0); - LockKey key = lockKey(); - - CompletableFuture fut3 = lockManager.acquire(txId3, key, S); - assertTrue(fut3.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, S); - assertTrue(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId2, key, S); - assertTrue(fut2.isDone()); - - CompletableFuture fut0 = lockManager.acquire(txId0, key, X); - assertFalse(fut0.isDone()); - - assertTrue(lockManager.waiter(key, txId3).locked()); - assertTrue(lockManager.waiter(key, txId2).locked()); - assertTrue(lockManager.waiter(key, txId1).locked()); - assertFalse(lockManager.waiter(key, txId0).locked()); - - lockManager.release(fut1.join()); - - assertTrue(lockManager.waiter(key, txId3).locked()); - assertTrue(lockManager.waiter(key, txId2).locked()); - assertNull(lockManager.waiter(key, txId1)); - assertFalse(lockManager.waiter(key, txId0).locked()); - - lockManager.release(fut3.join()); - - assertNull(lockManager.waiter(key, txId3)); - assertTrue(lockManager.waiter(key, txId2).locked()); - assertNull(lockManager.waiter(key, txId1)); - assertFalse(lockManager.waiter(key, txId0).locked()); - - lockManager.release(fut2.join()); - - assertNull(lockManager.waiter(key, txId3)); - assertNull(lockManager.waiter(key, txId2)); - assertNull(lockManager.waiter(key, txId1)); - assertTrue(lockManager.waiter(key, txId0).locked()); - } - - @Test - public void testSingleKeyReadWriteConflict() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - // Lock in order - CompletableFuture fut0 = lockManager.acquire(txId1, key, S); - assertTrue(fut0.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId0, key, X); - assertFalse(fut1.isDone()); - - lockManager.release(fut0.join()); - assertTrue(fut1.isDone()); - - lockManager.release(fut1.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - - // Lock not in order - fut0 = lockManager.acquire(txId0, key, S); - assertTrue(fut0.isDone()); - - try { - lockManager.acquire(txId1, key, X).join(); - - fail(); - } catch (CompletionException e) { - // Expected. - } - } - - @Test - public void testSingleKeyReadWriteConflict2() { - UUID[] txId = generate(3); - LockKey key = lockKey(); - - // Lock in order - CompletableFuture fut0 = lockManager.acquire(txId[1], key, S); - assertTrue(fut0.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId[0], key, X); - assertFalse(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId[2], key, S); - assertTrue(fut2.isDone()); - - lockManager.release(fut0.join()); - lockManager.release(fut2.join()); - - assertTrue(fut1.isDone()); - } - - @Test - public void testSingleKeyReadWriteConflict3() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - // Lock in order - CompletableFuture fut0 = lockManager.acquire(txId1, key, S); - assertTrue(fut0.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId0, key, X); - assertFalse(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId2, key, S); - assertTrue(fut2.isDone()); - - assertFalse(lockManager.waiter(key, txId0).locked()); - - lockManager.release(fut2.join()); - lockManager.release(fut0.join()); - - assertTrue(fut1.isDone()); - } - - @Test - public void testSingleKeyReadWriteConflict4() { - UUID txId1 = TestTransactionIds.newTransactionId(); - final UUID txId2 = TestTransactionIds.newTransactionId(); - UUID txId3 = TestTransactionIds.newTransactionId(); - UUID txId4 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - CompletableFuture fut4 = lockManager.acquire(txId4, key, S); - assertTrue(fut4.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, X); - assertFalse(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(txId3, key, X); - assertFalse(fut2.isDone()); - - CompletableFuture fut3 = lockManager.acquire(txId2, key, X); - assertFalse(fut3.isDone()); - } - - @Test - public void testSingleKeyReadWriteConflict5() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - lockManager.acquire(txId0, key, X).join(); - - expectConflict(lockManager.acquire(txId1, key, X)); - } - - @Test - public void testConflicts() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - List> lockModes = new ArrayList<>(); - - lockModes.add(new IgniteBiTuple<>(IS, X)); - lockModes.add(new IgniteBiTuple<>(IX, X)); - lockModes.add(new IgniteBiTuple<>(S, X)); - lockModes.add(new IgniteBiTuple<>(SIX, X)); - lockModes.add(new IgniteBiTuple<>(X, X)); - - lockModes.add(new IgniteBiTuple<>(IX, SIX)); - lockModes.add(new IgniteBiTuple<>(S, SIX)); - lockModes.add(new IgniteBiTuple<>(SIX, SIX)); - lockModes.add(new IgniteBiTuple<>(X, SIX)); - - lockModes.add(new IgniteBiTuple<>(IX, S)); - lockModes.add(new IgniteBiTuple<>(SIX, S)); - lockModes.add(new IgniteBiTuple<>(X, S)); - - lockModes.add(new IgniteBiTuple<>(S, IX)); - lockModes.add(new IgniteBiTuple<>(SIX, IX)); - lockModes.add(new IgniteBiTuple<>(X, IX)); - - lockModes.add(new IgniteBiTuple<>(X, IS)); - - for (IgniteBiTuple lockModePair : lockModes) { - CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModePair.get2()); - CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModePair.get1()); - - assertTrue(fut0.isDone()); - expectConflict(fut1); - - lockManager.release(fut0.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - } - } - - @Test - public void testSingleKeyWriteWriteConflict() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - // Lock in order - CompletableFuture fut0 = lockManager.acquire(txId1, key, X); - assertTrue(fut0.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId0, key, X); - assertFalse(fut1.isDone()); - - try { - lockManager.acquire(txId2, key, X).join(); - - fail(); - } catch (CompletionException e) { - // Expected. - } - } - - @Test - public void testSingleKeyWriteWriteConflict2() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - // Lock in order - CompletableFuture fut2 = lockManager.acquire(txId2, key, X); - assertTrue(fut2.isDone()); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, X); - assertFalse(fut1.isDone()); - - CompletableFuture fut0 = lockManager.acquire(txId0, key, X); - assertFalse(fut0.isDone()); - } - - @Test - public void testSingleKeyMultithreadedRead() throws InterruptedException { - LongAdder readLocks = new LongAdder(); - LongAdder writeLocks = new LongAdder(); - LongAdder failedLocks = new LongAdder(); - - doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 0); - - assertEquals(0, writeLocks.sum()); - assertEquals(0, failedLocks.sum()); - } - - @Test - public void testSingleKeyMultithreadedWrite() throws InterruptedException { - LongAdder readLocks = new LongAdder(); - LongAdder writeLocks = new LongAdder(); - LongAdder failedLocks = new LongAdder(); - - doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 1); - - assertEquals(0, readLocks.sum()); - } - - @Test - public void testSingleKeyMultithreadedRandom() throws InterruptedException { - LongAdder readLocks = new LongAdder(); - LongAdder writeLocks = new LongAdder(); - LongAdder failedLocks = new LongAdder(); - - doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 2); - } - - @Test - public void testLockUpgrade() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - lockManager.acquire(txId0, key, S).join(); - - Lock lock = lockManager.acquire(txId1, key, S).join(); - - CompletableFuture fut = lockManager.acquire(txId0, key, X); - assertFalse(fut.isDone()); - - lockManager.release(lock); - - fut.join(); - - lockManager.release(fut.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - } - - @Test - public void testLockUpgrade2() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - lockManager.acquire(txId0, key, S).join(); - - lockManager.acquire(txId1, key, S).join(); - - expectConflict(lockManager.acquire(txId1, key, X)); - } - - @Test - public void testLockUpgrade3() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - lockManager.acquire(txId1, key, S).join(); - - lockManager.acquire(txId0, key, S).join(); - - Lock lock2 = lockManager.acquire(txId2, key, S).join(); - - CompletableFuture fut1 = lockManager.acquire(txId1, key, X); - - assertFalse(fut1.isDone()); - - lockManager.release(lock2); - assertTrue(fut1.isDone()); - assertTrue(fut1.isCompletedExceptionally()); - } - - @Test - public void testLockUpgrade4() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - lockManager.acquire(txId0, key, S).join(); - - Lock lock = lockManager.acquire(txId1, key, S).join(); - - CompletableFuture fut = lockManager.acquire(txId0, key, X); - - assertFalse(fut.isDone()); - - lockManager.release(lock); - - fut.join(); - - assertThat(lockManager.queue(key), hasSize(1)); - } - - @Test - public void testLockUpgrade5() { - UUID txId0 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - for (LockMode lockMode : List.of(IS, IX, SIX, X)) { - lockManager.acquire(txId0, key, lockMode).join(); - - assertEquals(lockMode, lockManager.waiter(key, txId0).lockMode()); - } - - assertThat(lockManager.queue(key), hasSize(1)); - - lockManager.release(new Lock(key, X, txId0)); - - assertTrue(lockManager.queue(key).isEmpty()); - - List> lockModes = new ArrayList<>(); - - lockModes.add(List.of(IX, S, SIX)); - lockModes.add(List.of(S, IX, SIX)); - - for (List lockModes0 : lockModes) { - lockManager.acquire(txId0, key, lockModes0.get(0)).join(); - lockManager.acquire(txId0, key, lockModes0.get(1)).join(); - - assertEquals(lockModes0.get(2), lockManager.waiter(key, txId0).lockMode()); - - lockManager.release(new Lock(key, lockModes0.get(1), txId0)); - - assertTrue(lockManager.queue(key).isEmpty()); - } - } - - @Test - public void testReenter() { - UUID txId = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - CompletableFuture fut = lockManager.acquire(txId, key, X); - assertTrue(fut.isDone()); - - fut = lockManager.acquire(txId, key, X); - assertTrue(fut.isDone()); - - assertThat(lockManager.queue(key), hasSize(1)); - - lockManager.release(fut.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - - fut = lockManager.acquire(txId, key, S); - assertTrue(fut.isDone()); - - fut = lockManager.acquire(txId, key, S); - assertTrue(fut.isDone()); - - assertThat(lockManager.queue(key), hasSize(1)); - - lockManager.release(fut.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - } - - @Test - public void testAcquireReleasedLock() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - List> lockModes = new ArrayList<>(); - - lockModes.add(List.of(IS, X)); - lockModes.add(List.of(IX, X)); - lockModes.add(List.of(S, X)); - lockModes.add(List.of(SIX, X)); - lockModes.add(List.of(X, X)); - - lockModes.add(List.of(IX, SIX)); - lockModes.add(List.of(S, SIX)); - lockModes.add(List.of(SIX, SIX)); - lockModes.add(List.of(X, SIX)); - - lockModes.add(List.of(IX, S)); - lockModes.add(List.of(SIX, S)); - lockModes.add(List.of(X, S)); - - lockModes.add(List.of(S, IX)); - lockModes.add(List.of(SIX, IX)); - lockModes.add(List.of(X, IX)); - - lockModes.add(List.of(X, IS)); - - for (List lockModes0 : lockModes) { - CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModes0.get(1)); - CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModes0.get(0)); - - assertTrue(fut1.isDone()); - assertFalse(fut0.isDone()); - - lockManager.release(fut1.join()); - - assertTrue(fut0.isDone()); - - lockManager.release(fut0.join()); - } - } - - @Test - public void testCompatibleLockModes() { - UUID txId0 = TestTransactionIds.newTransactionId(); - UUID txId1 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - List> lockModes = new ArrayList<>(); - - lockModes.add(List.of(IS, IS)); - lockModes.add(List.of(IS, IX)); - lockModes.add(List.of(IS, S)); - lockModes.add(List.of(IS, SIX)); - - lockModes.add(List.of(IX, IS)); - lockModes.add(List.of(IX, IX)); - - lockModes.add(List.of(S, IS)); - lockModes.add(List.of(S, S)); - - lockModes.add(List.of(SIX, IS)); - - for (List lockModes0 : lockModes) { - CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModes0.get(0)); - CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModes0.get(1)); - - assertTrue(fut0.isDone()); - assertTrue(fut1.isDone()); - - lockManager.release(fut0.join()); - lockManager.release(fut1.join()); - - assertTrue(lockManager.queue(key).isEmpty()); - } - } - - @Test - public void testPossibleDowngradeLockModes() { - UUID txId0 = TestTransactionIds.newTransactionId(); - - LockKey key = lockKey(); - - for (LockMode lockMode : List.of(SIX, S, IS, IX)) { - CompletableFuture fut0 = lockManager.acquire(txId0, key, X); - - assertEquals(X, fut0.join().lockMode()); - - var lockFut = lockManager.acquire(txId0, key, lockMode); - - Waiter waiter = lockManager.waiter(fut0.join().lockKey(), txId0); - - assertEquals(LockMode.supremum(lockMode, X), waiter.lockMode()); - - lockManager.release(txId0, key, X); - - assertThat(lockManager.queue(key), hasSize(1)); - - waiter = lockManager.waiter(fut0.join().lockKey(), txId0); - - assertEquals(lockMode, waiter.lockMode()); - - assertThat(lockManager.queue(key), hasSize(1)); - - lockManager.release(lockFut.join()); - } - } - - @Test - public void testAcquireRelease() { - UUID txId = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - for (LockMode lockMode : LockMode.values()) { - lockManager.acquire(txId, key, lockMode); - lockManager.release(txId, key, lockMode); - - assertFalse(lockManager.locks(txId).hasNext()); - } - - assertTrue(lockManager.isEmpty()); - } - - @Test - public void testAcquireReleaseWhenHoldOther() { - UUID txId = TestTransactionIds.newTransactionId(); - LockKey key = lockKey(); - - for (LockMode holdLockMode : LockMode.values()) { - lockManager.acquire(txId, key, holdLockMode); - - assertTrue(lockManager.locks(txId).hasNext()); - assertSame(holdLockMode, lockManager.locks(txId).next().lockMode()); - - for (LockMode lockMode : LockMode.values()) { - lockManager.acquire(txId, key, lockMode); - lockManager.release(txId, key, lockMode); - } - - assertTrue(lockManager.locks(txId).hasNext()); - assertSame(holdLockMode, lockManager.locks(txId).next().lockMode()); - - lockManager.release(txId, key, holdLockMode); - - assertFalse(lockManager.locks(txId).hasNext()); - } - - assertTrue(lockManager.isEmpty()); - } - - @Test - public void testReleaseThenReleaseWeakerInHierarchy() { - LockKey key = lockKey(); - - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - - var tx1SharedLock = lockManager.acquire(txId2, key, S); - - assertTrue(tx1SharedLock.isDone()); - - var tx1ExclusiveLock = lockManager.acquire(txId2, key, X); - - assertTrue(tx1ExclusiveLock.isDone()); - - var tx2SharedLock = lockManager.acquire(txId1, key, S); - - assertFalse(tx2SharedLock.isDone()); - - lockManager.release(txId2, key, X); - - assertTrue(lockManager.locks(txId2).hasNext()); - - var lock = lockManager.locks(txId2).next(); - - assertSame(S, lock.lockMode()); - - assertTrue(tx2SharedLock.isDone()); - } - - @Test - public void testReleaseThenNoReleaseWeakerInHierarchy() { - LockKey key = lockKey(); - - UUID txId1 = TestTransactionIds.newTransactionId(); - UUID txId2 = TestTransactionIds.newTransactionId(); - - var tx1SharedLock = lockManager.acquire(txId2, key, S); - - assertTrue(tx1SharedLock.isDone()); - - var tx1ExclusiveLock = lockManager.acquire(txId2, key, X); - - assertTrue(tx1ExclusiveLock.isDone()); - - var tx2SharedLock = lockManager.acquire(txId1, key, S); - - assertFalse(tx2SharedLock.isDone()); - - lockManager.release(txId2, key, S); - - assertTrue(lockManager.locks(txId2).hasNext()); - - var lock = lockManager.locks(txId2).next(); - - assertSame(X, lock.lockMode()); - - assertFalse(tx2SharedLock.isDone()); - } - - @Test - public void testLockingOverloadAndUpgrade() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - - var tx1Lock = lockManager.acquire(tx2, key, X); - - assertTrue(tx1Lock.isDone()); - - var tx2sLock = lockManager.acquire(tx1, key, S); - - assertFalse(tx2sLock.isDone()); - - var tx2xLock = lockManager.acquire(tx1, key, X); - - assertFalse(tx2xLock.isDone()); - - lockManager.release(tx1Lock.join()); - - assertThat(tx2sLock, willSucceedFast()); - assertThat(tx2xLock, willSucceedFast()); - } - - @Test - public void testLockingOverload() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - - var tx1Lock = lockManager.acquire(tx2, key, X); - - assertTrue(tx1Lock.isDone()); - - var tx2xLock = lockManager.acquire(tx1, key, X); - - assertFalse(tx2xLock.isDone()); - - var tx2s1Lock = lockManager.acquire(tx1, key, S); - var tx2s2Lock = lockManager.acquire(tx1, key, S); - - assertFalse(tx2s1Lock.isDone()); - assertFalse(tx2s2Lock.isDone()); - - lockManager.release(tx1Lock.join()); - - assertThat(tx2xLock, willSucceedFast()); - assertThat(tx2s1Lock, willSucceedFast()); - assertThat(tx2s2Lock, willSucceedFast()); - } - - @Test - public void testFailUpgrade() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var tx1Lock = lockManager.acquire(tx1, key, S); - var tx2Lock = lockManager.acquire(tx2, key, S); - var tx3Lock = lockManager.acquire(tx3, key, S); - - assertTrue(tx1Lock.isDone()); - assertTrue(tx2Lock.isDone()); - assertTrue(tx3Lock.isDone()); - - var tx1xLock = lockManager.acquire(tx1, key, X); - var tx2xLock = lockManager.acquire(tx2, key, X); - - assertFalse(tx1xLock.isDone()); - assertFalse(tx2xLock.isDone()); - - lockManager.release(tx3Lock.join()); - - expectConflict(tx2xLock); - assertFalse(tx1xLock.isDone()); - - lockManager.release(tx2Lock.join()); - - assertTrue(tx1xLock.isDone()); - - lockManager.release(tx1xLock.join()); - } - - @Test - public void testDowngradeTargetLock() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - - var tx1Lock = lockManager.acquire(tx1, key, S); - var tx2Lock = lockManager.acquire(tx2, key, S); - - assertThat(tx1Lock, willSucceedFast()); - assertThat(tx2Lock, willSucceedFast()); - - var tx1IxLock = lockManager.acquire(tx1, key, IX); - - assertFalse(tx1IxLock.isDone()); - - assertEquals(SIX, lockManager.waiter(key, tx1).intendedLockMode()); - - lockManager.release(tx1, key, S); - - assertFalse(tx1IxLock.isDone()); - assertEquals(IX, lockManager.waiter(key, tx1).intendedLockMode()); - - lockManager.release(tx2, key, S); - - assertThat(tx1IxLock, willSucceedFast()); - } - - @Test - public void testFailWait() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var tx3Lock = lockManager.acquire(tx3, key, S); - - assertThat(tx3Lock, willSucceedFast()); - - var tx2Lock = lockManager.acquire(tx2, key, X); - - assertFalse(tx2Lock.isDone()); - - var tx1Lock = lockManager.acquire(tx1, key, X); - - assertFalse(tx1Lock.isDone()); - - lockManager.release(tx3, key, S); - - expectConflict(tx2Lock); - - assertThat(tx1Lock, willSucceedFast()); - } - - @Test - public void testWaitInOrder() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var tx3IxLock = lockManager.acquire(tx3, key, IX); - var tx3Lock = lockManager.acquire(tx3, key, S); - - assertThat(tx3IxLock, willSucceedFast()); - assertThat(tx3Lock, willSucceedFast()); - - var tx2Lock = lockManager.acquire(tx2, key, IX); - - assertFalse(tx2Lock.isDone()); - - var tx1Lock = lockManager.acquire(tx1, key, X); - - assertFalse(tx1Lock.isDone()); - - lockManager.release(tx3, key, S); - - assertThat(tx2Lock, willSucceedFast()); - - lockManager.release(tx3, key, IX); - lockManager.release(tx2, key, IX); - - assertThat(tx3Lock, willSucceedFast()); - } - - @Test - public void testWaitNotInOrder() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var tx3IxLock = lockManager.acquire(tx3, key, IX); - var tx3Lock = lockManager.acquire(tx3, key, S); - - assertThat(tx3IxLock, willSucceedFast()); - assertThat(tx3Lock, willSucceedFast()); - - var tx2Lock = lockManager.acquire(tx2, key, X); - - assertFalse(tx2Lock.isDone()); - - var tx1Lock = lockManager.acquire(tx1, key, IX); - - assertFalse(tx1Lock.isDone()); - - lockManager.release(tx3, key, S); - - assertThat(tx1Lock, willSucceedFast()); - - lockManager.release(tx1, key, IX); - lockManager.release(tx3, key, IX); - - assertThat(tx2Lock, willSucceedFast()); - } - - @Test - public void testWaitFailNotInOrder() { - LockKey key = lockKey(); - - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var tx3IxLock = lockManager.acquire(tx3, key, IX); - var tx3Lock = lockManager.acquire(tx3, key, S); - - assertThat(tx3IxLock, willSucceedFast()); - assertThat(tx3Lock, willSucceedFast()); - - var tx2Lock = lockManager.acquire(tx2, key, X); - - assertFalse(tx2Lock.isDone()); - - var tx1Lock = lockManager.acquire(tx1, key, IX); - - assertFalse(tx1Lock.isDone()); - - lockManager.release(tx3, key, S); - - assertThat(tx1Lock, willSucceedFast()); - - lockManager.release(tx3, key, IX); - lockManager.release(tx1, key, IX); - - expectConflict(tx2Lock); - } - - @Test - public void testLocksInIterator() { - UUID txId1 = TestTransactionIds.newTransactionId(); - - LockKey key = new LockKey(0); - - lockManager.acquire(txId1, key, S).join(); - - assertTrue(lockManager.locks(txId1).hasNext()); - - LockKey key2 = new LockKey(1, 1); - - lockManager.acquire(txId1, key2, S).join(); - - AtomicInteger counter = new AtomicInteger(); - - lockManager.locks(txId1).forEachRemaining(lock -> counter.incrementAndGet()); - - assertEquals(2, counter.get()); - } - - @Test - public void testLockIsReleased() { - LockKey key = lockKey(); - - UUID txId1 = TestTransactionIds.newTransactionId(); - - lockManager.acquire(txId1, key, X).join(); - - assertFalse(lockManager.isEmpty()); - - lockManager.release(txId1, key, X); - - assertTrue(lockManager.isEmpty()); - - UUID txId2 = TestTransactionIds.newTransactionId(); - - lockManager.acquire(txId2, key, X).join(); - - lockManager.release(txId2, key, X); - - assertTrue(lockManager.isEmpty()); - } - - @Test - public void testAcquireLockAfterFail() { - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - var key = lockKey(); - - assertThat(lockManager.acquire(tx1, key, S), willCompleteSuccessfully()); - assertThat(lockManager.acquire(tx2, key, IS), willCompleteSuccessfully()); - - assertThat(lockManager.acquire(tx2, key, X), willThrow(LockException.class)); - - assertThat(lockManager.acquire(tx2, key, S), willCompleteSuccessfully()); - - assertThat(lockManager.acquire(tx3, key, S), willCompleteSuccessfully()); - - lockManager.releaseAll(tx1); - - CompletableFuture f = lockManager.acquire(tx2, key, X); - assertFalse(f.isDone()); - - lockManager.releaseAll(tx3); - - assertThat(f, willCompleteSuccessfully()); - } - - @Test - public void testFailWaiter() { - UUID older = TestTransactionIds.newTransactionId(); - UUID newer = TestTransactionIds.newTransactionId(); - - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), X); - assertTrue(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(older, lockKey(), S); - assertFalse(fut2.isDone()); - - // Should do nothing then called on owner. - lockManager.failAllWaiters(newer, new Exception()); - assertFalse(fut2.isDone()); - - lockManager.failAllWaiters(older, new Exception("test")); - assertThat(fut2, willThrowWithCauseOrSuppressed(Exception.class, "test")); - - lockManager.releaseAll(older); - lockManager.releaseAll(newer); - } - - @Test - public void testFailWaiter2() { - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - CompletableFuture fut1 = lockManager.acquire(tx1, lockKey(), S); - assertTrue(fut1.isDone()); - - CompletableFuture fut2 = lockManager.acquire(tx2, lockKey(), S); - assertTrue(fut2.isDone()); - - CompletableFuture fut3 = lockManager.acquire(tx3, lockKey(), S); - assertTrue(fut3.isDone()); - - CompletableFuture fut4 = lockManager.acquire(tx2, lockKey(), X); - assertFalse(fut4.isDone()); - - lockManager.releaseAll(tx3); - - assertThat(fut4, willThrowWithCauseOrSuppressed(PossibleDeadlockOnLockAcquireException.class)); - // Failing already invalidated waiter should do nothing. - lockManager.failAllWaiters(tx2, new Exception()); - - lockManager.releaseAll(tx2); - lockManager.releaseAll(tx1); - } - - @Test - public void testFailWaiter3() { - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - CompletableFuture fut3 = lockManager.acquire(tx3, lockKey(), S); - assertTrue(fut3.isDone()); - - CompletableFuture fut2 = lockManager.acquire(tx2, lockKey(), X); - assertFalse(fut2.isDone()); - - CompletableFuture fut1 = lockManager.acquire(tx1, lockKey(), S); - assertTrue(fut1.isDone()); - - lockManager.releaseAll(tx3); - - assertThat(fut2, willThrowWithCauseOrSuppressed(PossibleDeadlockOnLockAcquireException.class)); - // Failing already invalidated waiter should do nothing. - lockManager.failAllWaiters(tx2, new Exception()); - - lockManager.releaseAll(tx2); - lockManager.releaseAll(tx1); - } - - @Test - public void testFailWaiter4() { - UUID tx1 = TestTransactionIds.newTransactionId(); - UUID tx2 = TestTransactionIds.newTransactionId(); - UUID tx3 = TestTransactionIds.newTransactionId(); - - CompletableFuture fut3 = lockManager.acquire(tx3, lockKey(), S); - assertTrue(fut3.isDone()); - - CompletableFuture fut2 = lockManager.acquire(tx2, lockKey(), X); - assertFalse(fut2.isDone()); - - CompletableFuture fut1 = lockManager.acquire(tx1, lockKey(), X); - assertFalse(fut1.isDone()); - - lockManager.failAllWaiters(tx2, new Exception("test")); - assertThat(fut2, willThrowWithCauseOrSuppressed(Exception.class, "test")); - assertFalse(fut1.isDone()); - - lockManager.releaseAll(tx3); - assertThat(fut1, willCompleteSuccessfully()); - - lockManager.releaseAll(tx1); - } - - /** - * Do test single key multithreaded. - * - * @param duration The duration. - * @param readLocks Read lock accumulator. - * @param writeLocks Write lock accumulator. - * @param failedLocks Failed lock accumulator. - * @param mode Mode: 0 - read only, 1 - write only, 2 - mixed random. - * @throws InterruptedException If interrupted while waiting. - */ - private void doTestSingleKeyMultithreaded( - long duration, - LongAdder readLocks, - LongAdder writeLocks, - LongAdder failedLocks, - int mode - ) throws InterruptedException { - LockKey key = lockKey(); - - Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors() * 2]; - - CyclicBarrier startBar = new CyclicBarrier(threads.length, () -> log.info("Before test")); - - AtomicBoolean stop = new AtomicBoolean(); - - Random r = new Random(); - - AtomicReference firstErr = new AtomicReference<>(); - - try { - for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(() -> { - try { - startBar.await(); - } catch (Exception e) { - fail(); - } - - while (!stop.get() && firstErr.get() == null) { - UUID txId = TestTransactionIds.newTransactionId(); - - if (mode == 0 ? false : mode == 1 ? true : r.nextBoolean()) { - Lock lock; - try { - lock = lockManager.acquire(txId, key, X).join(); - - writeLocks.increment(); - } catch (CompletionException e) { - failedLocks.increment(); - continue; - } - - lockManager.release(lock); - } else { - Lock lock; - try { - lock = lockManager.acquire(txId, key, S).join(); - - readLocks.increment(); - } catch (CompletionException e) { - if (mode == 0) { - fail("Unexpected exception for read only locking mode"); - } - - failedLocks.increment(); - - continue; - } - - lockManager.release(lock); - } - } - }); - - threads[i].setName("Worker" + i); - - threads[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - firstErr.compareAndExchange(null, e); - } - }); - - threads[i].start(); - } - - Thread.sleep(duration); - - stop.set(true); - } finally { - for (Thread thread : threads) { - thread.join(); - } - } - - if (firstErr.get() != null) { - throw new IgniteException(INTERNAL_ERR, firstErr.get()); - } - - log.info("After test readLocks={} writeLocks={} failedLocks={}", readLocks.sum(), writeLocks.sum(), - failedLocks.sum()); - - assertTrue(lockManager.queue(key).isEmpty()); - } - - private UUID[] generate(int num) { - UUID[] tmp = new UUID[num]; - - for (int i = 0; i < tmp.length; i++) { - tmp[i] = TestTransactionIds.newTransactionId(); - } - - for (int i = 1; i < tmp.length; i++) { - assertTrue(tmp[i - 1].compareTo(tmp[i]) < 0); - } - - return tmp; - } - - private void expectConflict(CompletableFuture fut) { - try { - fut.join(); - - fail(); - } catch (CompletionException e) { - assertTrue(IgniteTestUtils.hasCause(e, LockException.class, null), - "Wrong exception type, expecting LockException"); - } - } -} diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockingTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockingTest.java index 4e4294c214c3..3597d566f6a1 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockingTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/AbstractLockingTest.java @@ -24,6 +24,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -45,26 +47,49 @@ @ExtendWith(ConfigurationExtension.class) public abstract class AbstractLockingTest extends BaseIgniteAbstractTest { @InjectConfiguration - private SystemLocalConfiguration systemLocalConfiguration; + protected SystemLocalConfiguration systemLocalConfiguration; protected LockManager lockManager; protected VolatileTxStateMetaStorage txStateVolatileStorage; private final Map, CompletableFuture>> locks = new HashMap<>(); - @BeforeEach - void setUp() { - lockManager = lockManager(); + private UUID[] txns; + + protected UUID tx1() { + return txns[0]; + } + + protected UUID tx2() { + return txns[1]; + } + + protected UUID tx3() { + return txns[2]; + } + + protected UUID tx4() { + return txns[3]; } - protected abstract LockManager lockManager(); + @BeforeEach + void setUp() { + txns = new UUID[4]; + for (int i = 0; i < txns.length; i++) { + txns[i] = beginTx(); + } - protected LockManager lockManager(DeadlockPreventionPolicy deadlockPreventionPolicy) { txStateVolatileStorage = VolatileTxStateMetaStorage.createStarted(); - HeapLockManager lockManager = new HeapLockManager(systemLocalConfiguration, txStateVolatileStorage); - lockManager.start(deadlockPreventionPolicy); - return lockManager; + lockManager = new HeapLockManager(systemLocalConfiguration, txStateVolatileStorage); + DeadlockPreventionPolicy policy = deadlockPreventionPolicy(); + lockManager.start(policy); + if (!policy.invertedWaitOrder()) { + // Tests are written for inverted wait order first, so ids are reversed. Need to fix that to make test logic reusable. + Arrays.sort(txns, Comparator.reverseOrder()); + } } + protected abstract DeadlockPreventionPolicy deadlockPreventionPolicy(); + protected UUID beginTx() { return TestTransactionIds.newTransactionId(); } @@ -73,7 +98,7 @@ protected UUID beginTx(TxPriority priority) { return TestTransactionIds.newTransactionId(priority); } - protected LockKey key(Object key) { + protected static LockKey lockKey(Object key) { ByteBuffer b = ByteBuffer.allocate(Integer.BYTES); b.putInt(key.hashCode()); b.position(0); @@ -81,15 +106,19 @@ protected LockKey key(Object key) { return new LockKey(0, b); } - protected CompletableFuture xlock(UUID tx, LockKey key) { + protected static LockKey lockKey() { + return lockKey(0); + } + + protected CompletableFuture xlock(UUID tx, LockKey key) { return acquire(tx, key, X); } - protected CompletableFuture slock(UUID tx, LockKey key) { + protected CompletableFuture slock(UUID tx, LockKey key) { return acquire(tx, key, S); } - protected CompletableFuture acquire(UUID tx, LockKey key, LockMode mode) { + protected CompletableFuture acquire(UUID tx, LockKey key, LockMode mode) { CompletableFuture fut = lockManager.acquire(tx, key, mode); locks.compute(tx, (k, v) -> { @@ -117,7 +146,9 @@ protected void rollbackTx(UUID tx) { protected void finishTx(UUID tx) { Map, CompletableFuture> txLocks = locks.remove(tx); - assertNotNull(txLocks); + if (txLocks == null) { + return; // Finishing the tx is idempotent operation and allowed to call multiple times. + } for (Map.Entry, CompletableFuture> e : txLocks.entrySet()) { CompletableFuture fut = e.getValue(); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/CoarseGrainedLockManagerTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/CoarseGrainedLockManagerTest.java index 470b66589fae..d2dc107598f3 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/CoarseGrainedLockManagerTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/CoarseGrainedLockManagerTest.java @@ -70,16 +70,16 @@ public void testSimple() { UUID older = TestTransactionIds.newTransactionId(); UUID newer = TestTransactionIds.newTransactionId(); - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), LockMode.IX); + CompletableFuture fut1 = lockManager.acquire(older, lockKey(), LockMode.IX); assertTrue(fut1.isDone()); - CompletableFuture fut2 = lockManager.acquire(older, lockKey(), LockMode.S); + CompletableFuture fut2 = lockManager.acquire(newer, lockKey(), LockMode.S); assertFalse(fut2.isDone()); - lockManager.releaseAll(newer); + lockManager.releaseAll(older); fut2.join(); - lockManager.releaseAll(older); + lockManager.releaseAll(newer); } @Test @@ -104,13 +104,14 @@ public void testSimpleInverse() { @Test public void testComplex() { // Older. - UUID txId4 = TestTransactionIds.newTransactionId(); - UUID txId5 = TestTransactionIds.newTransactionId(); - // Newer. UUID txId1 = TestTransactionIds.newTransactionId(); UUID txId2 = TestTransactionIds.newTransactionId(); UUID txId3 = TestTransactionIds.newTransactionId(); + // Newer. + UUID txId4 = TestTransactionIds.newTransactionId(); + UUID txId5 = TestTransactionIds.newTransactionId(); + CompletableFuture fut1 = lockManager.acquire(txId1, lockKey(), LockMode.IX); assertTrue(fut1.isDone()); @@ -265,20 +266,20 @@ public void testUpgradeAndLockRequest() { UUID older = TestTransactionIds.newTransactionId(); UUID newer = TestTransactionIds.newTransactionId(); - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), LockMode.IX); + CompletableFuture fut1 = lockManager.acquire(older, lockKey(), LockMode.IX); assertTrue(fut1.isDone()); - CompletableFuture fut2 = lockManager.acquire(newer, lockKey(), LockMode.S); + CompletableFuture fut2 = lockManager.acquire(older, lockKey(), LockMode.S); assertTrue(fut2.isDone()); - CompletableFuture fut3 = lockManager.acquire(older, lockKey(), LockMode.S); + CompletableFuture fut3 = lockManager.acquire(newer, lockKey(), LockMode.S); assertFalse(fut3.isDone()); - lockManager.releaseAll(newer); + lockManager.releaseAll(older); fut3.join(); - lockManager.releaseAll(older); + lockManager.releaseAll(newer); } @Test @@ -286,20 +287,20 @@ public void testUpgradeAndLockRequestReverse() { UUID older = TestTransactionIds.newTransactionId(); UUID newer = TestTransactionIds.newTransactionId(); - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), LockMode.S); + CompletableFuture fut1 = lockManager.acquire(older, lockKey(), LockMode.S); assertTrue(fut1.isDone()); - CompletableFuture fut2 = lockManager.acquire(newer, lockKey(), LockMode.IX); + CompletableFuture fut2 = lockManager.acquire(older, lockKey(), LockMode.IX); assertTrue(fut2.isDone()); - CompletableFuture fut3 = lockManager.acquire(older, lockKey(), LockMode.S); + CompletableFuture fut3 = lockManager.acquire(newer, lockKey(), LockMode.S); assertFalse(fut3.isDone()); - lockManager.releaseAll(newer); + lockManager.releaseAll(older); fut3.join(); - lockManager.releaseAll(older); + lockManager.releaseAll(newer); } @Test @@ -345,17 +346,18 @@ public void testDeadlockAvoidance() { CompletableFuture fut2 = lockManager.acquire(older, lockKey2(), LockMode.IX); assertTrue(fut2.isDone()); + // For coarse locks only younger transactions are allowed to wait to break potential cycle in wait graph. CompletableFuture fut3 = lockManager.acquire(newer, lockKey2(), LockMode.S); - assertThrowsWithCause(fut3::join, LockException.class); + assertFalse(fut3.isDone()); CompletableFuture fut4 = lockManager.acquire(older, lockKey(), LockMode.S); - assertFalse(fut4.isDone()); + assertThrowsWithCause(fut4::join, LockException.class); - lockManager.releaseAll(newer); + lockManager.releaseAll(older); - fut4.join(); + fut3.join(); - lockManager.releaseAll(older); + lockManager.releaseAll(newer); } @Test @@ -363,17 +365,17 @@ public void testReleaseWaitingTx() { UUID older = TestTransactionIds.newTransactionId(); UUID newer = TestTransactionIds.newTransactionId(); - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), LockMode.IX); + CompletableFuture fut1 = lockManager.acquire(older, lockKey(), LockMode.IX); assertTrue(fut1.isDone()); - CompletableFuture fut2 = lockManager.acquire(older, lockKey(), LockMode.S); + CompletableFuture fut2 = lockManager.acquire(newer, lockKey(), LockMode.S); assertFalse(fut2.isDone()); - lockManager.releaseAll(older); + lockManager.releaseAll(newer); fut2.join(); - lockManager.releaseAll(newer); + lockManager.releaseAll(older); } @Test @@ -381,18 +383,18 @@ public void testFailWaiter() { UUID older = TestTransactionIds.newTransactionId(); UUID newer = TestTransactionIds.newTransactionId(); - CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), LockMode.IX); + CompletableFuture fut1 = lockManager.acquire(older, lockKey(), LockMode.IX); assertTrue(fut1.isDone()); // Currently only S locks are allowed to wait. - CompletableFuture fut2 = lockManager.acquire(older, lockKey(), LockMode.S); + CompletableFuture fut2 = lockManager.acquire(newer, lockKey(), LockMode.S); assertFalse(fut2.isDone()); // Should do nothing. - lockManager.failAllWaiters(newer, new Exception()); + lockManager.failAllWaiters(older, new Exception()); assertFalse(fut2.isDone()); - lockManager.failAllWaiters(older, new Exception("test")); + lockManager.failAllWaiters(newer, new Exception("test")); assertThat(fut2, willThrowWithCauseOrSuppressed(Exception.class, "test")); lockManager.releaseAll(older); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerEventsTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerEventsTest.java index 270116c2c4b7..9a1c38eb3fb8 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerEventsTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerEventsTest.java @@ -19,13 +19,26 @@ import org.apache.ignite.internal.tx.impl.HeapLockManager; import org.apache.ignite.internal.tx.impl.WaitDieDeadlockPreventionPolicy; +import org.apache.ignite.internal.tx.impl.WoundWaitDeadlockPreventionPolicy; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; /** * Class that contains the tests for lock manager events producing for {@link HeapLockManager}. */ +@ParameterizedClass +@ValueSource(classes = {WaitDieDeadlockPreventionPolicy.class, WoundWaitDeadlockPreventionPolicy.class}) public class HeapLockManagerEventsTest extends AbstractLockManagerEventsTest { + @Parameter + Class policy; + @Override - protected LockManager lockManager() { - return lockManager(new WaitDieDeadlockPreventionPolicy()); + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + try { + return policy.getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } } } diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerTest.java index e92341f0be37..69565c4425c3 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/HeapLockManagerTest.java @@ -17,41 +17,60 @@ package org.apache.ignite.internal.tx; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrowWithCauseOrSuppressed; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.LockMode.IS; +import static org.apache.ignite.internal.tx.LockMode.IX; +import static org.apache.ignite.internal.tx.LockMode.S; +import static org.apache.ignite.internal.tx.LockMode.SIX; +import static org.apache.ignite.internal.tx.LockMode.X; import static org.apache.ignite.internal.tx.impl.HeapLockManager.DEFAULT_SLOTS; import static org.apache.ignite.internal.tx.impl.HeapLockManager.LOCK_MAP_SIZE_PROPERTY_NAME; +import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; import org.apache.ignite.internal.configuration.SystemLocalConfiguration; import org.apache.ignite.internal.configuration.testframework.InjectConfiguration; import org.apache.ignite.internal.configuration.utils.SystemConfigurationPropertyCompatibilityChecker; +import org.apache.ignite.internal.lang.IgniteBiTuple; +import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.internal.tx.impl.HeapLockManager; import org.apache.ignite.internal.tx.impl.VolatileTxStateMetaStorage; import org.apache.ignite.internal.tx.impl.WaitDieDeadlockPreventionPolicy; import org.apache.ignite.internal.tx.test.TestTransactionIds; +import org.apache.ignite.lang.IgniteException; import org.junit.jupiter.api.Test; /** * Test class for {@link HeapLockManager}. */ -public class HeapLockManagerTest extends AbstractLockManagerTest { +public class HeapLockManagerTest extends AbstractLockingTest { @Override - protected LockManager newInstance(SystemLocalConfiguration systemLocalConfiguration) { - VolatileTxStateMetaStorage txStateVolatileStorage = VolatileTxStateMetaStorage.createStarted(); - HeapLockManager lockManager = new HeapLockManager(systemLocalConfiguration, txStateVolatileStorage); - lockManager.start(new WaitDieDeadlockPreventionPolicy()); - return lockManager; - } - - @Override - protected LockKey lockKey() { - return new LockKey(0, "test"); + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + return new WaitDieDeadlockPreventionPolicy(); } @Test @@ -66,12 +85,12 @@ public void testLockTableOverflow() throws Exception { for (int i = 0; i < maxSlots; i++) { txs[i] = TestTransactionIds.newTransactionId(); - lockManager.acquire(txs[i], new LockKey(txs[i], txs[i]), LockMode.S).get(); + lockManager.acquire(txs[i], new LockKey(txs[i], txs[i]), S).get(); } UUID overflowTx = TestTransactionIds.newTransactionId(); - CompletableFuture overflowLockFut = lockManager.acquire(overflowTx, new LockKey(overflowTx, overflowTx), LockMode.S); + CompletableFuture overflowLockFut = lockManager.acquire(overflowTx, new LockKey(overflowTx, overflowTx), S); assertThat(overflowLockFut, willThrowWithCauseOrSuppressed( LockTableOverflowException.class, @@ -82,7 +101,7 @@ public void testLockTableOverflow() throws Exception { lockManager.releaseAll(txs[i]); } - overflowLockFut = lockManager.acquire(overflowTx, new LockKey(overflowTx, overflowTx), LockMode.S); + overflowLockFut = lockManager.acquire(overflowTx, new LockKey(overflowTx, overflowTx), S); assertThat(overflowLockFut, willCompleteSuccessfully()); @@ -102,13 +121,13 @@ public void testLockTooManyKeysInTx() throws Exception { UUID txId = TestTransactionIds.newTransactionId(); for (int i = 0; i < maxSlots; i++) { - lockManager.acquire(txId, new LockKey(i, i), LockMode.S).get(); + lockManager.acquire(txId, new LockKey(i, i), S).get(); } int moreKeys = 2 * maxSlots; for (int i = maxSlots; i < moreKeys; i++) { - CompletableFuture overflowLockFut = lockManager.acquire(txId, new LockKey(i, i), LockMode.S); + CompletableFuture overflowLockFut = lockManager.acquire(txId, new LockKey(i, i), S); assertThat(overflowLockFut, willThrowWithCauseOrSuppressed( LockTableOverflowException.class, @@ -150,4 +169,1217 @@ public void testCompatibilityLockMapSizePropertyNameWasNotChanged() { LOCK_MAP_SIZE_PROPERTY_NAME ); } + + @Test + public void testSingleKeyWrite() { + UUID txId1 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + CompletableFuture fut0 = lockManager.acquire(txId1, key, X); + + assertTrue(fut0.isDone()); + + Collection queue = lockManager.queue(key); + + assertTrue(queue.size() == 1 && queue.iterator().next().equals(txId1)); + + Waiter waiter = lockManager.waiter(key, txId1); + + assertTrue(waiter.locked()); + + lockManager.release(fut0.join()); + } + + @Test + public void testSingleKeyWriteLock() { + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + CompletableFuture fut0 = lockManager.acquire(txId2, key, X); + + assertTrue(fut0.isDone()); + + assertTrue(txId2.compareTo(txId1) > 0); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, X); + + assertFalse(fut1.isDone()); + + assertTrue(lockManager.waiter(key, txId2).locked()); + assertFalse(lockManager.waiter(key, txId1).locked()); + + lockManager.release(fut0.join()); + + assertTrue(fut1.isDone()); + + assertNull(lockManager.waiter(key, txId2)); + assertTrue(lockManager.waiter(key, txId1).locked()); + + lockManager.release(fut1.join()); + + assertNull(lockManager.waiter(key, txId2)); + assertNull(lockManager.waiter(key, txId1)); + } + + @Test + public void downgradeLockOutOfTurnTest() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, S).join(); + Lock lock = lockManager.acquire(txId2, key, S).join(); + + CompletableFuture fut0 = lockManager.acquire(txId0, key, X); + assertFalse(fut0.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId2, key, X); + expectConflict(fut2); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, S); + Lock lock1 = fut1.join(); + + assertFalse(fut0.isDone()); + + lockManager.release(lock); + fut0.thenAccept(l -> lockManager.release(l)); + lockManager.release(lock1); + + assertThat(fut0, willCompleteSuccessfully()); + } + + @Test + public void upgradeLockImmediatelyTest() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + CompletableFuture fut = lockManager.acquire(txId0, key, IS); + assertTrue(fut.isDone()); + + CompletableFuture fut0 = lockManager.acquire(txId1, key, IS); + assertTrue(fut0.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId2, key, IS); + assertTrue(fut1.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId1, key, IX); + assertTrue(fut2.isDone()); + + lockManager.release(fut1.join()); + } + + @Test + public void testSingleKeyReadWriteLock() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + UUID txId3 = TestTransactionIds.newTransactionId(); + assertTrue(txId3.compareTo(txId2) > 0); + assertTrue(txId2.compareTo(txId1) > 0); + assertTrue(txId1.compareTo(txId0) > 0); + LockKey key = lockKey(); + + CompletableFuture fut3 = lockManager.acquire(txId3, key, S); + assertTrue(fut3.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, S); + assertTrue(fut1.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId2, key, S); + assertTrue(fut2.isDone()); + + CompletableFuture fut0 = lockManager.acquire(txId0, key, X); + assertFalse(fut0.isDone()); + + assertTrue(lockManager.waiter(key, txId3).locked()); + assertTrue(lockManager.waiter(key, txId2).locked()); + assertTrue(lockManager.waiter(key, txId1).locked()); + assertFalse(lockManager.waiter(key, txId0).locked()); + + lockManager.release(fut1.join()); + + assertTrue(lockManager.waiter(key, txId3).locked()); + assertTrue(lockManager.waiter(key, txId2).locked()); + assertNull(lockManager.waiter(key, txId1)); + assertFalse(lockManager.waiter(key, txId0).locked()); + + lockManager.release(fut3.join()); + + assertNull(lockManager.waiter(key, txId3)); + assertTrue(lockManager.waiter(key, txId2).locked()); + assertNull(lockManager.waiter(key, txId1)); + assertFalse(lockManager.waiter(key, txId0).locked()); + + lockManager.release(fut2.join()); + + assertNull(lockManager.waiter(key, txId3)); + assertNull(lockManager.waiter(key, txId2)); + assertNull(lockManager.waiter(key, txId1)); + assertTrue(lockManager.waiter(key, txId0).locked()); + } + + @Test + public void testSingleKeyReadWriteConflict() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + // Lock in order + CompletableFuture fut0 = lockManager.acquire(txId1, key, S); + assertTrue(fut0.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId0, key, X); + assertFalse(fut1.isDone()); + + lockManager.release(fut0.join()); + assertTrue(fut1.isDone()); + + lockManager.release(fut1.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + + // Lock not in order + fut0 = lockManager.acquire(txId0, key, S); + assertTrue(fut0.isDone()); + + assertThat(lockManager.acquire(txId1, key, X), willThrow(LockException.class)); + } + + @Test + public void testSingleKeyReadWriteConflict2() { + UUID[] txId = generate(3); + LockKey key = lockKey(); + + // Lock in order + CompletableFuture fut0 = lockManager.acquire(txId[1], key, S); + assertTrue(fut0.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId[0], key, X); + assertFalse(fut1.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId[2], key, S); + assertTrue(fut2.isDone()); + + lockManager.release(fut0.join()); + lockManager.release(fut2.join()); + + assertTrue(fut1.isDone()); + } + + @Test + public void testSingleKeyReadWriteConflict3() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + // Lock in order + CompletableFuture fut0 = lockManager.acquire(txId1, key, S); + assertTrue(fut0.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId0, key, X); + assertFalse(fut1.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId2, key, S); + assertTrue(fut2.isDone()); + + assertFalse(lockManager.waiter(key, txId0).locked()); + + lockManager.release(fut2.join()); + lockManager.release(fut0.join()); + + assertTrue(fut1.isDone()); + } + + @Test + public void testSingleKeyReadWriteConflict4() { + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + UUID txId3 = TestTransactionIds.newTransactionId(); + UUID txId4 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + CompletableFuture fut4 = lockManager.acquire(txId4, key, S); + assertTrue(fut4.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, X); + assertFalse(fut1.isDone()); + + CompletableFuture fut2 = lockManager.acquire(txId3, key, X); + assertFalse(fut2.isDone()); + + CompletableFuture fut3 = lockManager.acquire(txId2, key, X); + assertFalse(fut3.isDone()); + } + + @Test + public void testSingleKeyReadWriteConflict5() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, X).join(); + + expectConflict(lockManager.acquire(txId1, key, X)); + } + + @Test + public void testConflicts() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + List> lockModes = new ArrayList<>(); + + lockModes.add(new IgniteBiTuple<>(IS, X)); + lockModes.add(new IgniteBiTuple<>(IX, X)); + lockModes.add(new IgniteBiTuple<>(S, X)); + lockModes.add(new IgniteBiTuple<>(SIX, X)); + lockModes.add(new IgniteBiTuple<>(X, X)); + + lockModes.add(new IgniteBiTuple<>(IX, SIX)); + lockModes.add(new IgniteBiTuple<>(S, SIX)); + lockModes.add(new IgniteBiTuple<>(SIX, SIX)); + lockModes.add(new IgniteBiTuple<>(X, SIX)); + + lockModes.add(new IgniteBiTuple<>(IX, S)); + lockModes.add(new IgniteBiTuple<>(SIX, S)); + lockModes.add(new IgniteBiTuple<>(X, S)); + + lockModes.add(new IgniteBiTuple<>(S, IX)); + lockModes.add(new IgniteBiTuple<>(SIX, IX)); + lockModes.add(new IgniteBiTuple<>(X, IX)); + + lockModes.add(new IgniteBiTuple<>(X, IS)); + + for (IgniteBiTuple lockModePair : lockModes) { + CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModePair.get2()); + CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModePair.get1()); + + assertTrue(fut0.isDone()); + expectConflict(fut1); + + lockManager.release(fut0.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + } + } + + @Test + public void testSingleKeyWriteWriteConflict() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + // Lock in order + CompletableFuture fut0 = lockManager.acquire(txId1, key, X); + assertTrue(fut0.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId0, key, X); + assertFalse(fut1.isDone()); + + try { + lockManager.acquire(txId2, key, X).join(); + + fail(); + } catch (CompletionException e) { + // Expected. + } + } + + @Test + public void testSingleKeyWriteWriteConflict2() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + // Lock in order + CompletableFuture fut2 = lockManager.acquire(txId2, key, X); + assertTrue(fut2.isDone()); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, X); + assertFalse(fut1.isDone()); + + CompletableFuture fut0 = lockManager.acquire(txId0, key, X); + assertFalse(fut0.isDone()); + } + + @Test + public void testSingleKeyMultithreadedRead() throws InterruptedException { + LongAdder readLocks = new LongAdder(); + LongAdder writeLocks = new LongAdder(); + LongAdder failedLocks = new LongAdder(); + + doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 0); + + assertEquals(0, writeLocks.sum()); + assertEquals(0, failedLocks.sum()); + } + + @Test + public void testSingleKeyMultithreadedWrite() throws InterruptedException { + LongAdder readLocks = new LongAdder(); + LongAdder writeLocks = new LongAdder(); + LongAdder failedLocks = new LongAdder(); + + doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 1); + + assertEquals(0, readLocks.sum()); + } + + @Test + public void testSingleKeyMultithreadedRandom() throws InterruptedException { + LongAdder readLocks = new LongAdder(); + LongAdder writeLocks = new LongAdder(); + LongAdder failedLocks = new LongAdder(); + + doTestSingleKeyMultithreaded(5_000, readLocks, writeLocks, failedLocks, 2); + } + + @Test + public void testLockUpgrade() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, S).join(); + + Lock lock = lockManager.acquire(txId1, key, S).join(); + + CompletableFuture fut = lockManager.acquire(txId0, key, X); + assertFalse(fut.isDone()); + + lockManager.release(lock); + + fut.join(); + + lockManager.release(fut.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + } + + @Test + public void testLockUpgrade2() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, S).join(); + + lockManager.acquire(txId1, key, S).join(); + + expectConflict(lockManager.acquire(txId1, key, X)); + } + + @Test + public void testLockUpgrade3() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId1, key, S).join(); + + lockManager.acquire(txId0, key, S).join(); + + Lock lock2 = lockManager.acquire(txId2, key, S).join(); + + CompletableFuture fut1 = lockManager.acquire(txId1, key, X); + + assertFalse(fut1.isDone()); + + lockManager.release(lock2); + assertTrue(fut1.isDone()); + assertTrue(fut1.isCompletedExceptionally()); + } + + @Test + public void testLockUpgrade4() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, S).join(); + + Lock lock = lockManager.acquire(txId1, key, S).join(); + + CompletableFuture fut = lockManager.acquire(txId0, key, X); + + assertFalse(fut.isDone()); + + lockManager.release(lock); + + fut.join(); + + assertThat(lockManager.queue(key), hasSize(1)); + } + + @Test + public void testLockUpgrade5() { + UUID txId0 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + for (LockMode lockMode : List.of(IS, IX, SIX, X)) { + lockManager.acquire(txId0, key, lockMode).join(); + + assertEquals(lockMode, lockManager.waiter(key, txId0).lockMode()); + } + + assertThat(lockManager.queue(key), hasSize(1)); + + lockManager.release(new Lock(key, X, txId0)); + + assertTrue(lockManager.queue(key).isEmpty()); + + List> lockModes = new ArrayList<>(); + + lockModes.add(List.of(IX, S, SIX)); + lockModes.add(List.of(S, IX, SIX)); + + for (List lockModes0 : lockModes) { + lockManager.acquire(txId0, key, lockModes0.get(0)).join(); + lockManager.acquire(txId0, key, lockModes0.get(1)).join(); + + assertEquals(lockModes0.get(2), lockManager.waiter(key, txId0).lockMode()); + + lockManager.release(new Lock(key, lockModes0.get(1), txId0)); + + assertTrue(lockManager.queue(key).isEmpty()); + } + } + + @Test + public void testReenter() { + UUID txId = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + CompletableFuture fut = lockManager.acquire(txId, key, X); + assertTrue(fut.isDone()); + + fut = lockManager.acquire(txId, key, X); + assertTrue(fut.isDone()); + + assertThat(lockManager.queue(key), hasSize(1)); + + lockManager.release(fut.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + + fut = lockManager.acquire(txId, key, S); + assertTrue(fut.isDone()); + + fut = lockManager.acquire(txId, key, S); + assertTrue(fut.isDone()); + + assertThat(lockManager.queue(key), hasSize(1)); + + lockManager.release(fut.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + } + + @Test + public void testAcquireReleasedLock() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + List> lockModes = new ArrayList<>(); + + lockModes.add(List.of(IS, X)); + lockModes.add(List.of(IX, X)); + lockModes.add(List.of(S, X)); + lockModes.add(List.of(SIX, X)); + lockModes.add(List.of(X, X)); + + lockModes.add(List.of(IX, SIX)); + lockModes.add(List.of(S, SIX)); + lockModes.add(List.of(SIX, SIX)); + lockModes.add(List.of(X, SIX)); + + lockModes.add(List.of(IX, S)); + lockModes.add(List.of(SIX, S)); + lockModes.add(List.of(X, S)); + + lockModes.add(List.of(S, IX)); + lockModes.add(List.of(SIX, IX)); + lockModes.add(List.of(X, IX)); + + lockModes.add(List.of(X, IS)); + + for (List lockModes0 : lockModes) { + CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModes0.get(1)); + CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModes0.get(0)); + + assertTrue(fut1.isDone()); + assertFalse(fut0.isDone()); + + lockManager.release(fut1.join()); + + assertTrue(fut0.isDone()); + + lockManager.release(fut0.join()); + } + } + + @Test + public void testCompatibleLockModes() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + List> lockModes = new ArrayList<>(); + + lockModes.add(List.of(IS, IS)); + lockModes.add(List.of(IS, IX)); + lockModes.add(List.of(IS, S)); + lockModes.add(List.of(IS, SIX)); + + lockModes.add(List.of(IX, IS)); + lockModes.add(List.of(IX, IX)); + + lockModes.add(List.of(S, IS)); + lockModes.add(List.of(S, S)); + + lockModes.add(List.of(SIX, IS)); + + for (List lockModes0 : lockModes) { + CompletableFuture fut0 = lockManager.acquire(txId0, key, lockModes0.get(0)); + CompletableFuture fut1 = lockManager.acquire(txId1, key, lockModes0.get(1)); + + assertTrue(fut0.isDone()); + assertTrue(fut1.isDone()); + + lockManager.release(fut0.join()); + lockManager.release(fut1.join()); + + assertTrue(lockManager.queue(key).isEmpty()); + } + } + + @Test + public void testPossibleDowngradeLockModes() { + UUID txId0 = TestTransactionIds.newTransactionId(); + + LockKey key = lockKey(); + + for (LockMode lockMode : List.of(SIX, S, IS, IX)) { + CompletableFuture fut0 = lockManager.acquire(txId0, key, X); + + assertEquals(X, fut0.join().lockMode()); + + var lockFut = lockManager.acquire(txId0, key, lockMode); + + Waiter waiter = lockManager.waiter(fut0.join().lockKey(), txId0); + + assertEquals(LockMode.supremum(lockMode, X), waiter.lockMode()); + + lockManager.release(txId0, key, X); + + assertThat(lockManager.queue(key), hasSize(1)); + + waiter = lockManager.waiter(fut0.join().lockKey(), txId0); + + assertEquals(lockMode, waiter.lockMode()); + + assertThat(lockManager.queue(key), hasSize(1)); + + lockManager.release(lockFut.join()); + } + } + + @Test + public void testAcquireRelease() { + UUID txId = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + for (LockMode lockMode : LockMode.values()) { + lockManager.acquire(txId, key, lockMode); + lockManager.release(txId, key, lockMode); + + assertFalse(lockManager.locks(txId).hasNext()); + } + + assertTrue(lockManager.isEmpty()); + } + + @Test + public void testAcquireReleaseWhenHoldOther() { + UUID txId = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + for (LockMode holdLockMode : LockMode.values()) { + lockManager.acquire(txId, key, holdLockMode); + + assertTrue(lockManager.locks(txId).hasNext()); + assertSame(holdLockMode, lockManager.locks(txId).next().lockMode()); + + for (LockMode lockMode : LockMode.values()) { + lockManager.acquire(txId, key, lockMode); + lockManager.release(txId, key, lockMode); + } + + assertTrue(lockManager.locks(txId).hasNext()); + assertSame(holdLockMode, lockManager.locks(txId).next().lockMode()); + + lockManager.release(txId, key, holdLockMode); + + assertFalse(lockManager.locks(txId).hasNext()); + } + + assertTrue(lockManager.isEmpty()); + } + + @Test + public void testReleaseThenReleaseWeakerInHierarchy() { + LockKey key = lockKey(); + + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + + var tx1SharedLock = lockManager.acquire(txId2, key, S); + + assertTrue(tx1SharedLock.isDone()); + + var tx1ExclusiveLock = lockManager.acquire(txId2, key, X); + + assertTrue(tx1ExclusiveLock.isDone()); + + var tx2SharedLock = lockManager.acquire(txId1, key, S); + + assertFalse(tx2SharedLock.isDone()); + + lockManager.release(txId2, key, X); + + assertTrue(lockManager.locks(txId2).hasNext()); + + var lock = lockManager.locks(txId2).next(); + + assertSame(S, lock.lockMode()); + + assertTrue(tx2SharedLock.isDone()); + } + + @Test + public void testReleaseThenNoReleaseWeakerInHierarchy() { + LockKey key = lockKey(); + + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + + var tx1SharedLock = lockManager.acquire(txId2, key, S); + + assertTrue(tx1SharedLock.isDone()); + + var tx1ExclusiveLock = lockManager.acquire(txId2, key, X); + + assertTrue(tx1ExclusiveLock.isDone()); + + var tx2SharedLock = lockManager.acquire(txId1, key, S); + + assertFalse(tx2SharedLock.isDone()); + + lockManager.release(txId2, key, S); + + assertTrue(lockManager.locks(txId2).hasNext()); + + var lock = lockManager.locks(txId2).next(); + + assertSame(X, lock.lockMode()); + + assertFalse(tx2SharedLock.isDone()); + } + + @Test + public void testLockingOverloadAndUpgrade() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + + var tx1Lock = lockManager.acquire(tx2, key, X); + + assertTrue(tx1Lock.isDone()); + + var tx2sLock = lockManager.acquire(tx1, key, S); + + assertFalse(tx2sLock.isDone()); + + var tx2xLock = lockManager.acquire(tx1, key, X); + + assertFalse(tx2xLock.isDone()); + + lockManager.release(tx1Lock.join()); + + assertThat(tx2sLock, willSucceedFast()); + assertThat(tx2xLock, willSucceedFast()); + } + + @Test + public void testLockingOverload() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + + var tx1Lock = lockManager.acquire(tx2, key, X); + + assertTrue(tx1Lock.isDone()); + + var tx2xLock = lockManager.acquire(tx1, key, X); + + assertFalse(tx2xLock.isDone()); + + var tx2s1Lock = lockManager.acquire(tx1, key, S); + var tx2s2Lock = lockManager.acquire(tx1, key, S); + + assertFalse(tx2s1Lock.isDone()); + assertFalse(tx2s2Lock.isDone()); + + lockManager.release(tx1Lock.join()); + + assertThat(tx2xLock, willSucceedFast()); + assertThat(tx2s1Lock, willSucceedFast()); + assertThat(tx2s2Lock, willSucceedFast()); + } + + @Test + public void testFailUpgrade() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var tx1Lock = lockManager.acquire(tx1, key, S); + var tx2Lock = lockManager.acquire(tx2, key, S); + var tx3Lock = lockManager.acquire(tx3, key, S); + + assertTrue(tx1Lock.isDone()); + assertTrue(tx2Lock.isDone()); + assertTrue(tx3Lock.isDone()); + + var tx1xLock = lockManager.acquire(tx1, key, X); + var tx2xLock = lockManager.acquire(tx2, key, X); + + assertFalse(tx1xLock.isDone()); + assertFalse(tx2xLock.isDone()); + + lockManager.release(tx3Lock.join()); + + expectConflict(tx2xLock); + assertFalse(tx1xLock.isDone()); + + lockManager.release(tx2Lock.join()); + + assertTrue(tx1xLock.isDone()); + + lockManager.release(tx1xLock.join()); + } + + @Test + public void testDowngradeTargetLock() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + + var tx1Lock = lockManager.acquire(tx1, key, S); + var tx2Lock = lockManager.acquire(tx2, key, S); + + assertThat(tx1Lock, willSucceedFast()); + assertThat(tx2Lock, willSucceedFast()); + + var tx1IxLock = lockManager.acquire(tx1, key, IX); + + assertFalse(tx1IxLock.isDone()); + + assertEquals(SIX, lockManager.waiter(key, tx1).intendedLockMode()); + + lockManager.release(tx1, key, S); + + assertFalse(tx1IxLock.isDone()); + assertEquals(IX, lockManager.waiter(key, tx1).intendedLockMode()); + + lockManager.release(tx2, key, S); + + assertThat(tx1IxLock, willSucceedFast()); + } + + @Test + public void testFailWait() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var tx3Lock = lockManager.acquire(tx3, key, S); + + assertThat(tx3Lock, willSucceedFast()); + + var tx2Lock = lockManager.acquire(tx2, key, X); + + assertFalse(tx2Lock.isDone()); + + var tx1Lock = lockManager.acquire(tx1, key, X); + + assertFalse(tx1Lock.isDone()); + + lockManager.release(tx3, key, S); + + expectConflict(tx2Lock); + + assertThat(tx1Lock, willSucceedFast()); + } + + @Test + public void testWaitInOrder() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var tx3IxLock = lockManager.acquire(tx3, key, IX); + var tx3Lock = lockManager.acquire(tx3, key, S); + + assertThat(tx3IxLock, willSucceedFast()); + assertThat(tx3Lock, willSucceedFast()); + + var tx2Lock = lockManager.acquire(tx2, key, IX); + + assertFalse(tx2Lock.isDone()); + + var tx1Lock = lockManager.acquire(tx1, key, X); + + assertFalse(tx1Lock.isDone()); + + lockManager.release(tx3, key, S); + + assertThat(tx2Lock, willSucceedFast()); + + lockManager.release(tx3, key, IX); + lockManager.release(tx2, key, IX); + + assertThat(tx3Lock, willSucceedFast()); + } + + @Test + public void testWaitNotInOrder() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var tx3IxLock = lockManager.acquire(tx3, key, IX); + var tx3Lock = lockManager.acquire(tx3, key, S); + + assertThat(tx3IxLock, willSucceedFast()); + assertThat(tx3Lock, willSucceedFast()); + + var tx2Lock = lockManager.acquire(tx2, key, X); + + assertFalse(tx2Lock.isDone()); + + var tx1Lock = lockManager.acquire(tx1, key, IX); + + assertFalse(tx1Lock.isDone()); + + lockManager.release(tx3, key, S); + + assertThat(tx1Lock, willSucceedFast()); + + lockManager.release(tx1, key, IX); + lockManager.release(tx3, key, IX); + + assertThat(tx2Lock, willSucceedFast()); + } + + @Test + public void testWaitNotInOrder2() { + UUID txId0 = TestTransactionIds.newTransactionId(); + UUID txId1 = TestTransactionIds.newTransactionId(); + UUID txId2 = TestTransactionIds.newTransactionId(); + LockKey key = lockKey(); + + lockManager.acquire(txId0, key, S).join(); + + lockManager.acquire(txId2, key, S).join(); + + var tx1Lock = lockManager.acquire(txId1, key, X); + + assertFalse(tx1Lock.isDone()); + } + + @Test + public void testWaitFailNotInOrder() { + LockKey key = lockKey(); + + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var tx3IxLock = lockManager.acquire(tx3, key, IX); + var tx3Lock = lockManager.acquire(tx3, key, S); + + assertThat(tx3IxLock, willSucceedFast()); + assertThat(tx3Lock, willSucceedFast()); + + var tx2Lock = lockManager.acquire(tx2, key, X); + + assertFalse(tx2Lock.isDone()); + + var tx1Lock = lockManager.acquire(tx1, key, IX); + + assertFalse(tx1Lock.isDone()); + + lockManager.release(tx3, key, S); + + assertThat(tx1Lock, willSucceedFast()); + + lockManager.release(tx3, key, IX); + lockManager.release(tx1, key, IX); + + expectConflict(tx2Lock); + } + + @Test + public void testLocksInIterator() { + UUID txId1 = TestTransactionIds.newTransactionId(); + + LockKey key = new LockKey(0); + + lockManager.acquire(txId1, key, S).join(); + + assertTrue(lockManager.locks(txId1).hasNext()); + + LockKey key2 = new LockKey(1, 1); + + lockManager.acquire(txId1, key2, S).join(); + + AtomicInteger counter = new AtomicInteger(); + + lockManager.locks(txId1).forEachRemaining(lock -> counter.incrementAndGet()); + + assertEquals(2, counter.get()); + } + + @Test + public void testLockIsReleased() { + LockKey key = lockKey(); + + UUID txId1 = TestTransactionIds.newTransactionId(); + + lockManager.acquire(txId1, key, X).join(); + + assertFalse(lockManager.isEmpty()); + + lockManager.release(txId1, key, X); + + assertTrue(lockManager.isEmpty()); + + UUID txId2 = TestTransactionIds.newTransactionId(); + + lockManager.acquire(txId2, key, X).join(); + + lockManager.release(txId2, key, X); + + assertTrue(lockManager.isEmpty()); + } + + @Test + public void testAcquireLockAfterFail() { + UUID tx1 = TestTransactionIds.newTransactionId(); + UUID tx2 = TestTransactionIds.newTransactionId(); + UUID tx3 = TestTransactionIds.newTransactionId(); + + var key = lockKey(); + + assertThat(lockManager.acquire(tx1, key, S), willCompleteSuccessfully()); + assertThat(lockManager.acquire(tx2, key, IS), willCompleteSuccessfully()); + + assertThat(lockManager.acquire(tx2, key, X), willThrow(LockException.class)); + + assertThat(lockManager.acquire(tx2, key, S), willCompleteSuccessfully()); + + assertThat(lockManager.acquire(tx3, key, S), willCompleteSuccessfully()); + + lockManager.releaseAll(tx1); + + CompletableFuture f = lockManager.acquire(tx2, key, X); + assertFalse(f.isDone()); + + lockManager.releaseAll(tx3); + + assertThat(f, willCompleteSuccessfully()); + } + + @Test + public void testFailWaiter() { + UUID older = TestTransactionIds.newTransactionId(); + UUID newer = TestTransactionIds.newTransactionId(); + + CompletableFuture fut1 = lockManager.acquire(newer, lockKey(), X); + assertTrue(fut1.isDone()); + + // Currently only S locks are allowed to wait. + CompletableFuture fut2 = lockManager.acquire(older, lockKey(), S); + assertFalse(fut2.isDone()); + + // Should do nothing. + lockManager.failAllWaiters(newer, new Exception()); + assertFalse(fut2.isDone()); + + lockManager.failAllWaiters(older, new Exception("test")); + assertThat(fut2, willThrowWithCauseOrSuppressed(Exception.class, "test")); + + lockManager.releaseAll(older); + lockManager.releaseAll(newer); + } + + /** + * Do test single key multithreaded. + * + * @param duration The duration. + * @param readLocks Read lock accumulator. + * @param writeLocks Write lock accumulator. + * @param failedLocks Failed lock accumulator. + * @param mode Mode: 0 - read only, 1 - write only, 2 - mixed random. + * @throws InterruptedException If interrupted while waiting. + */ + private void doTestSingleKeyMultithreaded( + long duration, + LongAdder readLocks, + LongAdder writeLocks, + LongAdder failedLocks, + int mode + ) throws InterruptedException { + LockKey key = lockKey(); + + Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors() * 2]; + + CyclicBarrier startBar = new CyclicBarrier(threads.length, () -> log.info("Before test")); + + AtomicBoolean stop = new AtomicBoolean(); + + Random r = new Random(); + + AtomicReference firstErr = new AtomicReference<>(); + + try { + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(() -> { + try { + startBar.await(); + } catch (Exception e) { + fail(); + } + + while (!stop.get() && firstErr.get() == null) { + UUID txId = TestTransactionIds.newTransactionId(); + + if (mode != 0 && (mode == 1 || r.nextBoolean())) { + Lock lock; + try { + lock = lockManager.acquire(txId, key, X).join(); + + writeLocks.increment(); + } catch (CompletionException e) { + failedLocks.increment(); + continue; + } + + lockManager.release(lock); + } else { + Lock lock; + try { + lock = lockManager.acquire(txId, key, S).join(); + + readLocks.increment(); + } catch (CompletionException e) { + if (mode == 0) { + fail("Unexpected exception for read only locking mode"); + } + + failedLocks.increment(); + + continue; + } + + lockManager.release(lock); + } + } + }); + + threads[i].setName("Worker" + i); + + threads[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + firstErr.compareAndExchange(null, e); + } + }); + + threads[i].start(); + } + + Thread.sleep(duration); + + stop.set(true); + } finally { + for (Thread thread : threads) { + thread.join(); + } + } + + if (firstErr.get() != null) { + throw new IgniteException(INTERNAL_ERR, firstErr.get()); + } + + log.info("After test readLocks={} writeLocks={} failedLocks={}", readLocks.sum(), writeLocks.sum(), + failedLocks.sum()); + + assertTrue(lockManager.queue(key).isEmpty()); + } + + private UUID[] generate(int num) { + UUID[] tmp = new UUID[num]; + + for (int i = 0; i < tmp.length; i++) { + tmp[i] = TestTransactionIds.newTransactionId(); + } + + for (int i = 1; i < tmp.length; i++) { + assertTrue(tmp[i - 1].compareTo(tmp[i]) < 0); + } + + return tmp; + } + + private void expectConflict(CompletableFuture fut) { + try { + fut.join(); + + fail(); + } catch (CompletionException e) { + assertTrue(IgniteTestUtils.hasCause(e, LockException.class, null), + "Wrong exception type, expecting LockException"); + } + } } diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/LockManagerTxLabelTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/LockManagerTxLabelTest.java index 259b30fbd103..66e15e732aa1 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/LockManagerTxLabelTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/LockManagerTxLabelTest.java @@ -28,8 +28,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl.TxIdComparators; +import org.apache.ignite.internal.tx.impl.NoWaitDeadlockPreventionPolicy; import org.junit.jupiter.api.Test; /** @@ -37,9 +36,8 @@ */ public class LockManagerTxLabelTest extends AbstractLockingTest { @Override - protected LockManager lockManager() { - // NO-WAIT: conflicting lock requests fail fast with a deadlock-prevention exception. - return lockManager(new DeadlockPreventionPolicyImpl(TxIdComparators.NONE, 0)); + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + return new NoWaitDeadlockPreventionPolicy(); } @Test @@ -53,7 +51,7 @@ void txLabelsArePresentInPossibleDeadlockExceptionMessage() { txStateVolatileStorage.updateMeta(lockHolderTx, old -> TxStateMeta.builder(PENDING).txLabel(lockHolderLabel).build()); txStateVolatileStorage.updateMeta(failedToAcquireTx, old -> TxStateMeta.builder(PENDING).txLabel(failedToAcquireLabel).build()); - LockKey key = key("test"); + LockKey key = lockKey("test"); assertThat(xlock(lockHolderTx, key), willSucceedFast()); @@ -71,7 +69,7 @@ void emptyTxLabelsAreNotPrintedInPossibleDeadlockExceptionMessage() { txStateVolatileStorage.updateMeta(lockHolderTx, old -> TxStateMeta.builder(PENDING).txLabel("").build()); txStateVolatileStorage.updateMeta(failedToAcquireTx, old -> TxStateMeta.builder(PENDING).txLabel("").build()); - LockKey key = key("test"); + LockKey key = lockKey("test"); assertThat(xlock(lockHolderTx, key), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoWaitDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoWaitDeadlockPreventionTest.java index d84c0969ed64..549ee4b52ffd 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoWaitDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoWaitDeadlockPreventionTest.java @@ -23,8 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.concurrent.CompletableFuture; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl.TxIdComparators; +import org.apache.ignite.internal.tx.impl.NoWaitDeadlockPreventionPolicy; import org.junit.jupiter.api.Test; /** @@ -32,13 +31,9 @@ * another transaction. */ public class NoWaitDeadlockPreventionTest extends AbstractLockingTest { - DeadlockPreventionPolicy deadlockPreventionPolicy() { - return new DeadlockPreventionPolicyImpl(TxIdComparators.NONE, 0); - } - @Override - protected LockManager lockManager() { - return lockManager(deadlockPreventionPolicy()); + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + return new NoWaitDeadlockPreventionPolicy(); } @Test @@ -46,7 +41,7 @@ public void noWaitFail() { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); for (LockMode m1 : LockMode.values()) { for (LockMode m2 : LockMode.values()) { @@ -70,7 +65,7 @@ public void noWaitFailReverseOrder() { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); for (LockMode m2 : LockMode.values()) { for (LockMode m1 : LockMode.values()) { @@ -94,7 +89,7 @@ public void allowNoWaitOnDeadlockOnOne() { var tx0 = beginTx(); var tx1 = beginTx(); - var key = key("test0"); + var key = lockKey("test0"); assertThat(slock(tx0, key), willSucceedFast()); assertThat(slock(tx1, key), willSucceedFast()); @@ -108,8 +103,8 @@ public void allowNoWaitOnDeadlockOnTwoKeys() { var tx0 = beginTx(); var tx1 = beginTx(); - var key0 = key("test0"); - var key1 = key("test1"); + var key0 = lockKey("test0"); + var key1 = lockKey("test1"); assertThat(xlock(tx0, key0), willSucceedFast()); assertThat(xlock(tx1, key1), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoneDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoneDeadlockPreventionTest.java index f5a35109ad9b..9a9e9b0e3945 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoneDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/NoneDeadlockPreventionTest.java @@ -18,9 +18,13 @@ package org.apache.ignite.internal.tx; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.test.LockWaiterMatcher.awaits; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import org.hamcrest.Matcher; import org.junit.jupiter.api.Test; /** @@ -32,12 +36,17 @@ protected DeadlockPreventionPolicy deadlockPreventionPolicy() { return DeadlockPreventionPolicy.NO_OP; } + @Override + protected Matcher> conflictMatcher(UUID txId) { + return awaits(); + } + @Test public void allowDeadlockOnOneKey() { var tx0 = beginTx(); var tx1 = beginTx(); - var key = key("test0"); + var key = lockKey("test0"); assertThat(slock(tx0, key), willSucceedFast()); assertThat(slock(tx1, key), willSucceedFast()); @@ -51,8 +60,8 @@ public void allowDeadlockOnTwoKeys() { var tx0 = beginTx(); var tx1 = beginTx(); - var key0 = key("test0"); - var key1 = key("test1"); + var key0 = lockKey("test0"); + var key1 = lockKey("test1"); assertThat(xlock(tx0, key0), willSucceedFast()); assertThat(xlock(tx1, key1), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedWaitDieDeadlockPreventionTest.java similarity index 72% rename from modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedDeadlockPreventionTest.java rename to modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedWaitDieDeadlockPreventionTest.java index 7656ab27b1ad..7cb50582ede9 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/ReversedWaitDieDeadlockPreventionTest.java @@ -20,40 +20,27 @@ import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.test.LockConflictMatcher.conflictsWith; import static org.hamcrest.MatcherAssert.assertThat; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl.TxIdComparators; -import org.junit.jupiter.api.BeforeEach; +import org.apache.ignite.internal.tx.impl.ReversedWaitDieDeadlockPreventionPolicy; +import org.hamcrest.Matcher; import org.junit.jupiter.api.Test; /** - * Test for WOUND-WAIT deadlock prevention policy. + * Test for reversed WAIT_DIE deadlock prevention policy. */ -public class ReversedDeadlockPreventionTest extends AbstractDeadlockPreventionTest { - private long counter; - - @BeforeEach - public void before() { - counter = 0; - } - - @Override - protected UUID beginTx() { - return beginTx(TxPriority.NORMAL); - } - +public class ReversedWaitDieDeadlockPreventionTest extends AbstractDeadlockPreventionTest { @Override - protected UUID beginTx(TxPriority priority) { - counter++; - return TransactionIds.transactionId(Long.MAX_VALUE - counter, 1, priority); + protected Matcher> conflictMatcher(UUID txId) { + return conflictsWith(txId); } @Override protected DeadlockPreventionPolicy deadlockPreventionPolicy() { - return new DeadlockPreventionPolicyImpl(TxIdComparators.REVERSED, 0); + return new ReversedWaitDieDeadlockPreventionPolicy(); } @Test @@ -61,7 +48,7 @@ public void youngLowTxShouldWaitForOldNormalTx() { var oldNormalTx = beginTx(TxPriority.NORMAL); var youngLowTx = beginTx(TxPriority.LOW); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(oldNormalTx, key1), willSucceedFast()); @@ -77,7 +64,7 @@ public void youngNormalTxShouldBeAborted() { var tx1 = beginTx(TxPriority.LOW); var tx2 = beginTx(TxPriority.NORMAL); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(tx1, key1), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TimeoutDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TimeoutDeadlockPreventionTest.java index d57610d3080a..25d4e3d6187d 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TimeoutDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TimeoutDeadlockPreventionTest.java @@ -19,12 +19,14 @@ import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrowFast; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.test.LockWaiterMatcher.awaits; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; +import java.util.UUID; import java.util.concurrent.CompletableFuture; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl; -import org.apache.ignite.internal.tx.impl.DeadlockPreventionPolicyImpl.TxIdComparators; +import org.apache.ignite.internal.tx.impl.TimeoutDeadlockPreventionPolicy; +import org.hamcrest.Matcher; import org.junit.jupiter.api.Test; /** @@ -33,7 +35,17 @@ public class TimeoutDeadlockPreventionTest extends AbstractDeadlockPreventionTest { @Override protected DeadlockPreventionPolicy deadlockPreventionPolicy() { - return new DeadlockPreventionPolicyImpl(TxIdComparators.NONE, 200); + return new TimeoutDeadlockPreventionPolicy() { + @Override + public long waitTimeout() { + return 500; + } + }; + } + + @Override + protected Matcher> conflictMatcher(UUID txId) { + return awaits(); } @Test @@ -41,7 +53,7 @@ public void timeoutTest() { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); assertThat(xlock(tx1, key), willSucceedFast()); CompletableFuture tx2Fut = xlock(tx2, key); @@ -58,7 +70,7 @@ public void timeoutTestReverseOrder() { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); assertThat(xlock(tx2, key), willSucceedFast()); CompletableFuture tx2Fut = xlock(tx1, key); @@ -75,7 +87,7 @@ public void timeoutFail() throws InterruptedException { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); assertThat(xlock(tx1, key), willSucceedFast()); CompletableFuture tx2Fut = xlock(tx2, key); @@ -94,7 +106,7 @@ public void timeoutFailReverseOrder() throws InterruptedException { var tx1 = beginTx(); var tx2 = beginTx(); - var key = key("test"); + var key = lockKey("test"); assertThat(xlock(tx2, key), willSucceedFast()); CompletableFuture tx2Fut = xlock(tx1, key); @@ -113,7 +125,7 @@ public void allowDeadlockOnOneKeyWithTimeout() { var tx0 = beginTx(); var tx1 = beginTx(); - var key = key("test0"); + var key = lockKey("test0"); assertThat(slock(tx0, key), willSucceedFast()); assertThat(slock(tx1, key), willSucceedFast()); @@ -127,8 +139,8 @@ public void allowDeadlockOnTwoKeysWithTimeout() { var tx0 = beginTx(); var tx1 = beginTx(); - var key0 = key("test0"); - var key1 = key("test1"); + var key0 = lockKey("test0"); + var key1 = lockKey("test1"); assertThat(xlock(tx0, key0), willSucceedFast()); assertThat(xlock(tx1, key1), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TransactionIdsTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TransactionIdsTest.java index cf15e5f01cbd..5534032b7476 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TransactionIdsTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/TransactionIdsTest.java @@ -19,9 +19,12 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Random; import java.util.UUID; import org.apache.ignite.internal.hlc.HybridTimestamp; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -41,4 +44,13 @@ void transactionIdIsBuiltCorrectly(TxPriority priority) { assertThat(extractedNodeId, is(1)); assertThat(extractedPriority, is(priority)); } + + @RepeatedTest(10) + public void testHash() { + Random r = new Random(0); + UUID id = UUID.randomUUID(); + int div = 1 + r.nextInt(32); + int hash = TransactionIds.hash(id, div); + assertTrue(hash >= 0 && hash < div, id + " " + div); + } } diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WaitDieDeadlockPreventionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WaitDieDeadlockPreventionTest.java index b3be16b08991..c19587132f13 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WaitDieDeadlockPreventionTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WaitDieDeadlockPreventionTest.java @@ -20,10 +20,13 @@ import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully; import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.apache.ignite.internal.tx.test.LockConflictMatcher.conflictsWith; import static org.hamcrest.MatcherAssert.assertThat; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import org.apache.ignite.internal.tx.impl.WaitDieDeadlockPreventionPolicy; +import org.hamcrest.Matcher; import org.junit.jupiter.api.Test; /** @@ -35,12 +38,17 @@ protected DeadlockPreventionPolicy deadlockPreventionPolicy() { return new WaitDieDeadlockPreventionPolicy(); } + @Override + protected Matcher> conflictMatcher(UUID txId) { + return conflictsWith(txId); + } + @Test public void youngNormalTxShouldWaitForOldLowTx() { var oldLowTx = beginTx(TxPriority.LOW); var youngNormalTx = beginTx(TxPriority.NORMAL); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(oldLowTx, key1), willSucceedFast()); @@ -56,7 +64,7 @@ public void youngLowTxShouldBeAborted() { var oldNormalTx = beginTx(TxPriority.NORMAL); var youngLowTx = beginTx(TxPriority.LOW); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(oldNormalTx, key1), willSucceedFast()); @@ -68,7 +76,7 @@ public void youngSamePriorityTxShouldBeAborted() { var oldNormalTx = beginTx(TxPriority.NORMAL); var youngNormalTx = beginTx(TxPriority.NORMAL); - var key1 = key("test"); + var key1 = lockKey("test"); assertThat(xlock(oldNormalTx, key1), willSucceedFast()); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionNoOpFailActionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionNoOpFailActionTest.java new file mode 100644 index 000000000000..816388206be7 --- /dev/null +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionNoOpFailActionTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx; + +import static org.apache.ignite.internal.tx.test.LockWaiterMatcher.awaits; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import org.apache.ignite.internal.tx.impl.WoundWaitDeadlockPreventionPolicy; +import org.hamcrest.Matcher; + +/** + * Test for {@link WoundWaitDeadlockPreventionPolicy} with no-op fail action. + */ +public class WoundWaitDeadlockPreventionNoOpFailActionTest extends AbstractDeadlockPreventionTest { + @Override + protected Matcher> conflictMatcher(UUID txId) { + return awaits(); + } + + @Override + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + return new WoundWaitDeadlockPreventionPolicy() { + @Override + public void failAction(UUID owner) { + // No-op action causes wound wait to wait on conflict. + } + }; + } +} diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionRollbackFailActionTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionRollbackFailActionTest.java new file mode 100644 index 000000000000..820be61bb45a --- /dev/null +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/WoundWaitDeadlockPreventionRollbackFailActionTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx; + +import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willSucceedFast; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.util.UUID; +import org.apache.ignite.internal.tx.impl.WoundWaitDeadlockPreventionPolicy; +import org.junit.jupiter.api.Test; + +/** + * Test for {@link WoundWaitDeadlockPreventionPolicy} with rollback fail action. + */ +public class WoundWaitDeadlockPreventionRollbackFailActionTest extends AbstractLockingTest { + @Override + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + return new WoundWaitDeadlockPreventionPolicy() { + @Override + public void failAction(UUID owner) { + rollbackTx(owner); + } + }; + } + + @Test + public void testInvalidate() { + var tx1 = beginTx(); + var tx2 = beginTx(); + var tx3 = beginTx(); + + var k = lockKey("test"); + + assertThat(slock(tx2, k), willSucceedFast()); + assertThat(slock(tx3, k), willSucceedFast()); + + // Should invalidate younger owners. + assertThat(xlock(tx1, k), willSucceedFast()); + } + + @Test + public void testInvalidate2() { + var tx1 = beginTx(); + var tx2 = beginTx(); + var tx3 = beginTx(); + + var k = lockKey("test"); + + assertThat(slock(tx1, k), willSucceedFast()); + assertThat(slock(tx2, k), willSucceedFast()); + assertThat(slock(tx3, k), willSucceedFast()); + + // Should invalidate younger owners. + assertThat(xlock(tx1, k), willSucceedFast()); + } +} diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTest.java index fd9e3f78106f..33b7239a2556 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTest.java @@ -52,6 +52,7 @@ import org.apache.ignite.internal.replicator.ZonePartitionId; import org.apache.ignite.internal.storage.RowId; import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; import org.apache.ignite.internal.tx.Lock; import org.apache.ignite.internal.tx.LockException; import org.apache.ignite.internal.tx.LockKey; @@ -65,6 +66,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -72,6 +76,8 @@ * Test how OrphanDetector reacts on tx lock conflicts. */ @ExtendWith({MockitoExtension.class, ConfigurationExtension.class}) +@ParameterizedClass +@ValueSource(classes = {WaitDieDeadlockPreventionPolicy.class, WoundWaitDeadlockPreventionPolicy.class}) public class OrphanDetectorTest extends BaseIgniteAbstractTest { private static final InternalClusterNode LOCAL_NODE = new ClusterNodeImpl(randomUUID(), "local", new NetworkAddress("127.0.0.1", 2024), null); @@ -79,6 +85,9 @@ public class OrphanDetectorTest extends BaseIgniteAbstractTest { private static final InternalClusterNode REMOTE_NODE = new ClusterNodeImpl(randomUUID(), "remote", new NetworkAddress("127.1.1.1", 2024), null); + @Parameter + private Class policy; + @Mock(answer = RETURNS_DEEP_STUBS) private TopologyService topologyService; @@ -88,7 +97,7 @@ public class OrphanDetectorTest extends BaseIgniteAbstractTest { @Mock private PlacementDriver placementDriver; - private final LockManager lockManager = lockManager(); + private LockManager lockManager; private final HybridClock clock = new HybridClockImpl(); @@ -105,14 +114,22 @@ public class OrphanDetectorTest extends BaseIgniteAbstractTest { private OrphanDetector orphanDetector; - private static LockManager lockManager() { - HeapLockManager lockManager = HeapLockManager.smallInstance(); - lockManager.start(new WaitDieDeadlockPreventionPolicy()); - return lockManager; + private LockManager lockManager() { + try { + HeapLockManager lockManager = HeapLockManager.smallInstance(); + DeadlockPreventionPolicy deadlockPreventionPolicy = policy.getDeclaredConstructor().newInstance(); + + lockManager.start(deadlockPreventionPolicy); + return lockManager; + } catch (Exception e) { + throw new RuntimeException(e); + } } @BeforeEach public void setup() { + lockManager = lockManager(); + idGenerator = new TransactionIdGenerator(LOCAL_NODE.name().hashCode()); PlacementDriverHelper placementDriverHelper = new PlacementDriverHelper(placementDriver, clockService); diff --git a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTxLabelTest.java b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTxLabelTest.java index 451944f05b22..12246061302c 100644 --- a/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTxLabelTest.java +++ b/modules/transactions/src/test/java/org/apache/ignite/internal/tx/impl/OrphanDetectorTxLabelTest.java @@ -48,6 +48,7 @@ import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest; import org.apache.ignite.internal.testframework.IgniteTestUtils; import org.apache.ignite.internal.testframework.log4j2.LogInspector; +import org.apache.ignite.internal.tx.DeadlockPreventionPolicy; import org.apache.ignite.internal.tx.Lock; import org.apache.ignite.internal.tx.LockException; import org.apache.ignite.internal.tx.LockKey; @@ -58,6 +59,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -65,6 +69,8 @@ * Tests tx labels usage in exception messages / logs on OrphanDetector-related flows. */ @ExtendWith(MockitoExtension.class) +@ParameterizedClass +@ValueSource(classes = {WaitDieDeadlockPreventionPolicy.class, WoundWaitDeadlockPreventionPolicy.class}) public class OrphanDetectorTxLabelTest extends BaseIgniteAbstractTest { private static final UUID LOCAL_NODE_ID = randomUUID(); private static final String LOCAL_NODE_NAME = "local"; @@ -94,6 +100,17 @@ public class OrphanDetectorTxLabelTest extends BaseIgniteAbstractTest { private InternalClusterNode remoteNode; + @Parameter + private Class policy; + + protected DeadlockPreventionPolicy deadlockPreventionPolicy() { + try { + return policy.getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @BeforeEach void setUp() { idGenerator = new TransactionIdGenerator(LOCAL_NODE_NAME.hashCode()); @@ -103,7 +120,7 @@ void setUp() { txStateMetaStorage = VolatileTxStateMetaStorage.createStarted(); lockManager = new HeapLockManager(1024, txStateMetaStorage); - lockManager.start(new WaitDieDeadlockPreventionPolicy()); + lockManager.start(deadlockPreventionPolicy()); PlacementDriverHelper placementDriverHelper = new PlacementDriverHelper(placementDriver, clockService); diff --git a/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockConflictMatcher.java b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockConflictMatcher.java new file mode 100644 index 000000000000..986047b951df --- /dev/null +++ b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockConflictMatcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.test; + +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.ignite.internal.tx.Lock; +import org.apache.ignite.internal.tx.PossibleDeadlockOnLockAcquireException; +import org.apache.ignite.internal.util.ExceptionUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +/** + * Validates if a lock operations ends with a conflict with expected transaction. + */ +public class LockConflictMatcher extends TypeSafeMatcher> { + private final UUID conflictId; + + private LockConflictMatcher(UUID txId) { + this.conflictId = txId; + } + + @Override + protected boolean matchesSafely(CompletableFuture item) { + try { + item.get(100, TimeUnit.MILLISECONDS); + return false; // Exception is expected. + } catch (InterruptedException | TimeoutException e) { + throw new AssertionError(e); + } catch (ExecutionException | CancellationException e) { + Throwable cause = ExceptionUtils.unwrapCause(e); + + if (cause instanceof PossibleDeadlockOnLockAcquireException && conflictId != null) { + PossibleDeadlockOnLockAcquireException e0 = (PossibleDeadlockOnLockAcquireException) cause; + + return e0.getMessage().contains(conflictId.toString()); + } + + throw new AssertionError(e); + } + } + + @Override + protected void describeMismatchSafely(CompletableFuture item, Description mismatchDescription) { + if (item.isDone() && !item.isCompletedExceptionally()) { + mismatchDescription.appendText("lock future completes without a conflict on a locker ").appendValue(conflictId); + } else { + mismatchDescription.appendText("was ").appendValue(item); + } + } + + @Override + public void describeTo(Description description) { + description.appendText("lock future which awaits conflict on ").appendValue(conflictId); + } + + public static LockConflictMatcher conflictsWith(UUID txId) { + return new LockConflictMatcher(txId); + } +} diff --git a/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockFutureMatcher.java b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockFutureMatcher.java new file mode 100644 index 000000000000..5c8387bbb389 --- /dev/null +++ b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockFutureMatcher.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.test; + +import static org.hamcrest.Matchers.equalTo; + +import java.util.UUID; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.ignite.internal.tx.Lock; +import org.apache.ignite.internal.tx.LockKey; +import org.apache.ignite.internal.tx.LockMode; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; + +/** + * Validates if a lock future will be granted to expected owner. + */ +public class LockFutureMatcher extends TypeSafeMatcher> { + private final Matcher lockModeMatcher; + + private LockFutureMatcher(Matcher lockModeMatcher) { + this.lockModeMatcher = lockModeMatcher; + } + + @Override + protected boolean matchesSafely(CompletableFuture item) { + try { + Lock lock = item.get(100, TimeUnit.MILLISECONDS); + + return lockModeMatcher.matches(lock); + } catch (ExecutionException | CancellationException | InterruptedException | TimeoutException e) { + throw new AssertionError(e); + } + } + + @Override + protected void describeMismatchSafely(CompletableFuture item, Description mismatchDescription) { + if (item.isDone()) { + lockModeMatcher.describeMismatch(item.join(), mismatchDescription); + } else { + mismatchDescription.appendText("was ").appendValue(item); + } + } + + @Override + public void describeTo(Description description) { + description.appendText("is a lock future that completes successfully with ").appendDescriptionOf(lockModeMatcher); + } + + public static LockFutureMatcher isGranted(LockKey key, LockMode lockMode, UUID owner) { + return new LockFutureMatcher(equalTo(new Lock(key, lockMode, owner))); + } +} diff --git a/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockWaiterMatcher.java b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockWaiterMatcher.java new file mode 100644 index 000000000000..9ebaa82ab75d --- /dev/null +++ b/modules/transactions/src/testFixtures/java/org/apache/ignite/internal/tx/test/LockWaiterMatcher.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.tx.test; + +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.ignite.internal.tx.Lock; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +/** + * Validates if a lock future waits for an owner. + */ +public class LockWaiterMatcher extends TypeSafeMatcher> { + @Override + protected boolean matchesSafely(CompletableFuture item) { + try { + item.get(50, TimeUnit.MILLISECONDS); + return false; // Timeout exception is expected. + } catch (TimeoutException e) { + return true; + } catch (InterruptedException | ExecutionException | CancellationException e) { + throw new AssertionError(e); + } + } + + @Override + protected void describeMismatchSafely(CompletableFuture item, Description mismatchDescription) { + mismatchDescription.appendText("lock future is completed ").appendValue(item); + } + + @Override + public void describeTo(Description description) { + description.appendText("lock future which should wait"); + } + + public static LockWaiterMatcher awaits() { + return new LockWaiterMatcher(); + } +}