From 1c94117c5873b8d860c79897426f876288930e97 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Thu, 10 Sep 2026 14:54:01 +0300 Subject: [PATCH 1/2] [#1011] Wait out the connection limit an account is given of its own The gate deciding whether a refused connect is waited out or reported knew the limits of the server alone, so a grant's MAX_USER_CONNECTIONS (mysql 1226) and a profile's SESSIONS_PER_USER (ORA-02391) failed every borrower at once instead of waiting for a connection of the pool to come back. Both join the codes they belong with, along with ORA-00018; 1226 is reported at once where its message names a resource granted per hour, which no wait clears. mysql 1129 is left out on purpose: a blocked host stays blocked until the host cache is flushed. --- .../backends/jdbc/CachedConnection.java | 72 ++++++++++++++-- .../jdbc/CachedConnectionTestCase.java | 86 +++++++++++++++++++ .../server/backends/jdbc/MySqlTestCase.java | 57 +++++++++++- 3 files changed, 207 insertions(+), 8 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java index c03b8f6414..358b449926 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java @@ -868,7 +868,13 @@ enum ConnectDialect { MYSQL("jdbc:mysql:", '?', new String[]{"connectTimeout"}, 1000, 0, new String[]{"socketTimeout"}, 1000, true, - new int[]{1040, 1203}, new int[]{1053}), + // 1040 is the max_connections of the server, 1203 the max_user_connections an account + // inherits from it, and 1226 the limit granted to the account itself - the last of them + // in accountLimitCodes, since the same code carries the limits granted per hour. + // 1129, a host the server blocked after too many failed connects, is left out of all + // three on purpose: the host cache holds that block until an administrator flushes it, + // so waiting a borrow's deadline out would only hide the message naming the remedy. + new int[]{1040, 1203}, new int[]{1053}, new int[]{1226}), /** oracle: both properties take milliseconds; ReadTimeout is a socket read timeout that outlives the login. */ ORACLE("jdbc:oracle:", '?', new String[]{"oracle.net.CONNECT_TIMEOUT"}, 1000, 0, @@ -884,10 +890,14 @@ enum ConnectDialect { // and the name does not appear in ojdbc8 at all, so a descriptor carrying one would // have taken our bound off a connection that never had one of its own. new String[]{"oracle.jdbc.ReadTimeout", "oracle.net.READ_TIMEOUT"}, 1000, true, + // ORA-00020 is the processes of the instance and ORA-00018 its sessions; ORA-02391 is + // the SESSIONS_PER_USER of the account's own profile, the per-account sibling of the + // two, and every one of the three is cleared by a session ending - which for this pool + // is a connection of its own going back to it. // ORA-01033 and ORA-01034: the instance is starting up or not there yet; ORA-01089: // it is shutting down. ORA-12514 is left out of these on purpose - a listener that // does not know the service is also what a service name of a typo looks like, forever - new int[]{20, 12516, 12518, 12519, 12520}, new int[]{1033, 1034, 1089}), + new int[]{18, 20, 2391, 12516, 12518, 12519, 12520}, new int[]{1033, 1034, 1089}), /** * ms sql server: loginTimeout takes seconds, socketTimeout milliseconds; the latter is a * socket read timeout that outlives the login. loginTimeout is the one property of the @@ -918,11 +928,31 @@ enum ConnectDialect { final int[] connectionLimitCodes; /** the vendor codes of this dialect for "not accepting connections yet": a database on its way up */ final int[] notAcceptingYetCodes; + /** + * The vendor codes for a limit an account is given of its own, which this dialect reports + * with the code of limits that no wait can clear. mysql answers 1226 ER_USER_LIMIT_REACHED + * both for the MAX_USER_CONNECTIONS of a grant - the connections this account may hold at + * once, which a connection of this pool going back clears, exactly as the limit of the + * server does - and for the resources it is granted per hour, which the top of the hour + * clears and nothing else does. The resource the server names in the message tells the two + * apart: it is filled into the text as a literal of its own rather than translated with + * the rest of it, which is why it can be read there (#1011). + */ + final int[] accountLimitCodes; ConnectDialect(String urlPrefix, char parameterSeparator, String[] connectProperties, int connectUnitsPerSecond, long maxConnectSeconds, String[] readProperties, int readUnitsPerSecond, boolean readBoundOutlivesLogin, int[] connectionLimitCodes, int[] notAcceptingYetCodes) { + this(urlPrefix, parameterSeparator, connectProperties, connectUnitsPerSecond, maxConnectSeconds, + readProperties, readUnitsPerSecond, readBoundOutlivesLogin, + connectionLimitCodes, notAcceptingYetCodes, new int[]{}); + } + + ConnectDialect(String urlPrefix, char parameterSeparator, + String[] connectProperties, int connectUnitsPerSecond, long maxConnectSeconds, + String[] readProperties, int readUnitsPerSecond, boolean readBoundOutlivesLogin, + int[] connectionLimitCodes, int[] notAcceptingYetCodes, int[] accountLimitCodes) { this.urlPrefix = urlPrefix; this.parameterSeparator = parameterSeparator; this.connectProperties = connectProperties; @@ -933,6 +963,7 @@ enum ConnectDialect { this.readBoundOutlivesLogin = readBoundOutlivesLogin; this.connectionLimitCodes = connectionLimitCodes; this.notAcceptingYetCodes = notAcceptingYetCodes; + this.accountLimitCodes = accountLimitCodes; } /** The dialect of a connection string, or null for a driver whose property names are not known here. */ @@ -1008,9 +1039,30 @@ private void reportBoundTurnedOffInUrl(String connectionString) { } } - /** Whether a vendor code of this dialect is one that waiting for the database can clear. */ - boolean isWorthRetrying(int errorCode) { - return contains(connectionLimitCodes, errorCode) || contains(notAcceptingYetCodes, errorCode); + /** Whether a failure of this dialect is one that waiting for the database can clear. */ + boolean isWorthRetrying(SQLException e) { + final int errorCode = e.getErrorCode(); + if (contains(connectionLimitCodes, errorCode) || contains(notAcceptingYetCodes, errorCode)) { + return true; + } + // asked of the message of this link and not of the failure as a whole: a wrapper is + // free to carry the text of something else entirely beside the code of this one + return contains(accountLimitCodes, errorCode) && !namesAnHourlyResource(e.getMessage()); + } + + /** + * Whether the message of a failure names a resource an account is granted per hour. Every + * one of them ends in _per_hour - max_connections_per_hour is the one a connect meets, + * max_queries_per_hour and max_updates_per_hour reach a statement - so the suffix is asked + * for rather than the three names: a resource added to that family would otherwise be + * waited out an hour long, a borrow at a time, with a worker thread parked in each of them. + * A message naming none of them - a proxy that rewrote it, a driver that kept the code and + * dropped the text - is taken for the concurrent limit: that is the one an account is given + * in practice, the wait it costs is bounded by the deadline of the borrow, and reading it + * as permanent fails an operation a connection of ours coming back would have served. + */ + private static boolean namesAnHourlyResource(String message) { + return message != null && message.contains("_per_hour"); } private static boolean contains(int[] codes, int code) { @@ -1669,6 +1721,14 @@ private static boolean setNetworkTimeout(Connection con, int millis, String cons * recovers - the one JDBCStorage.open() has no second attempt of its own for, so a backend * that meets it stays locked down until the server is restarted. Both clear themselves in * seconds; every other failure is the caller's to see. + *

+ * The limit an account is given of its own is the first of those cases and not a refusal: the + * connections it caps are the connections of this pool, and one of them is on its way back. + * Which is not how a database reports it - mysql answers a grant's MAX_USER_CONNECTIONS in the + * syntax error class, oracle a profile's SESSIONS_PER_USER with no connection class at all - + * so the vendor code of the dialect is the whole of what tells such a limit from a statement + * the database rejected, and a code that also carries a limit granted per hour is asked about + * the resource its message names ({@link ConnectDialect#accountLimitCodes}, #1011). */ static boolean isWorthRetrying(SQLException e, ConnectDialect dialect) { // a failure of the driver is often wrapped, and a SQLException carries two chains of its @@ -1682,7 +1742,7 @@ static boolean isWorthRetrying(SQLException e, ConnectDialect dialect) { final SQLException sql = (SQLException) t; final String sqlState = sql.getSQLState(); if (CONNECTION_LIMIT_SQL_STATE.equals(sqlState) || NOT_ACCEPTING_YET_SQL_STATE.equals(sqlState) - || (dialect != null && dialect.isWorthRetrying(sql.getErrorCode()))) { + || (dialect != null && dialect.isWorthRetrying(sql))) { return true; } enqueue(pending, visited, sql.getNextException()); diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java index 3e834f39b7..560277d69c 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java @@ -1103,6 +1103,92 @@ public void testRejectedLoginIsNotRetried() throws Exception { assertEquals(stub.attempts.get(), 1, "a rejected login must be attempted once"); } + /** + * The limit an account is given of its own is the same failure as the limit of the server, and + * clears the same way: the connections this account may hold at once are held by this pool, and + * one of them is on its way back to it. mysql reports it as 1226 ER_USER_LIMIT_REACHED and in + * the syntax error class - 42000, where a statement the database refused lands - so the vendor + * code is the whole of what tells the two apart (#1011). + */ + @Test(timeOut = 120000) + public void testThePerAccountConnectionLimitOfMysqlIsRetried() { + assertTrue(CachedConnection.isWorthRetrying( + new SQLException("User 'opendj' has exceeded the 'max_user_connections' resource (current value: 4)", + "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "a per-account connection limit clears when a connection of this pool goes back to it"); + } + + /** + * The same code carries the limits an account is given per hour, and those are cleared by the + * top of the hour rather than by a connection coming back. Waiting one out would cost every + * borrow the whole deadline of the pool - a worker thread apiece, for as long as the hour lasts + * - and would hide the message naming the resource behind a timeout, so it is reported at once. + * The resource is named by the server as a literal of its own, which is why it can be read out + * of a message whose text is otherwise the server's to translate. + */ + @Test(timeOut = 120000) + public void testTheHourlyLimitOfAMysqlAccountIsNotRetried() { + assertFalse(CachedConnection.isWorthRetrying( + new SQLException("User 'opendj' has exceeded the 'max_connections_per_hour' resource (current value: 5)", + "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "an hourly limit is not cleared by a connection of this pool coming back"); + } + + /** + * A 1226 that names no resource - a proxy that rewrote the message, a driver that kept the code + * and not the text - is waited out rather than reported: the concurrent limit is the one an + * account is given in practice, the wait it costs is bounded by the deadline of the borrow, and + * reading it as permanent fails an operation a connection of ours would have served. + */ + @Test(timeOut = 120000) + public void testAMysqlAccountLimitWhoseResourceIsNotNamedIsRetried() { + assertTrue(CachedConnection.isWorthRetrying(new SQLException("connection rejected", "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "a limit whose resource is not named is taken for the concurrent one"); + } + + /** + * A host the server blocked is no limit that clears itself: the host cache holds the block until + * an administrator flushes it, so waiting the deadline of a borrow out would only hide the one + * message naming the remedy behind a timeout. It belongs with the password that is not accepted. + */ + @Test(timeOut = 120000) + public void testAHostMysqlBlockedIsNotRetried() { + assertFalse(CachedConnection.isWorthRetrying( + new SQLException("Host 'ldap1.example.com' is blocked because of many connection errors;" + + " unblock with 'mysqladmin flush-hosts'", "HY000", 1129), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "a blocked host is cleared by an administrator, not by waiting"); + } + + /** + * The sessions an oracle account may hold at once are the SESSIONS_PER_USER of its profile, and + * ORA-02391 is the per-account sibling of the ORA-00020 this table knew already: both are + * cleared by a session ending, which for this pool is a connection of its own going back to it. + */ + @Test(timeOut = 120000) + public void testTheSessionLimitOfAnOracleProfileIsRetried() { + // the state as ojdbc reported it against a profile with sessions_per_user 1: 61000, no + // connection class of its own either, so the vendor code is again the whole of the verdict + assertTrue(CachedConnection.isWorthRetrying( + new SQLException("ORA-02391: exceeded simultaneous SESSIONS_PER_USER limit", "61000", 2391), + CachedConnection.ConnectDialect.of("jdbc:oracle:thin:@db.example.com:1521/FREEPDB1")), + "the session limit of a profile is cleared by a session of this pool ending"); + } + + /** ORA-00018, the same limit as the instance keeps it: a session of somebody's has to end. */ + @Test(timeOut = 120000) + public void testTheSessionLimitOfAnOracleInstanceIsRetried() { + // no state: an instance out of sessions is not something this test could provoke to measure + // one from, and the vendor code is what the verdict is made of + assertTrue(CachedConnection.isWorthRetrying( + new SQLException("ORA-00018: maximum number of sessions exceeded", null, 18), + CachedConnection.ConnectDialect.of("jdbc:oracle:thin:@db.example.com:1521/FREEPDB1")), + "the session limit of an instance is cleared by a session ending"); + } + /** A connection the setup of which failed belongs to nobody: it has to be closed, not leaked. */ @Test(timeOut = 120000) public void testConnectionIsClosedWhenItsSetupFails() throws Exception { diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java index a7462dfbb1..8d310f4915 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java @@ -11,7 +11,7 @@ * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions Copyright [year] [name of copyright owner]". * - * Copyright 2025 3A Systems, LLC. + * Copyright 2025-2026 3A Systems, LLC. */ package org.opends.server.backends.jdbc; @@ -19,9 +19,20 @@ import org.testcontainers.containers.MySQLContainer; import org.testng.annotations.Test; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + //docker run --rm --name mysql -p 3306:3306 -e MYSQL_DATABASE=database_name -e MYSQL_ROOT_PASSWORD=password mysql:latest -@Test +// sequential, as every suite declaring a test of its own is: TestListener asks it of the class a +// running test is declared by, and the inherited ones answer for the class that declares them +@Test(sequential = true) public class MySqlTestCase extends TestCase { @Override @@ -48,4 +59,46 @@ protected String getJdbcUrl() { return "jdbc:mysql://root:password@localhost:" + ((container==null)?"3306":container.getMappedPort(3306)) + "/database_name"; } + /** + * The vendor code a per-account connection limit is classified by is the code the server sends + * for it, and it is the whole of what this verdict can be made of: an account whose grant caps + * its simultaneous connections refuses the next connect with 1226 in the syntax error class - + * 42000, where a statement the database rejected lands - rather than in a connection class of + * its own. The unit tests pin what the pool does with the code; this pins that the code is the + * one arriving from a real server through the driver this backend ships with (#1011). + */ + @Test(timeOut = 120000) + public void testAPerAccountConnectionLimitIsWorthRetrying() throws Exception { + final String url = getJdbcUrl(); + final String limited = url.replace("root:password", "limited1011:secret"); + // dropped first: a run this one was killed in the middle of leaves the account behind + grant(url, "drop user if exists 'limited1011'@'%'", + "create user 'limited1011'@'%' identified by 'secret' with max_user_connections 1", + "grant all on database_name.* to 'limited1011'@'%'"); + try (final Connection held = DriverManager.getConnection(limited)) { + assertTrue(held.isValid(CachedConnection.VALIDATION_TIMEOUT_SECONDS), + "the account is refused its first connection already"); + try (final Connection second = DriverManager.getConnection(limited)) { + fail("an account limited to one connection must be refused a second: " + second); + } catch (SQLException refused) { + assertEquals(refused.getErrorCode(), 1226, + "the server reports a per-account connection limit as: " + refused); + assertTrue(CachedConnection.isWorthRetrying(refused, CachedConnection.ConnectDialect.of(limited)), + "a borrow must wait a per-account limit out rather than fail on it: " + refused); + } + } finally { + grant(url, "drop user if exists 'limited1011'@'%'"); + } + } + + /** The account of the test is made and unmade on the connection of the suite's own credentials. */ + private static void grant(String url, String... statements) throws SQLException { + try (final Connection admin = DriverManager.getConnection(url); + final Statement st = admin.createStatement()) { + for (final String statement : statements) { + st.execute(statement); + } + } + } + } From 2b0cb073d9c5b2014d46e7c7924bdf594a9e8e7a Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Mon, 21 Sep 2026 16:27:44 +0300 Subject: [PATCH 2/2] [#1011] Read the resource of a mysql account limit by name, not by its suffix The gate told the connections an account may hold at once from the resources it is granted per hour by the _per_hour of the message, and mysql names two of the three hourly ones without it: max_questions and max_updates, measured against mysql:9.2. A 1226 naming max_questions reaches the road this gate guards - an account whose hourly quota is spent is refused the login itself, Connector/J spending what is left of it on the queries of its own handshake - so every borrow and every catalog connect waited its deadline out until the hour turned, where master reported the refusal at once. The resource is read by name now: max_user_connections is waited out, everything else the server names is reported, and a message naming no resource at all is still taken for the concurrent limit. --- .../backends/jdbc/CachedConnection.java | 62 ++++++++++--- .../jdbc/CachedConnectionTestCase.java | 86 +++++++++++++++++++ .../server/backends/jdbc/MySqlTestCase.java | 83 ++++++++++++++++++ 3 files changed, 219 insertions(+), 12 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java index 358b449926..1512b87dc0 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java @@ -939,6 +939,14 @@ enum ConnectDialect { * the rest of it, which is why it can be read there (#1011). */ final int[] accountLimitCodes; + /** + * The resource such a code names, as the server writes it: quoted, lower case and starting + * in max_, which is what every resource of ER_USER_LIMIT_REACHED is called - and what the + * user name beside it in the same message is not, save for an account named after one. + */ + private static final Pattern GRANTED_RESOURCE = Pattern.compile("'(max_[a-z_]+)'"); + /** the one resource of those codes that a connection of this pool going back clears */ + private static final String CONCURRENT_ACCOUNT_LIMIT = "max_user_connections"; ConnectDialect(String urlPrefix, char parameterSeparator, String[] connectProperties, int connectUnitsPerSecond, long maxConnectSeconds, @@ -1047,22 +1055,52 @@ boolean isWorthRetrying(SQLException e) { } // asked of the message of this link and not of the failure as a whole: a wrapper is // free to carry the text of something else entirely beside the code of this one - return contains(accountLimitCodes, errorCode) && !namesAnHourlyResource(e.getMessage()); + return contains(accountLimitCodes, errorCode) && !namesALimitNoConnectionClears(e.getMessage()); } /** - * Whether the message of a failure names a resource an account is granted per hour. Every - * one of them ends in _per_hour - max_connections_per_hour is the one a connect meets, - * max_queries_per_hour and max_updates_per_hour reach a statement - so the suffix is asked - * for rather than the three names: a resource added to that family would otherwise be - * waited out an hour long, a borrow at a time, with a worker thread parked in each of them. - * A message naming none of them - a proxy that rewrote it, a driver that kept the code and - * dropped the text - is taken for the concurrent limit: that is the one an account is given - * in practice, the wait it costs is bounded by the deadline of the borrow, and reading it - * as permanent fails an operation a connection of ours coming back would have served. + * Whether the message of a failure names a resource of an account that no connection of + * this pool coming back can clear. + *

+ * The resource is asked for by name rather than by the shape of the name: mysql fills it + * into ER_USER_LIMIT_REACHED - "User '%s' has exceeded the '%s' resource (current value: + * %ld)" - as a literal of its own, and the literals are not the keywords of GRANT. + * Measured against mysql:9.2, a grant of MAX_USER_CONNECTIONS is named + * max_user_connections and MAX_CONNECTIONS_PER_HOUR is named max_connections_per_hour, + * while MAX_QUERIES_PER_HOUR and MAX_UPDATES_PER_HOUR are named max_questions and + * max_updates - two of the three granted per hour carrying no _per_hour about them, so a + * suffix is no way to tell the families apart. + *

+ * What is asked instead is the one resource of this code a wait does clear: the + * connections this account may hold at once are held by this pool, and one of them is on + * its way back. Everything else the server names is left to the caller - the resources + * granted per hour are cleared by the top of the hour and nothing else, and waiting one of + * those out would cost every borrow the whole deadline of the pool, a worker thread parked + * in each, for as long as the hour lasts, with the message naming the resource hidden + * behind a timeout. A resource this code carries and this server has yet to be given is + * treated the same way: reported, the way master reported every one of them. + *

+ * A message naming no resource at all - a proxy that rewrote it, a driver that kept the + * code and dropped the text - is taken for the concurrent limit: that is the one an + * account is given in practice, the wait it costs is bounded by the deadline of the + * borrow, and reading it as permanent fails an operation a connection of ours coming back + * would have served. The literal is read wherever it stands rather than out of the + * sentence around it, since the sentence is the server's to translate while the resource + * it fills in is not (#1011). */ - private static boolean namesAnHourlyResource(String message) { - return message != null && message.contains("_per_hour"); + private static boolean namesALimitNoConnectionClears(String message) { + if (message == null) { + return false; + } + final Matcher resource = GRANTED_RESOURCE.matcher(message); + boolean named = false; + while (resource.find()) { + if (CONCURRENT_ACCOUNT_LIMIT.equals(resource.group(1))) { + return false; // the limit of this account on the connections this pool holds + } + named = true; + } + return named; } private static boolean contains(int[] codes, int code) { diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java index 560277d69c..2994cee74f 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java @@ -1136,6 +1136,54 @@ public void testTheHourlyLimitOfAMysqlAccountIsNotRetried() { "an hourly limit is not cleared by a connection of this pool coming back"); } + /** + * ... and the resource is read by the name the server gives it rather than by the shape of that + * name: the queries an account is granted per hour are named max_questions, with no _per_hour + * about them (measured against mysql:9.2). This one reaches the road this gate guards - an + * account whose quota is spent is refused the connect itself, Connector/J spending what is left + * of it on the queries of its own login - so a wait would park a worker thread in every borrow + * and every catalog connect until the hour turns, with the message naming the resource hidden + * behind the timeout of the borrow. + */ + @Test(timeOut = 120000) + public void testTheHourlyQueryLimitOfAMysqlAccountIsNotRetried() { + assertFalse(CachedConnection.isWorthRetrying( + new SQLException("User 'opendj' has exceeded the 'max_questions' resource (current value: 5)", + "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "the queries granted per hour are named max_questions, and no connection coming back clears them"); + } + + /** + * The same for the updates granted per hour, which the server names max_updates: this one is met + * by a statement that changes data rather than by a connect, so it reaches this gate only where + * a driver hands the pool a failure of something else entirely - and it is no more cleared by a + * connection coming back than the queries are. + */ + @Test(timeOut = 120000) + public void testTheHourlyUpdateLimitOfAMysqlAccountIsNotRetried() { + assertFalse(CachedConnection.isWorthRetrying( + new SQLException("User 'opendj' has exceeded the 'max_updates' resource (current value: 5)", + "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "the updates granted per hour are named max_updates, and no connection coming back clears them"); + } + + /** + * A resource this code carries that this server has yet to be given is reported rather than + * waited out: what the wait clears is the connections of this pool, and a resource nobody here + * has heard of is not those. That is what master did with every one of these codes, so an + * unknown resource costs a borrow nothing it did not cost before (#1011). + */ + @Test(timeOut = 120000) + public void testAMysqlResourceThisGateDoesNotKnowIsNotRetried() { + assertFalse(CachedConnection.isWorthRetrying( + new SQLException("User 'opendj' has exceeded the 'max_statements_per_minute' resource" + + " (current value: 5)", "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "a resource of this code that is not the concurrent limit is the caller's to see"); + } + /** * A 1226 that names no resource - a proxy that rewrote the message, a driver that kept the code * and not the text - is waited out rather than reported: the concurrent limit is the one an @@ -1149,6 +1197,14 @@ public void testAMysqlAccountLimitWhoseResourceIsNotNamedIsRetried() { "a limit whose resource is not named is taken for the concurrent one"); } + /** ... which is the message of a driver that kept the code and dropped the text: none at all. */ + @Test(timeOut = 120000) + public void testAMysqlAccountLimitWithNoMessageIsRetried() { + assertTrue(CachedConnection.isWorthRetrying(new SQLException(null, "42000", 1226), + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "a limit arriving without a message is taken for the concurrent one rather than thrown at"); + } + /** * A host the server blocked is no limit that clears itself: the host cache holds the block until * an administrator flushes it, so waiting the deadline of a borrow out would only hide the one @@ -1163,6 +1219,36 @@ public void testAHostMysqlBlockedIsNotRetried() { "a blocked host is cleared by an administrator, not by waiting"); } + /** + * The resource is read off the link the code arrived on, not off the failure as a whole: a + * wrapper - a proxy, a DataSource of a container - is free to carry the text of something else + * entirely above the exception that carries the code, and a verdict made of the two together + * belongs to neither of them. + */ + @Test(timeOut = 120000) + public void testTheResourceIsReadOffTheLinkCarryingTheCode() { + final SQLException wrapped = new SQLException("could not connect: the account has exceeded the" + + " 'max_connections_per_hour' resource", "08006", + new SQLException("User 'opendj' has exceeded the 'max_user_connections' resource (current value: 4)", + "42000", 1226)); + + assertTrue(CachedConnection.isWorthRetrying(wrapped, + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "the resource of the link carrying the code is what the verdict is made of"); + } + + /** ... and the other way around: the text of the wrapper decides nothing for the code beneath it. */ + @Test(timeOut = 120000) + public void testTheResourceOfAWrapperDecidesNothing() { + final SQLException wrapped = new SQLException("could not connect to the server", "08006", + new SQLException("User 'opendj' has exceeded the 'max_connections_per_hour' resource (current value: 5)", + "42000", 1226)); + + assertFalse(CachedConnection.isWorthRetrying(wrapped, + CachedConnection.ConnectDialect.of("jdbc:mysql://db.example.com:3306/opendj")), + "an hourly limit stays an hourly limit under a wrapper that names no resource"); + } + /** * The sessions an oracle account may hold at once are the SESSIONS_PER_USER of its profile, and * ORA-02391 is the per-account sibling of the ORA-00020 this table knew already: both are diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java index 8d310f4915..a3d325c4c7 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/MySqlTestCase.java @@ -25,6 +25,7 @@ import java.sql.Statement; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -83,6 +84,10 @@ public void testAPerAccountConnectionLimitIsWorthRetrying() throws Exception { } catch (SQLException refused) { assertEquals(refused.getErrorCode(), 1226, "the server reports a per-account connection limit as: " + refused); + // the resource is what tells this code from the ones no wait clears, and the server + // writes it as a literal of its own: read here from the server rather than assumed + assertTrue(refused.getMessage().contains("'max_user_connections'"), + "the server names the resource of the concurrent limit as: " + refused.getMessage()); assertTrue(CachedConnection.isWorthRetrying(refused, CachedConnection.ConnectDialect.of(limited)), "a borrow must wait a per-account limit out rather than fail on it: " + refused); } @@ -91,6 +96,84 @@ public void testAPerAccountConnectionLimitIsWorthRetrying() throws Exception { } } + /** + * The connections an account is granted per hour carry the same 1226 as the ones it may hold at + * once, and only the top of the hour clears them: a borrow waiting one out would park a worker + * thread in every attempt for as long as the hour lasts. What tells the two apart is the + * resource the server names, and this pins that name against the server itself (#1011). + */ + @Test(timeOut = 120000) + public void testTheHourlyConnectionLimitIsNotWorthRetrying() throws Exception { + final String url = getJdbcUrl(); + final String limited = url.replace("root:password", "perhour1011:secret"); + grant(url, "drop user if exists 'perhour1011'@'%'", + "create user 'perhour1011'@'%' identified by 'secret' with max_connections_per_hour 1", + "grant all on database_name.* to 'perhour1011'@'%'"); + try { + try (final Connection spent = DriverManager.getConnection(limited)) { + assertTrue(spent.isValid(CachedConnection.VALIDATION_TIMEOUT_SECONDS), + "the account is refused the one connection of its hour already"); + } + try (final Connection second = DriverManager.getConnection(limited)) { + fail("an account granted one connection an hour must be refused a second: " + second); + } catch (SQLException refused) { + assertEquals(refused.getErrorCode(), 1226, + "the server reports an hourly connection limit as: " + refused); + assertTrue(refused.getMessage().contains("'max_connections_per_hour'"), + "the server names the hourly resource as: " + refused.getMessage()); + assertFalse(CachedConnection.isWorthRetrying(refused, CachedConnection.ConnectDialect.of(limited)), + "an hourly limit must be reported rather than waited out: " + refused); + } + } finally { + grant(url, "drop user if exists 'perhour1011'@'%'"); + } + } + + /** + * The queries an account is granted per hour are named by the server without the _per_hour of + * the GRANT keyword - max_questions - and they reach this gate on the road it guards: an + * account whose quota is spent is refused the connect itself, Connector/J spending what is + * left of the quota on the queries of its own login. Waited out, that would cost every borrow + * and every catalog connect the whole deadline of the pool until the hour turned (#1011). + */ + @Test(timeOut = 120000) + public void testTheHourlyQueryLimitIsNotWorthRetrying() throws Exception { + final String url = getJdbcUrl(); + final String limited = url.replace("root:password", "hourly1011:secret"); + grant(url, "drop user if exists 'hourly1011'@'%'", + "create user 'hourly1011'@'%' identified by 'secret' with max_queries_per_hour 1", + "grant all on database_name.* to 'hourly1011'@'%'"); + try { + final SQLException refused = spendTheHourlyQueries(limited); + assertEquals(refused.getErrorCode(), 1226, + "the server reports an hourly query limit as: " + refused); + assertTrue(refused.getMessage().contains("'max_questions'"), + "the server names the hourly query resource as: " + refused.getMessage()); + assertFalse(CachedConnection.isWorthRetrying(refused, CachedConnection.ConnectDialect.of(limited)), + "an hourly query limit must be reported rather than waited out: " + refused); + } finally { + grant(url, "drop user if exists 'hourly1011'@'%'"); + } + } + + /** + * Spends the hourly query quota of an account and hands back what the server refused it with. + * The connect is attempted rather than one statement over a connection held open, since the + * login of the driver spends the quota as readily as a statement does: whichever of the two + * meets the limit, the failure is the one a borrow of this pool would catch. + */ + private static SQLException spendTheHourlyQueries(String url) throws SQLException { + for (int attempt = 0; attempt < 4; attempt++) { + try (final Connection con = DriverManager.getConnection(url); + final Statement st = con.createStatement()) { + st.execute("select 1"); + } catch (SQLException refused) { + return refused; + } + } + throw new AssertionError("an account granted one query an hour must be refused within four connects"); + } + /** The account of the test is made and unmade on the connection of the suite's own credentials. */ private static void grant(String url, String... statements) throws SQLException { try (final Connection admin = DriverManager.getConnection(url);