1 /*
2  * Copyright (C) 2013,2014 Brett Wooldridge
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 package com.zaxxer.hikari.pool;
18
19 import com.codahale.metrics.MetricRegistry;
20 import com.codahale.metrics.health.HealthCheckRegistry;
21 import com.zaxxer.hikari.HikariConfig;
22 import com.zaxxer.hikari.HikariPoolMXBean;
23 import com.zaxxer.hikari.metrics.MetricsTrackerFactory;
24 import com.zaxxer.hikari.metrics.PoolStats;
25 import com.zaxxer.hikari.metrics.dropwizard.CodahaleHealthChecker;
26 import com.zaxxer.hikari.metrics.dropwizard.CodahaleMetricsTrackerFactory;
27 import com.zaxxer.hikari.metrics.micrometer.MicrometerMetricsTrackerFactory;
28 import com.zaxxer.hikari.util.ConcurrentBag;
29 import com.zaxxer.hikari.util.ConcurrentBag.IBagStateListener;
30 import com.zaxxer.hikari.util.SuspendResumeLock;
31 import com.zaxxer.hikari.util.UtilityElf.DefaultThreadFactory;
32 import io.micrometer.core.instrument.MeterRegistry;
33 import org.slf4j.Logger;
34 import org.slf4j.LoggerFactory;
35
36 import java.sql.Connection;
37 import java.sql.SQLException;
38 import java.sql.SQLTransientConnectionException;
39 import java.util.Collection;
40 import java.util.List;
41 import java.util.Optional;
42 import java.util.concurrent.Callable;
43 import java.util.concurrent.ExecutorService;
44 import java.util.concurrent.LinkedBlockingQueue;
45 import java.util.concurrent.ScheduledExecutorService;
46 import java.util.concurrent.ScheduledFuture;
47 import java.util.concurrent.ScheduledThreadPoolExecutor;
48 import java.util.concurrent.ThreadFactory;
49 import java.util.concurrent.ThreadLocalRandom;
50 import java.util.concurrent.ThreadPoolExecutor;
51
52 import static com.zaxxer.hikari.util.ClockSource.currentTime;
53 import static com.zaxxer.hikari.util.ClockSource.elapsedDisplayString;
54 import static com.zaxxer.hikari.util.ClockSource.elapsedMillis;
55 import static com.zaxxer.hikari.util.ClockSource.plusMillis;
56 import static com.zaxxer.hikari.util.ConcurrentBag.IConcurrentBagEntry.STATE_IN_USE;
57 import static com.zaxxer.hikari.util.ConcurrentBag.IConcurrentBagEntry.STATE_NOT_IN_USE;
58 import static com.zaxxer.hikari.util.UtilityElf.createThreadPoolExecutor;
59 import static com.zaxxer.hikari.util.UtilityElf.quietlySleep;
60 import static com.zaxxer.hikari.util.UtilityElf.safeIsAssignableFrom;
61 import static java.util.Collections.unmodifiableCollection;
62 import static java.util.concurrent.TimeUnit.MILLISECONDS;
63 import static java.util.concurrent.TimeUnit.SECONDS;
64
65 /**
66  * This is the primary connection pool class that provides the basic
67  * pooling behavior for HikariCP.
68  *
69  * @author Brett Wooldridge
70  */
71 public final class HikariPool extends PoolBase implements HikariPoolMXBean, IBagStateListener
72 {
73    private final Logger logger = LoggerFactory.getLogger(HikariPool.class);
74
75    public static final int POOL_NORMAL = 0;
76    public static final int POOL_SUSPENDED = 1;
77    public static final int POOL_SHUTDOWN = 2;
78
79    public volatile int poolState;
80
81    private final long aliveBypassWindowMs = Long.getLong("com.zaxxer.hikari.aliveBypassWindowMs", MILLISECONDS.toMillis(500));
82    private final long housekeepingPeriodMs = Long.getLong("com.zaxxer.hikari.housekeeping.periodMs", SECONDS.toMillis(30));
83
84    private static final String EVICTED_CONNECTION_MESSAGE = "(connection was evicted)";
85    private static final String DEAD_CONNECTION_MESSAGE = "(connection is dead)";
86
87    private final PoolEntryCreator poolEntryCreator = new PoolEntryCreator(null /*logging prefix*/);
88    private final PoolEntryCreator postFillPoolEntryCreator = new PoolEntryCreator("After adding ");
89    private final Collection<Runnable> addConnectionQueueReadOnlyView;
90    private final ThreadPoolExecutor addConnectionExecutor;
91    private final ThreadPoolExecutor closeConnectionExecutor;
92
93    private final ConcurrentBag<PoolEntry> connectionBag;
94
95    private final ProxyLeakTaskFactory leakTaskFactory;
96    private final SuspendResumeLock suspendResumeLock;
97
98    private final ScheduledExecutorService houseKeepingExecutorService;
99    private ScheduledFuture<?> houseKeeperTask;
100
101    /**
102     * Construct a HikariPool with the specified configuration.
103     *
104     * @param config a HikariConfig instance
105     */
106    public HikariPool(final HikariConfig config)
107    {
108       super(config);
109
110       this.connectionBag = new ConcurrentBag<>(this);
111       this.suspendResumeLock = config.isAllowPoolSuspension() ? new SuspendResumeLock() : SuspendResumeLock.FAUX_LOCK;
112
113       this.houseKeepingExecutorService = initializeHouseKeepingExecutorService();
114
115       checkFailFast();
116
117       if (config.getMetricsTrackerFactory() != null) {
118          setMetricsTrackerFactory(config.getMetricsTrackerFactory());
119       }
120       else {
121          setMetricRegistry(config.getMetricRegistry());
122       }
123
124       setHealthCheckRegistry(config.getHealthCheckRegistry());
125
126       handleMBeans(this, true);
127
128       ThreadFactory threadFactory = config.getThreadFactory();
129
130       final int maxPoolSize = config.getMaximumPoolSize();
131       LinkedBlockingQueue<Runnable> addConnectionQueue = new LinkedBlockingQueue<>(maxPoolSize);
132       this.addConnectionQueueReadOnlyView = unmodifiableCollection(addConnectionQueue);
133       this.addConnectionExecutor = createThreadPoolExecutor(addConnectionQueue, poolName + " connection adder", threadFactory, new ThreadPoolExecutor.DiscardOldestPolicy());
134       this.closeConnectionExecutor = createThreadPoolExecutor(maxPoolSize, poolName + " connection closer", threadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
135
136       this.leakTaskFactory = new ProxyLeakTaskFactory(config.getLeakDetectionThreshold(), houseKeepingExecutorService);
137
138       this.houseKeeperTask = houseKeepingExecutorService.scheduleWithFixedDelay(new HouseKeeper(), 100L, housekeepingPeriodMs, MILLISECONDS);
139
140       if (Boolean.getBoolean("com.zaxxer.hikari.blockUntilFilled") && config.getInitializationFailTimeout() > 1) {
141          addConnectionExecutor.setCorePoolSize(Math.min(16, Runtime.getRuntime().availableProcessors()));
142          addConnectionExecutor.setMaximumPoolSize(Math.min(16, Runtime.getRuntime().availableProcessors()));
143
144          final long startTime = currentTime();
145          while (elapsedMillis(startTime) < config.getInitializationFailTimeout() && getTotalConnections() < config.getMinimumIdle()) {
146             quietlySleep(MILLISECONDS.toMillis(100));
147          }
148
149          addConnectionExecutor.setCorePoolSize(1);
150          addConnectionExecutor.setMaximumPoolSize(1);
151       }
152    }
153
154    /**
155     * Get a connection from the pool, or timeout after connectionTimeout milliseconds.
156     *
157     * @return a java.sql.Connection instance
158     * @throws SQLException thrown if a timeout occurs trying to obtain a connection
159     */
160    public Connection getConnection() throws SQLException
161    {
162       return getConnection(connectionTimeout);
163    }
164
165    /**
166     * Get a connection from the pool, or timeout after the specified number of milliseconds.
167     *
168     * @param hardTimeout the maximum time to wait for a connection from the pool
169     * @return a java.sql.Connection instance
170     * @throws SQLException thrown if a timeout occurs trying to obtain a connection
171     */
172    public Connection getConnection(final long hardTimeout) throws SQLException
173    {
174       suspendResumeLock.acquire();
175       final long startTime = currentTime();
176
177       try {
178          long timeout = hardTimeout;
179          do {
180             PoolEntry poolEntry = connectionBag.borrow(timeout, MILLISECONDS);
181             if (poolEntry == null) {
182                break; // We timed out... break and throw exception
183             }
184
185             final long now = currentTime();
186             if (poolEntry.isMarkedEvicted() || (elapsedMillis(poolEntry.lastAccessed, now) > aliveBypassWindowMs && !isConnectionAlive(poolEntry.connection))) {
187                closeConnection(poolEntry, poolEntry.isMarkedEvicted() ? EVICTED_CONNECTION_MESSAGE : DEAD_CONNECTION_MESSAGE);
188                timeout = hardTimeout - elapsedMillis(startTime);
189             }
190             else {
191                metricsTracker.recordBorrowStats(poolEntry, startTime);
192                return poolEntry.createProxyConnection(leakTaskFactory.schedule(poolEntry), now);
193             }
194          } while (timeout > 0L);
195
196          metricsTracker.recordBorrowTimeoutStats(startTime);
197          throw createTimeoutException(startTime);
198       }
199       catch (InterruptedException e) {
200          Thread.currentThread().interrupt();
201          throw new SQLException(poolName + " - Interrupted during connection acquisition", e);
202       }
203       finally {
204          suspendResumeLock.release();
205       }
206    }
207
208    /**
209     * Shutdown the pool, closing all idle connections and aborting or closing
210     * active connections.
211     *
212     * @throws InterruptedException thrown if the thread is interrupted during shutdown
213     */
214    public synchronized void shutdown() throws InterruptedException
215    {
216       try {
217          poolState = POOL_SHUTDOWN;
218
219          if (addConnectionExecutor == null) { // pool never started
220             return;
221          }
222
223          logPoolState("Before shutdown ");
224
225          if (houseKeeperTask != null) {
226             houseKeeperTask.cancel(false);
227             houseKeeperTask = null;
228          }
229
230          softEvictConnections();
231
232          addConnectionExecutor.shutdown();
233          addConnectionExecutor.awaitTermination(getLoginTimeout(), SECONDS);
234
235          destroyHouseKeepingExecutorService();
236
237          connectionBag.close();
238
239          final ExecutorService assassinExecutor = createThreadPoolExecutor(config.getMaximumPoolSize(), poolName + " connection assassinator",
240                                                                            config.getThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy());
241          try {
242             final long start = currentTime();
243             do {
244                abortActiveConnections(assassinExecutor);
245                softEvictConnections();
246             } while (getTotalConnections() > 0 && elapsedMillis(start) < SECONDS.toMillis(10));
247          }
248          finally {
249             assassinExecutor.shutdown();
250             assassinExecutor.awaitTermination(10L, SECONDS);
251          }
252
253          shutdownNetworkTimeoutExecutor();
254          closeConnectionExecutor.shutdown();
255          closeConnectionExecutor.awaitTermination(10L, SECONDS);
256       }
257       finally {
258          logPoolState("After shutdown ");
259          handleMBeans(this, false);
260          metricsTracker.close();
261       }
262    }
263
264    /**
265     * Evict a Connection from the pool.
266     *
267     * @param connection the Connection to evict (actually a {@link ProxyConnection})
268     */
269    public void evictConnection(Connection connection)
270    {
271       ProxyConnection proxyConnection = (ProxyConnection) connection;
272       proxyConnection.cancelLeakTask();
273
274       try {
275          softEvictConnection(proxyConnection.getPoolEntry(), "(connection evicted by user)", !connection.isClosed() /* owner */);
276       }
277       catch (SQLException e) {
278          // unreachable in HikariCP, but we're still forced to catch it
279       }
280    }
281
282    /**
283     * Set a metrics registry to be used when registering metrics collectors.  The HikariDataSource prevents this
284     * method from being called more than once.
285     *
286     * @param metricRegistry the metrics registry instance to use
287     */
288    public void setMetricRegistry(Object metricRegistry)
289    {
290       if (metricRegistry != null && safeIsAssignableFrom(metricRegistry, "com.codahale.metrics.MetricRegistry")) {
291          setMetricsTrackerFactory(new CodahaleMetricsTrackerFactory((MetricRegistry) metricRegistry));
292       }
293       else if (metricRegistry != null && safeIsAssignableFrom(metricRegistry, "io.micrometer.core.instrument.MeterRegistry")) {
294          setMetricsTrackerFactory(new MicrometerMetricsTrackerFactory((MeterRegistry) metricRegistry));
295       }
296       else {
297          setMetricsTrackerFactory(null);
298       }
299    }
300
301    /**
302     * Set the MetricsTrackerFactory to be used to create the IMetricsTracker instance used by the pool.
303     *
304     * @param metricsTrackerFactory an instance of a class that subclasses MetricsTrackerFactory
305     */
306    public void setMetricsTrackerFactory(MetricsTrackerFactory metricsTrackerFactory)
307    {
308       if (metricsTrackerFactory != null) {
309          this.metricsTracker = new MetricsTrackerDelegate(metricsTrackerFactory.create(config.getPoolName(), getPoolStats()));
310       }
311       else {
312          this.metricsTracker = new NopMetricsTrackerDelegate();
313       }
314    }
315
316    /**
317     * Set the health check registry to be used when registering health checks.  Currently only Codahale health
318     * checks are supported.
319     *
320     * @param healthCheckRegistry the health check registry instance to use
321     */
322    public void setHealthCheckRegistry(Object healthCheckRegistry)
323    {
324       if (healthCheckRegistry != null) {
325          CodahaleHealthChecker.registerHealthChecks(this, config, (HealthCheckRegistry) healthCheckRegistry);
326       }
327    }
328
329    // ***********************************************************************
330    //                        IBagStateListener callback
331    // ***********************************************************************
332
333    /** {@inheritDoc} */
334    @Override
335    public void addBagItem(final int waiting)
336    {
337       final boolean shouldAdd = waiting - addConnectionQueueReadOnlyView.size() >= 0; // Yes, >= is intentional.
338       if (shouldAdd) {
339          addConnectionExecutor.submit(poolEntryCreator);
340       }
341       else {
342          logger.debug("{} - Add connection elided, waiting {}, queue {}", poolName, waiting, addConnectionQueueReadOnlyView.size());
343       }
344    }
345
346    // ***********************************************************************
347    //                        HikariPoolMBean methods
348    // ***********************************************************************
349
350    /** {@inheritDoc} */
351    @Override
352    public int getActiveConnections()
353    {
354       return connectionBag.getCount(STATE_IN_USE);
355    }
356
357    /** {@inheritDoc} */
358    @Override
359    public int getIdleConnections()
360    {
361       return connectionBag.getCount(STATE_NOT_IN_USE);
362    }
363
364    /** {@inheritDoc} */
365    @Override
366    public int getTotalConnections()
367    {
368       return connectionBag.size();
369    }
370
371    /** {@inheritDoc} */
372    @Override
373    public int getThreadsAwaitingConnection()
374    {
375       return connectionBag.getWaitingThreadCount();
376    }
377
378    /** {@inheritDoc} */
379    @Override
380    public void softEvictConnections()
381    {
382       connectionBag.values().forEach(poolEntry -> softEvictConnection(poolEntry, "(connection evicted)", false /* not owner */));
383    }
384
385    /** {@inheritDoc} */
386    @Override
387    public synchronized void suspendPool()
388    {
389       if (suspendResumeLock == SuspendResumeLock.FAUX_LOCK) {
390          throw new IllegalStateException(poolName + " - is not suspendable");
391       }
392       else if (poolState != POOL_SUSPENDED) {
393          suspendResumeLock.suspend();
394          poolState = POOL_SUSPENDED;
395       }
396    }
397
398    /** {@inheritDoc} */
399    @Override
400    public synchronized void resumePool()
401    {
402       if (poolState == POOL_SUSPENDED) {
403          poolState = POOL_NORMAL;
404          fillPool();
405          suspendResumeLock.resume();
406       }
407    }
408
409    // ***********************************************************************
410    //                           Package methods
411    // ***********************************************************************
412
413    /**
414     * Log the current pool state at debug level.
415     *
416     * @param prefix an optional prefix to prepend the log message
417     */
418    void logPoolState(String... prefix)
419    {
420       if (logger.isDebugEnabled()) {
421          logger.debug("{} - {}stats (total={}, active={}, idle={}, waiting={})",
422                       poolName, (prefix.length > 0 ? prefix[0] : ""),
423                       getTotalConnections(), getActiveConnections(), getIdleConnections(), getThreadsAwaitingConnection());
424       }
425    }
426
427    /**
428     * Recycle PoolEntry (add back to the pool)
429     *
430     * @param poolEntry the PoolEntry to recycle
431     */
432    @Override
433    void recycle(final PoolEntry poolEntry)
434    {
435       metricsTracker.recordConnectionUsage(poolEntry);
436
437       connectionBag.requite(poolEntry);
438    }
439
440    /**
441     * Permanently close the real (underlying) connection (eat any exception).
442     *
443     * @param poolEntry poolEntry having the connection to close
444     * @param closureReason reason to close
445     */
446    void closeConnection(final PoolEntry poolEntry, final String closureReason)
447    {
448       if (connectionBag.remove(poolEntry)) {
449          final Connection connection = poolEntry.close();
450          closeConnectionExecutor.execute(() -> {
451             quietlyCloseConnection(connection, closureReason);
452             if (poolState == POOL_NORMAL) {
453                fillPool();
454             }
455          });
456       }
457    }
458
459    @SuppressWarnings("unused")
460    int[] getPoolStateCounts()
461    {
462       return connectionBag.getStateCounts();
463    }
464
465
466    // ***********************************************************************
467    //                           Private methods
468    // ***********************************************************************
469
470    /**
471     * Creating new poolEntry.  If maxLifetime is configured, create a future End-of-life task with 2.5% variance from
472     * the maxLifetime time to ensure there is no massive die-off of Connections in the pool.
473     */
474    private PoolEntry createPoolEntry()
475    {
476       try {
477          final PoolEntry poolEntry = newPoolEntry();
478
479          final long maxLifetime = config.getMaxLifetime();
480          if (maxLifetime > 0) {
481             // variance up to 2.5% of the maxlifetime
482             final long variance = maxLifetime > 10_000 ? ThreadLocalRandom.current().nextLong( maxLifetime / 40 ) : 0;
483             final long lifetime = maxLifetime - variance;
484             poolEntry.setFutureEol(houseKeepingExecutorService.schedule(
485                () -> {
486                   if (softEvictConnection(poolEntry, "(connection has passed maxLifetime)", false /* not owner */)) {
487                      addBagItem(connectionBag.getWaitingThreadCount());
488                   }
489                },
490                lifetime, MILLISECONDS));
491          }
492
493          return poolEntry;
494       }
495       catch (ConnectionSetupException e) {
496          if (poolState == POOL_NORMAL) { // we check POOL_NORMAL to avoid a flood of messages if shutdown() is running concurrently
497             logger.error("{} - Error thrown while acquiring connection from data source", poolName, e.getCause());
498             lastConnectionFailure.set(e);
499          }
500       }
501       catch (Exception e) {
502          if (poolState == POOL_NORMAL) { // we check POOL_NORMAL to avoid a flood of messages if shutdown() is running concurrently
503             logger.debug("{} - Cannot acquire connection from data source", poolName, e);
504          }
505       }
506
507       return null;
508    }
509
510    /**
511     * Fill pool up from current idle connections (as they are perceived at the point of execution) to minimumIdle connections.
512     */
513    private synchronized void fillPool()
514    {
515       final int connectionsToAdd = Math.min(config.getMaximumPoolSize() - getTotalConnections(), config.getMinimumIdle() - getIdleConnections())
516                                    - addConnectionQueueReadOnlyView.size();
517       if (connectionsToAdd <= 0) logger.debug("{} - Fill pool skipped, pool is at sufficient level.", poolName);
518
519       for (int i = 0; i < connectionsToAdd; i++) {
520          addConnectionExecutor.submit((i < connectionsToAdd - 1) ? poolEntryCreator : postFillPoolEntryCreator);
521       }
522    }
523
524    /**
525     * Attempt to abort or close active connections.
526     *
527     * @param assassinExecutor the ExecutorService to pass to Connection.abort()
528     */
529    private void abortActiveConnections(final ExecutorService assassinExecutor)
530    {
531       for (PoolEntry poolEntry : connectionBag.values(STATE_IN_USE)) {
532          Connection connection = poolEntry.close();
533          try {
534             connection.abort(assassinExecutor);
535          }
536          catch (Throwable e) {
537             quietlyCloseConnection(connection, "(connection aborted during shutdown)");
538          }
539          finally {
540             connectionBag.remove(poolEntry);
541          }
542       }
543    }
544
545    /**
546     * If initializationFailFast is configured, check that we have DB connectivity.
547     *
548     * @throws PoolInitializationException if fails to create or validate connection
549     * @see HikariConfig#setInitializationFailTimeout(long)
550     */
551    private void checkFailFast()
552    {
553       final long initializationTimeout = config.getInitializationFailTimeout();
554       if (initializationTimeout < 0) {
555          return;
556       }
557
558       final long startTime = currentTime();
559       do {
560          final PoolEntry poolEntry = createPoolEntry();
561          if (poolEntry != null) {
562             if (config.getMinimumIdle() > 0) {
563                connectionBag.add(poolEntry);
564                logger.debug("{} - Added connection {}", poolName, poolEntry.connection);
565             }
566             else {
567                quietlyCloseConnection(poolEntry.close(), "(initialization check complete and minimumIdle is zero)");
568             }
569
570             return;
571          }
572
573          if (getLastConnectionFailure() instanceof ConnectionSetupException) {
574             throwPoolInitializationException(getLastConnectionFailure().getCause());
575          }
576
577          quietlySleep(SECONDS.toMillis(1));
578       } while (elapsedMillis(startTime) < initializationTimeout);
579
580       if (initializationTimeout > 0) {
581          throwPoolInitializationException(getLastConnectionFailure());
582       }
583    }
584
585    /**
586     * Log the Throwable that caused pool initialization to fail, and then throw a PoolInitializationException with
587     * that cause attached.
588     *
589     * @param t the Throwable that caused the pool to fail to initialize (possibly null)
590     */
591    private void throwPoolInitializationException(Throwable t)
592    {
593       logger.error("{} - Exception during pool initialization.", poolName, t);
594       destroyHouseKeepingExecutorService();
595       throw new PoolInitializationException(t);
596    }
597
598    /**
599     * "Soft" evict a Connection (/PoolEntry) from the pool.  If this method is being called by the user directly
600     * through {@link com.zaxxer.hikari.HikariDataSource#evictConnection(Connection)} then {@code owner} is {@code true}.
601     *
602     * If the caller is the owner, or if the Connection is idle (i.e. can be "reserved" in the {@link ConcurrentBag}),
603     * then we can close the connection immediately.  Otherwise, we leave it "marked" for eviction so that it is evicted
604     * the next time someone tries to acquire it from the pool.
605     *
606     * @param poolEntry the PoolEntry (/Connection) to "soft" evict from the pool
607     * @param reason the reason that the connection is being evicted
608     * @param owner true if the caller is the owner of the connection, false otherwise
609     * @return true if the connection was evicted (closed), false if it was merely marked for eviction
610     */
611    private boolean softEvictConnection(final PoolEntry poolEntry, final String reason, final boolean owner)
612    {
613       poolEntry.markEvicted();
614       if (owner || connectionBag.reserve(poolEntry)) {
615          closeConnection(poolEntry, reason);
616          return true;
617       }
618
619       return false;
620    }
621
622    /**
623     * Create/initialize the Housekeeping service {@link ScheduledExecutorService}.  If the user specified an Executor
624     * to be used in the {@link HikariConfig}, then we use that.  If no Executor was specified (typical), then create
625     * an Executor and configure it.
626     *
627     * @return either the user specified {@link ScheduledExecutorService}, or the one we created
628     */
629    private ScheduledExecutorService initializeHouseKeepingExecutorService()
630    {
631       if (config.getScheduledExecutor() == null) {
632          final ThreadFactory threadFactory = Optional.ofNullable(config.getThreadFactory()).orElseGet(() -> new DefaultThreadFactory(poolName + " housekeeper", true));
633          final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1, threadFactory, new ThreadPoolExecutor.DiscardPolicy());
634          executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
635          executor.setRemoveOnCancelPolicy(true);
636          return executor;
637       }
638       else {
639          return config.getScheduledExecutor();
640       }
641    }
642
643    /**
644     * Destroy (/shutdown) the Housekeeping service Executor, if it was the one that we created.
645     */
646    private void destroyHouseKeepingExecutorService()
647    {
648       if (config.getScheduledExecutor() == null) {
649          houseKeepingExecutorService.shutdownNow();
650       }
651    }
652
653    /**
654     * Create a PoolStats instance that will be used by metrics tracking, with a pollable resolution of 1 second.
655     *
656     * @return a PoolStats instance
657     */
658    private PoolStats getPoolStats()
659    {
660       return new PoolStats(SECONDS.toMillis(1)) {
661          @Override
662          protected void update() {
663             this.pendingThreads = HikariPool.this.getThreadsAwaitingConnection();
664             this.idleConnections = HikariPool.this.getIdleConnections();
665             this.totalConnections = HikariPool.this.getTotalConnections();
666             this.activeConnections = HikariPool.this.getActiveConnections();
667             this.maxConnections = config.getMaximumPoolSize();
668             this.minConnections = config.getMinimumIdle();
669          }
670       };
671    }
672
673    /**
674     * Create a timeout exception (specifically, {@link SQLTransientConnectionException}) to be thrown, because a
675     * timeout occurred when trying to acquire a Connection from the pool.  If there was an underlying cause for the
676     * timeout, e.g. a SQLException thrown by the driver while trying to create a new Connection, then use the
677     * SQL State from that exception as our own and additionally set that exception as the "next" SQLException inside
678     * of our exception.
679     *
680     * As a side-effect, log the timeout failure at DEBUG, and record the timeout failure in the metrics tracker.
681     *
682     * @param startTime the start time (timestamp) of the acquisition attempt
683     * @return a SQLException to be thrown from {@link #getConnection()}
684     */
685    private SQLException createTimeoutException(long startTime)
686    {
687       logPoolState("Timeout failure ");
688       metricsTracker.recordConnectionTimeout();
689
690       String sqlState = null;
691       final Throwable originalException = getLastConnectionFailure();
692       if (originalException instanceof SQLException) {
693          sqlState = ((SQLException) originalException).getSQLState();
694       }
695       final SQLException connectionException = new SQLTransientConnectionException(poolName + " - Connection is not available, request timed out after " + elapsedMillis(startTime) + "ms.", sqlState, originalException);
696       if (originalException instanceof SQLException) {
697          connectionException.setNextException((SQLException) originalException);
698       }
699
700       return connectionException;
701    }
702
703
704    // ***********************************************************************
705    //                      Non-anonymous Inner-classes
706    // ***********************************************************************
707
708    /**
709     * Creating and adding poolEntries (connections) to the pool.
710     */
711    private final class PoolEntryCreator implements Callable<Boolean>
712    {
713       private final String loggingPrefix;
714
715       PoolEntryCreator(String loggingPrefix)
716       {
717          this.loggingPrefix = loggingPrefix;
718       }
719
720       @Override
721       public Boolean call()
722       {
723          long sleepBackoff = 250L;
724          while (poolState == POOL_NORMAL && shouldCreateAnotherConnection()) {
725             final PoolEntry poolEntry = createPoolEntry();
726             if (poolEntry != null) {
727                connectionBag.add(poolEntry);
728                logger.debug("{} - Added connection {}", poolName, poolEntry.connection);
729                if (loggingPrefix != null) {
730                   logPoolState(loggingPrefix);
731                }
732                return Boolean.TRUE;
733             }
734
735             // failed to get connection from db, sleep and retry
736             if (loggingPrefix != null) logger.debug("{} - Connection add failed, sleeping with backoff: {}ms", poolName, sleepBackoff);
737             quietlySleep(sleepBackoff);
738             sleepBackoff = Math.min(SECONDS.toMillis(10), Math.min(connectionTimeout, (long) (sleepBackoff * 1.5)));
739          }
740
741          // Pool is suspended or shutdown or at max size
742          return Boolean.FALSE;
743       }
744
745       /**
746        * We only create connections if we need another idle connection or have threads still waiting
747        * for a new connection.  Otherwise we bail out of the request to create.
748        *
749        * @return true if we should create a connection, false if the need has disappeared
750        */
751       private synchronized boolean shouldCreateAnotherConnection() {
752          return getTotalConnections() < config.getMaximumPoolSize() &&
753             (connectionBag.getWaitingThreadCount() > 0 || getIdleConnections() < config.getMinimumIdle());
754       }
755    }
756
757    /**
758     * The house keeping task to retire and maintain minimum idle connections.
759     */
760    private final class HouseKeeper implements Runnable
761    {
762       private volatile long previous = plusMillis(currentTime(), -housekeepingPeriodMs);
763
764       @Override
765       public void run()
766       {
767          try {
768             // refresh values in case they changed via MBean
769             connectionTimeout = config.getConnectionTimeout();
770             validationTimeout = config.getValidationTimeout();
771             leakTaskFactory.updateLeakDetectionThreshold(config.getLeakDetectionThreshold());
772             catalog = (config.getCatalog() != null && !config.getCatalog().equals(catalog)) ? config.getCatalog() : catalog;
773
774             final long idleTimeout = config.getIdleTimeout();
775             final long now = currentTime();
776
777             // Detect retrograde time, allowing +128ms as per NTP spec.
778             if (plusMillis(now, 128) < plusMillis(previous, housekeepingPeriodMs)) {
779                logger.warn("{} - Retrograde clock change detected (housekeeper delta={}), soft-evicting connections from pool.",
780                            poolName, elapsedDisplayString(previous, now));
781                previous = now;
782                softEvictConnections();
783                return;
784             }
785             else if (now > plusMillis(previous, (3 * housekeepingPeriodMs) / 2)) {
786                // No point evicting for forward clock motion, this merely accelerates connection retirement anyway
787                logger.warn("{} - Thread starvation or clock leap detected (housekeeper delta={}).", poolName, elapsedDisplayString(previous, now));
788             }
789
790             previous = now;
791
792             String afterPrefix = "Pool ";
793             if (idleTimeout > 0L && config.getMinimumIdle() < config.getMaximumPoolSize()) {
794                logPoolState("Before cleanup ");
795                afterPrefix = "After cleanup  ";
796
797                final List<PoolEntry> notInUse = connectionBag.values(STATE_NOT_IN_USE);
798                int toRemove = notInUse.size() - config.getMinimumIdle();
799                for (PoolEntry entry : notInUse) {
800                   if (toRemove > 0 && elapsedMillis(entry.lastAccessed, now) > idleTimeout && connectionBag.reserve(entry)) {
801                      closeConnection(entry, "(connection has passed idleTimeout)");
802                      toRemove--;
803                   }
804                }
805             }
806
807             logPoolState(afterPrefix);
808
809             fillPool(); // Try to maintain minimum connections
810          }
811          catch (Exception e) {
812             logger.error("Unexpected exception in housekeeping task", e);
813          }
814       }
815    }
816
817    public static class PoolInitializationException extends RuntimeException
818    {
819       private static final long serialVersionUID = 929872118275916520L;
820
821       /**
822        * Construct an exception, possibly wrapping the provided Throwable as the cause.
823        * @param t the Throwable to wrap
824        */
825       public PoolInitializationException(Throwable t)
826       {
827          super("Failed to initialize pool: " + t.getMessage(), t);
828       }
829    }
830 }
831