Automatically reconnecting to the database when safe to do so

This has been anf often requested feature for the database layer,
specially by people using the database in long running background tasks.

Instead of instructing them to figure out how to configure their database
to not drop connections, better just give them an extra hand by reconnecting
to the database whenever it is safe to do so.

That is, reconnection are attempted only on fresh commands to the database
outside of a transaction.

src/Core/Retry/CommandRetry.php

@@ -0,0 +1,80 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Core\Retry;
+
+use Exception;
+
+/**
+ * Allows any action to be retried in case of an exception.
+ *
+ * This class can be parametrized with an strategy, which will be followed
+ * to determine whether or not the action should be retried.
+ */
+class CommandRetry
+{
+
+    /**
+     * The strategy to follow should the executed action fail
+     *
+     * @var \Cake\Core\Retry\RetryStrategyInterface
+     */
+    protected $strategy;
+
+    /**
+     * The number of retries to perform in case of failure
+     *
+     * @var int
+     */
+    protected $retries;
+
+    /**
+     * Creates the CommandRetry object with the given strategy and retry count
+     *
+     * @param \Cake\Core\Retry\RetryStrategyInterface $strategy The strategy to follow should the action fail
+     * @param int $retries The number of times the action has been already called
+     */
+    public function __construct(RetryStrategyInterface $strategy, $retries = 1)
+    {
+        $this->strategy = $strategy;
+        $this->retries = $retries;
+    }
+
+    /**
+     * The number of retries to perform in case of failure
+     *
+     * @param callable $action The callable action to execute with a retry strategy
+     * @return mixed The return value of the passed action callable
+     */
+    public function run(callable $action)
+    {
+        $retryCount = 0;
+        $lastException = null;
+        while ($this->retries > $retryCount) {
+            $retryCount++;
+            try {
+                return $action();
+            } catch (Exception $e) {
+                $lastException = $e;
+                if (!$this->strategy->shouldRetry($e, $retryCount)) {
+                    throw $e;
+                }
+            }
+        }
+
+        if ($lastException !== null) {
+            throw $lastException;
+        }
+    }
+}

src/Core/Retry/RetryStrategyInterface.php

@@ -0,0 +1,33 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Core\Retry;
+
+use Exception;
+
+/**
+ * Makes sure the connection to the database is alive before authorizing
+ * the retry of an action.
+ */
+interface RetryStrategyInterface
+{
+    /**
+     * Returns true if the action can be retried, false otherwise.
+     *
+     * @param Exception $exception The exception that caused the action to fail
+     * @param int $retryCount The number of times the action has been already called
+     * @return bool Whether or not it is OK to retry the action
+     */
+    public function shouldRetry(Exception $exception, $retryCount);
+}

src/Database/Connection.php

@@ -15,6 +15,7 @@
 namespace Cake\Database;
 
 use Cake\Core\App;
+use Cake\Core\Retry\CommandRetry;
 use Cake\Database\Exception\MissingConnectionException;
 use Cake\Database\Exception\MissingDriverException;
 use Cake\Database\Exception\MissingExtensionException;
@@ -22,6 +23,7 @@ use Cake\Database\Exception\NestedTransactionRollbackException;
 use Cake\Database\Log\LoggedQuery;
 use Cake\Database\Log\LoggingStatement;
 use Cake\Database\Log\QueryLogger;
+use Cake\Database\Retry\ReconnectStrategy;
 use Cake\Database\Schema\CachedCollection;
 use Cake\Database\Schema\Collection as SchemaCollection;
 use Cake\Datasource\ConnectionInterface;
@@ -183,6 +185,17 @@ class Connection implements ConnectionInterface
     }
 
     /**
+     * Get the retry wrapper object, that is used to recover from server disconnects
+     * while performing ceratain database actions, such as executing a query
+     *
+     * @return \Cake\Core\Retry\CommandRetry The retry wrapper
+     */
+    public function getDisconnectRetry()
+    {
+        return new CommandRetry(new ReconnectStrategy($this));
+    }
+
+    /**
      * Gets the driver instance.
      *
      * @return \Cake\Database\Driver
@@ -258,13 +271,15 @@ class Connection implements ConnectionInterface
      */
     public function prepare($sql)
     {
-        $statement = $this->_driver->prepare($sql);
+        return $this->getDisconnectRetry()->run(function () use ($sql) {
+            $statement = $this->_driver->prepare($sql);
 
-        if ($this->_logQueries) {
-            $statement = $this->_newLogger($statement);
-        }
+            if ($this->_logQueries) {
+                $statement = $this->_newLogger($statement);
+            }
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -278,15 +293,17 @@ class Connection implements ConnectionInterface
      */
     public function execute($query, array $params = [], array $types = [])
     {
-        if (!empty($params)) {
-            $statement = $this->prepare($query);
-            $statement->bind($params, $types);
-            $statement->execute();
-        } else {
-            $statement = $this->query($query);
-        }
+        return $this->getDisconnectRetry()->run(function () use ($query, $params, $types) {
+            if (!empty($params)) {
+                $statement = $this->prepare($query);
+                $statement->bind($params, $types);
+                $statement->execute();
+            } else {
+                $statement = $this->query($query);
+            }
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -311,11 +328,13 @@ class Connection implements ConnectionInterface
      */
     public function run(Query $query)
     {
-        $statement = $this->prepare($query);
-        $query->getValueBinder()->attachTo($statement);
-        $statement->execute();
+        return $this->getDisconnectRetry()->run(function () use ($query) {
+            $statement = $this->prepare($query);
+            $query->getValueBinder()->attachTo($statement);
+            $statement->execute();
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -326,10 +345,12 @@ class Connection implements ConnectionInterface
      */
     public function query($sql)
     {
-        $statement = $this->prepare($sql);
-        $statement->execute();
+        return $this->getDisconnectRetry()->run(function () use ($sql) {
+            $statement = $this->prepare($sql);
+            $statement->execute();
 
-        return $statement;
+            return $statement;
+        });
     }
 
     /**
@@ -403,12 +424,14 @@ class Connection implements ConnectionInterface
      */
     public function insert($table, array $data, array $types = [])
     {
-        $columns = array_keys($data);
+        return $this->getDisconnectRetry()->run(function () use ($table, $data, $types) {
+            $columns = array_keys($data);
 
-        return $this->newQuery()->insert($columns, $types)
-            ->into($table)
-            ->values($data)
-            ->execute();
+            return $this->newQuery()->insert($columns, $types)
+                ->into($table)
+                ->values($data)
+                ->execute();
+        });
     }
 
     /**
@@ -422,10 +445,12 @@ class Connection implements ConnectionInterface
      */
     public function update($table, array $data, array $conditions = [], $types = [])
     {
-        return $this->newQuery()->update($table)
-            ->set($data, $types)
-            ->where($conditions, $types)
-            ->execute();
+        return $this->getDisconnectRetry()->run(function () use ($table, $data, $conditions, $types) {
+            return $this->newQuery()->update($table)
+                ->set($data, $types)
+                ->where($conditions, $types)
+                ->execute();
+        });
     }
 
     /**
@@ -438,9 +463,11 @@ class Connection implements ConnectionInterface
      */
     public function delete($table, $conditions = [], $types = [])
     {
-        return $this->newQuery()->delete($table)
-            ->where($conditions, $types)
-            ->execute();
+        return $this->getDisconnectRetry()->run(function () use ($table, $conditions, $types) {
+            return $this->newQuery()->delete($table)
+                ->where($conditions, $types)
+                ->execute();
+        });
     }
 
     /**
@@ -454,7 +481,11 @@ class Connection implements ConnectionInterface
             if ($this->_logQueries) {
                 $this->log('BEGIN');
             }
-            $this->_driver->beginTransaction();
+
+            $this->getDisconnectRetry()->run(function () {
+                $this->_driver->beginTransaction();
+            });
+
             $this->_transactionLevel = 0;
             $this->_transactionStarted = true;
             $this->nestedTransactionRollbackException = null;
@@ -648,7 +679,9 @@ class Connection implements ConnectionInterface
      */
     public function disableForeignKeys()
     {
-        $this->execute($this->_driver->disableForeignKeySQL())->closeCursor();
+        $this->getDisconnectRetry()->run(function () {
+            $this->execute($this->_driver->disableForeignKeySQL())->closeCursor();
+        });
     }
 
     /**
@@ -658,7 +691,9 @@ class Connection implements ConnectionInterface
      */
     public function enableForeignKeys()
     {
-        $this->execute($this->_driver->enableForeignKeySQL())->closeCursor();
+        $this->getDisconnectRetry()->run(function () {
+            $this->execute($this->_driver->enableForeignKeySQL())->closeCursor();
+        });
     }
 
     /**
@@ -733,18 +768,20 @@ class Connection implements ConnectionInterface
      */
     public function disableConstraints(callable $callback)
     {
-        $this->disableForeignKeys();
+        return $this->getDisconnectRetry()->run(function () use ($callback) {
+            $this->disableForeignKeys();
 
-        try {
-            $result = $callback($this);
-        } catch (Exception $e) {
-            $this->enableForeignKeys();
-            throw $e;
-        }
+            try {
+                $result = $callback($this);
+            } catch (Exception $e) {
+                $this->enableForeignKeys();
+                throw $e;
+            }
 
-        $this->enableForeignKeys();
+            $this->enableForeignKeys();
 
-        return $result;
+            return $result;
+        });
     }
 
     /**

src/Database/Retry/ReconnectStrategy.php

@@ -0,0 +1,120 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * For full copyright and license information, please see the LICENSE.txt
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Database\Retry;
+
+use Cake\Core\Retry\RetryStrategyInterface;
+use Cake\Database\Connection;
+use Exception;
+
+/**
+ * Makes sure the connection to the database is alive before authorizing
+ * the retry of an action.
+ */
+class ReconnectStrategy implements RetryStrategyInterface
+{
+    /**
+     * The list of errors strings to match when looking for disconnection error.
+     * This is a static variable to avoid holding all the strings in the array can
+     * be inlined by the opcache.
+     *
+     * @var array
+     */
+    protected static $causes = [
+        'gone away',
+        'Lost connection',
+        'Transaction() on null',
+        'closed the connection unexpectedly',
+        'closed unexpectedly',
+        'deadlock avoided',
+        'decryption failed or bad record mac',
+        'is dead or not enabled',
+        'no connection to the server',
+        'query_wait_timeout',
+        'reset by peer',
+        'terminate due to client_idle_limit',
+        'while sending',
+        'writing data to the connection',
+    ];
+
+    /**
+     * The connection to check for validity
+     *
+     * @var Connection
+     */
+    protected $connection;
+
+    /**
+     * Creates the ReconnectStrategy object by storing a reference to the
+     * passed connection. This reference will be used to automatically
+     * reconnect to the server in case of failure.
+     *
+     * @param \Cake\Database\Connection $connection The connection to check
+     */
+    public function __construct(Connection $connection)
+    {
+        $this->connection = $connection;
+    }
+
+    /**
+     * Checks whether or not the exception was caused by a lost connection,
+     * and returns true if it was able to successfully reconnect.
+     *
+     * @param Exception $exception The exception to check for its message
+     * @param int $retryCount The number of times the action has been already called
+     * @return bool Whether or not it is OK to retry the action
+     */
+    public function shouldRetry(Exception $exception, $retryCount)
+    {
+        $message = $exception->getMessage();
+
+        foreach (static::$causes as $cause) {
+            if (strstr($message, $cause) !== false) {
+                return $this->reconnect();
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Tries to re-establish the connection to the server, if it is safe to do so
+     *
+     * @return bool Whether or not the connection was re-established
+     */
+    protected function reconnect()
+    {
+        if ($this->connection->inTransaction()) {
+            // It is not safe to blindly reconnect in the middle of a transaction
+            return false;
+        }
+
+        try {
+            // Make sure we free any resources associated with the old connection
+            $this->connection->disconnect();
+        } catch (Exception $e) {
+        }
+
+        try {
+            $this->connection->connect();
+            $this->connection->log('[RECONNECT]');
+
+            return true;
+        } catch (Exception $e) {
+            // If there was an error connecting again, don't report it back,
+            // let the retry handler do it.
+            return false;
+        }
+    }
+}

tests/TestCase/Core/Retry/CommandRetryTest.php

@@ -0,0 +1,110 @@
+<?php
+/**
+ * CakePHP(tm) : Rapid Development Framework (https://cakephp.org)
+ * Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ *
+ * Licensed under The MIT License
+ * Redistributions of files must retain the above copyright notice.
+ *
+ * @copyright     Copyright (c) Cake Software Foundation, Inc. (https://cakefoundation.org)
+ * @link          https://cakephp.org CakePHP(tm) Project
+ * @since         3.6.0
+ * @license       https://opensource.org/licenses/mit-license.php MIT License
+ */
+namespace Cake\Test\TestCase\Core\Retry;
+
+use Cake\Core\Retry\CommandRetry;
+use Cake\Core\Retry\RetryStrategyInterface;
+use Cake\TestSuite\TestCase;
+use Exception;
+
+/**
+ * Tests for the CommandRetry class
+ */
+class CommandRetryTest extends TestCase
+{
+
+    /**
+     * Simple retry test
+     *
+     * @return void
+     */
+    public function testRetry()
+    {
+        $count = 0;
+        $exception = new Exception('this is failing');
+        $action = function () use (&$count, $exception) {
+            $count++;
+
+            if ($count < 4) {
+                throw $exception;
+            }
+
+            return $count;
+        };
+
+        $strategy = $this->getMockBuilder(RetryStrategyInterface::class)->getMock();
+        $strategy
+            ->expects($this->exactly(3))
+            ->method('shouldRetry')
+            ->will($this->returnCallback(function ($e, $c) use ($exception, &$count) {
+                $this->assertSame($e, $exception);
+                $this->assertEquals($c, $count);
+
+                return true;
+            }));
+
+        $retry = new CommandRetry($strategy, 5);
+        $retry->run($action);
+    }
+
+    /**
+     * Test attempts exceeded
+     *
+     * @return void
+     */
+    public function testExceedAttempts()
+    {
+        $exception = new Exception('this is failing');
+        $action = function () use ($exception) {
+            throw $exception;
+        };
+
+        $strategy = $this->getMockBuilder(RetryStrategyInterface::class)->getMock();
+        $strategy
+            ->expects($this->exactly(3))
+            ->method('shouldRetry')
+            ->will($this->returnCallback(function ($e) use ($exception) {
+                return true;
+            }));
+
+        $retry = new CommandRetry($strategy, 3);
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('this is failing');
+        $retry->run($action);
+    }
+    /**
+     * Test that the strategy is respected
+     *
+     * @return void
+     */
+    public function testRespectStrategy()
+    {
+        $action = function () {
+            throw new Exception('this is failing');
+        };
+
+        $strategy = $this->getMockBuilder(RetryStrategyInterface::class)->getMock();
+        $strategy
+            ->expects($this->once())
+            ->method('shouldRetry')
+            ->will($this->returnCallback(function () {
+                return false;
+            }));
+
+        $retry = new CommandRetry($strategy, 3);
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('this is failing');
+        $retry->run($action);
+    }
+}

tests/TestCase/Database/ConnectionTest.php

@@ -20,10 +20,14 @@ use Cake\Database\Exception\MissingConnectionException;
 use Cake\Database\Exception\NestedTransactionRollbackException;
 use Cake\Database\Log\LoggingStatement;
 use Cake\Database\Log\QueryLogger;
+use Cake\Database\Retry\CommandRetry;
+use Cake\Database\Retry\ReconnectStrategy;
 use Cake\Datasource\ConnectionManager;
 use Cake\Log\Log;
 use Cake\TestSuite\TestCase;
+use Exception;
 use ReflectionMethod;
+use ReflectionProperty;
 
 /**
  * Tests Connection class
@@ -1215,4 +1219,65 @@ class ConnectionTest extends TestCase
         $method->setAccessible(true);
         $this->nestedTransactionStates[] = $method->invoke($this->connection);
     }
+
+    /**
+     * Tests that the connection is restablished whenever it is interrupted
+     * after having used the connection at least once.
+     *
+     * @return void
+     */
+    public function testAutomaticReconnect()
+    {
+        $conn = clone $this->connection;
+        $statement = $conn->query('SELECT 1');
+        $statement->execute();
+        $statement->closeCursor();
+
+        $prop = new ReflectionProperty($conn, '_driver');
+        $prop->setAccessible(true);
+        $oldDriver = $prop->getValue($conn);
+        $newDriver = $this->getMockBuilder('Cake\Database\Driver')->getMock();
+        $prop->setValue($conn, $newDriver);
+
+        $newDriver->expects($this->at(0))
+            ->method('prepare')
+            ->will($this->throwException(new Exception('server gone away')));
+
+        $newDriver->expects($this->at(1))->method('disconnect');
+        $newDriver->expects($this->at(2))->method('connect');
+        $newDriver->expects($this->at(3))
+            ->method('prepare')
+            ->will($this->returnValue($statement));
+
+        $this->assertSame($statement, $conn->query('SELECT 1'));
+    }
+
+    /**
+     * Tests that the connection is not restablished whenever it is interrupted
+     * inside a transaction.
+     *
+     * @return void
+     */
+    public function testNoAutomaticReconnect()
+    {
+        $conn = clone $this->connection;
+        $statement = $conn->query('SELECT 1');
+        $statement->execute();
+        $statement->closeCursor();
+
+        $conn->begin();
+
+        $prop = new ReflectionProperty($conn, '_driver');
+        $prop->setAccessible(true);
+        $oldDriver = $prop->getValue($conn);
+        $newDriver = $this->getMockBuilder('Cake\Database\Driver')->getMock();
+        $prop->setValue($conn, $newDriver);
+
+        $newDriver->expects($this->once())
+            ->method('prepare')
+            ->will($this->throwException(new Exception('server gone away')));
+
+        $this->expectException(Exception::class);
+        $conn->query('SELECT 1');
+    }
 }