minor #28269 [Cache] improve perf when using RedisCluster by reducing roundtrips to the servers (nicolas-grekas)
This PR was merged into the 4.2-dev branch.
Discussion
----------
[Cache] improve perf when using RedisCluster by reducing roundtrips to the servers
| Q | A
| ------------- | ---
| Branch? | master
| Bug fix? | no
| New feature? | yes
| BC breaks? | no
| Deprecations? | no
| Tests pass? | yes
| Fixed tickets | -
| License | MIT
| Doc PR | -
Improves perf when using RedisCluster by:
- disabling versioning and replacing by per-master clear/flush
- grouping multiple "get" in one "mget"
- enabling pipelining in PredisCluster mode
~Might need adjustment depending on the answer to https://github.com/nrk/predis/issues/520~
Commits
-------
5155f48029
[Cache] improve perf when using RedisCluster by reducing roundtrips to the servers
This commit is contained in:
commit
c3ec061fd0
@ -0,0 +1,28 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This file is part of the Symfony package.
|
||||||
|
*
|
||||||
|
* (c) Fabien Potencier <fabien@symfony.com>
|
||||||
|
*
|
||||||
|
* For the full copyright and license information, please view the LICENSE
|
||||||
|
* file that was distributed with this source code.
|
||||||
|
*/
|
||||||
|
|
||||||
|
namespace Symfony\Component\Cache\Tests\Adapter;
|
||||||
|
|
||||||
|
class PredisRedisClusterAdapterTest extends AbstractRedisAdapterTest
|
||||||
|
{
|
||||||
|
public static function setupBeforeClass()
|
||||||
|
{
|
||||||
|
if (!$hosts = getenv('REDIS_CLUSTER_HOSTS')) {
|
||||||
|
self::markTestSkipped('REDIS_CLUSTER_HOSTS env var is not defined.');
|
||||||
|
}
|
||||||
|
self::$redis = new \Predis\Client(explode(' ', $hosts), array('cluster' => 'redis'));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function tearDownAfterClass()
|
||||||
|
{
|
||||||
|
self::$redis = null;
|
||||||
|
}
|
||||||
|
}
|
@ -12,7 +12,6 @@
|
|||||||
namespace Symfony\Component\Cache\Traits;
|
namespace Symfony\Component\Cache\Traits;
|
||||||
|
|
||||||
use Predis\Connection\Aggregate\ClusterInterface;
|
use Predis\Connection\Aggregate\ClusterInterface;
|
||||||
use Predis\Connection\Aggregate\PredisCluster;
|
|
||||||
use Predis\Connection\Aggregate\RedisCluster;
|
use Predis\Connection\Aggregate\RedisCluster;
|
||||||
use Predis\Connection\Factory;
|
use Predis\Connection\Factory;
|
||||||
use Predis\Response\Status;
|
use Predis\Response\Status;
|
||||||
@ -53,9 +52,7 @@ trait RedisTrait
|
|||||||
if (preg_match('#[^-+_.A-Za-z0-9]#', $namespace, $match)) {
|
if (preg_match('#[^-+_.A-Za-z0-9]#', $namespace, $match)) {
|
||||||
throw new InvalidArgumentException(sprintf('RedisAdapter namespace contains "%s" but only characters in [-+_.A-Za-z0-9] are allowed.', $match[0]));
|
throw new InvalidArgumentException(sprintf('RedisAdapter namespace contains "%s" but only characters in [-+_.A-Za-z0-9] are allowed.', $match[0]));
|
||||||
}
|
}
|
||||||
if ($redisClient instanceof \RedisCluster) {
|
if (!$redisClient instanceof \Redis && !$redisClient instanceof \RedisArray && !$redisClient instanceof \RedisCluster && !$redisClient instanceof \Predis\Client && !$redisClient instanceof RedisProxy) {
|
||||||
$this->enableVersioning();
|
|
||||||
} elseif (!$redisClient instanceof \Redis && !$redisClient instanceof \RedisArray && !$redisClient instanceof \Predis\Client && !$redisClient instanceof RedisProxy) {
|
|
||||||
throw new InvalidArgumentException(sprintf('%s() expects parameter 1 to be Redis, RedisArray, RedisCluster or Predis\Client, %s given', __METHOD__, \is_object($redisClient) ? \get_class($redisClient) : \gettype($redisClient)));
|
throw new InvalidArgumentException(sprintf('%s() expects parameter 1 to be Redis, RedisArray, RedisCluster or Predis\Client, %s given', __METHOD__, \is_object($redisClient) ? \get_class($redisClient) : \gettype($redisClient)));
|
||||||
}
|
}
|
||||||
$this->redis = $redisClient;
|
$this->redis = $redisClient;
|
||||||
@ -182,18 +179,30 @@ trait RedisTrait
|
|||||||
*/
|
*/
|
||||||
protected function doFetch(array $ids)
|
protected function doFetch(array $ids)
|
||||||
{
|
{
|
||||||
if ($ids) {
|
if (!$ids) {
|
||||||
|
return array();
|
||||||
|
}
|
||||||
|
|
||||||
|
$i = -1;
|
||||||
|
$result = array();
|
||||||
|
|
||||||
|
if ($this->redis instanceof \Predis\Client) {
|
||||||
$values = $this->pipeline(function () use ($ids) {
|
$values = $this->pipeline(function () use ($ids) {
|
||||||
foreach ($ids as $id) {
|
foreach ($ids as $id) {
|
||||||
yield 'get' => array($id);
|
yield 'get' => array($id);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
foreach ($values as $id => $v) {
|
} else {
|
||||||
if ($v) {
|
$values = array_combine($ids, $this->redis->mget($ids));
|
||||||
yield $id => $this->marshaller->unmarshall($v);
|
}
|
||||||
}
|
|
||||||
|
foreach ($values as $id => $v) {
|
||||||
|
if ($v) {
|
||||||
|
$result[$id] = $this->marshaller->unmarshall($v);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return $result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -209,9 +218,6 @@ trait RedisTrait
|
|||||||
*/
|
*/
|
||||||
protected function doClear($namespace)
|
protected function doClear($namespace)
|
||||||
{
|
{
|
||||||
// When using a native Redis cluster, clearing the cache is done by versioning in AbstractTrait::clear().
|
|
||||||
// This means old keys are not really removed until they expire and may need garbage collection.
|
|
||||||
|
|
||||||
$cleared = true;
|
$cleared = true;
|
||||||
$hosts = array($this->redis);
|
$hosts = array($this->redis);
|
||||||
$evalArgs = array(array($namespace), 0);
|
$evalArgs = array(array($namespace), 0);
|
||||||
@ -220,13 +226,11 @@ trait RedisTrait
|
|||||||
$evalArgs = array(0, $namespace);
|
$evalArgs = array(0, $namespace);
|
||||||
|
|
||||||
$connection = $this->redis->getConnection();
|
$connection = $this->redis->getConnection();
|
||||||
if ($connection instanceof PredisCluster) {
|
if ($connection instanceof ClusterInterface && $connection instanceof \Traversable) {
|
||||||
$hosts = array();
|
$hosts = array();
|
||||||
foreach ($connection as $c) {
|
foreach ($connection as $c) {
|
||||||
$hosts[] = new \Predis\Client($c);
|
$hosts[] = new \Predis\Client($c);
|
||||||
}
|
}
|
||||||
} elseif ($connection instanceof RedisCluster) {
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
} elseif ($this->redis instanceof \RedisArray) {
|
} elseif ($this->redis instanceof \RedisArray) {
|
||||||
$hosts = array();
|
$hosts = array();
|
||||||
@ -234,7 +238,11 @@ trait RedisTrait
|
|||||||
$hosts[] = $this->redis->_instance($host);
|
$hosts[] = $this->redis->_instance($host);
|
||||||
}
|
}
|
||||||
} elseif ($this->redis instanceof \RedisCluster) {
|
} elseif ($this->redis instanceof \RedisCluster) {
|
||||||
return false;
|
$hosts = array();
|
||||||
|
foreach ($this->redis->_masters() as $host) {
|
||||||
|
$hosts[] = $h = new \Redis();
|
||||||
|
$h->connect($host[0], $host[1]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
foreach ($hosts as $host) {
|
foreach ($hosts as $host) {
|
||||||
if (!isset($namespace[0])) {
|
if (!isset($namespace[0])) {
|
||||||
@ -261,7 +269,7 @@ trait RedisTrait
|
|||||||
$keys = $keys[1];
|
$keys = $keys[1];
|
||||||
}
|
}
|
||||||
if ($keys) {
|
if ($keys) {
|
||||||
$host->del($keys);
|
$this->doDelete($keys);
|
||||||
}
|
}
|
||||||
} while ($cursor = (int) $cursor);
|
} while ($cursor = (int) $cursor);
|
||||||
}
|
}
|
||||||
@ -274,7 +282,17 @@ trait RedisTrait
|
|||||||
*/
|
*/
|
||||||
protected function doDelete(array $ids)
|
protected function doDelete(array $ids)
|
||||||
{
|
{
|
||||||
if ($ids) {
|
if (!$ids) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($this->redis instanceof \Predis\Client) {
|
||||||
|
$this->pipeline(function () use ($ids) {
|
||||||
|
foreach ($ids as $id) {
|
||||||
|
yield 'del' => array($id);
|
||||||
|
}
|
||||||
|
})->rewind();
|
||||||
|
} else {
|
||||||
$this->redis->del($ids);
|
$this->redis->del($ids);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -312,7 +330,16 @@ trait RedisTrait
|
|||||||
{
|
{
|
||||||
$ids = array();
|
$ids = array();
|
||||||
|
|
||||||
if ($this->redis instanceof \Predis\Client && !$this->redis->getConnection() instanceof ClusterInterface) {
|
if ($this->redis instanceof \RedisCluster || ($this->redis instanceof \Predis\Client && $this->redis->getConnection() instanceof RedisCluster)) {
|
||||||
|
// phpredis & predis don't support pipelining with RedisCluster
|
||||||
|
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
|
||||||
|
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
|
||||||
|
$results = array();
|
||||||
|
foreach ($generator() as $command => $args) {
|
||||||
|
$results[] = \call_user_func_array(array($this->redis, $command), $args);
|
||||||
|
$ids[] = $args[0];
|
||||||
|
}
|
||||||
|
} elseif ($this->redis instanceof \Predis\Client) {
|
||||||
$results = $this->redis->pipeline(function ($redis) use ($generator, &$ids) {
|
$results = $this->redis->pipeline(function ($redis) use ($generator, &$ids) {
|
||||||
foreach ($generator() as $command => $args) {
|
foreach ($generator() as $command => $args) {
|
||||||
\call_user_func_array(array($redis, $command), $args);
|
\call_user_func_array(array($redis, $command), $args);
|
||||||
@ -336,15 +363,6 @@ trait RedisTrait
|
|||||||
foreach ($results as $k => list($h, $c)) {
|
foreach ($results as $k => list($h, $c)) {
|
||||||
$results[$k] = $connections[$h][$c];
|
$results[$k] = $connections[$h][$c];
|
||||||
}
|
}
|
||||||
} elseif ($this->redis instanceof \RedisCluster || ($this->redis instanceof \Predis\Client && $this->redis->getConnection() instanceof ClusterInterface)) {
|
|
||||||
// phpredis & predis don't support pipelining with RedisCluster
|
|
||||||
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
|
|
||||||
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
|
|
||||||
$results = array();
|
|
||||||
foreach ($generator() as $command => $args) {
|
|
||||||
$results[] = \call_user_func_array(array($this->redis, $command), $args);
|
|
||||||
$ids[] = $args[0];
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
$this->redis->multi(\Redis::PIPELINE);
|
$this->redis->multi(\Redis::PIPELINE);
|
||||||
foreach ($generator() as $command => $args) {
|
foreach ($generator() as $command => $args) {
|
||||||
|
Reference in New Issue
Block a user