[Cache] add TagAwareMarshaller to optimize data storage when using AbstractTagAwareAdapter
This commit is contained in:
parent
6e7f3257ce
commit
5a4a30c6ef
@ -149,6 +149,16 @@ abstract class AbstractTagAwareAdapter implements TagAwareAdapterInterface, TagA
|
||||
*/
|
||||
abstract protected function doInvalidate(array $tagIds): bool;
|
||||
|
||||
/**
|
||||
* Returns the tags bound to the provided ids.
|
||||
*/
|
||||
protected function doFetchTags(array $ids): iterable
|
||||
{
|
||||
foreach ($this->doFetch($ids) as $id => $value) {
|
||||
yield $id => \is_array($value) && \is_array($value['tags'] ?? null) ? $value['tags'] : [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*
|
||||
@ -233,8 +243,8 @@ abstract class AbstractTagAwareAdapter implements TagAwareAdapterInterface, TagA
|
||||
}
|
||||
|
||||
try {
|
||||
foreach ($this->doFetch($ids) as $id => $value) {
|
||||
foreach ($value['tags'] ?? [] as $tag) {
|
||||
foreach ($this->doFetchTags($ids) as $id => $tags) {
|
||||
foreach ($tags as $tag) {
|
||||
$tagData[$this->getId(self::TAGS_PREFIX.$tag)][] = $id;
|
||||
}
|
||||
}
|
||||
|
@ -11,8 +11,8 @@
|
||||
|
||||
namespace Symfony\Component\Cache\Adapter;
|
||||
|
||||
use Symfony\Component\Cache\Marshaller\DefaultMarshaller;
|
||||
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
|
||||
use Symfony\Component\Cache\Marshaller\TagAwareMarshaller;
|
||||
use Symfony\Component\Cache\PruneableInterface;
|
||||
use Symfony\Component\Cache\Traits\FilesystemTrait;
|
||||
|
||||
@ -37,7 +37,7 @@ class FilesystemTagAwareAdapter extends AbstractTagAwareAdapter implements Prune
|
||||
|
||||
public function __construct(string $namespace = '', int $defaultLifetime = 0, string $directory = null, MarshallerInterface $marshaller = null)
|
||||
{
|
||||
$this->marshaller = $marshaller ?? new DefaultMarshaller();
|
||||
$this->marshaller = new TagAwareMarshaller($marshaller);
|
||||
parent::__construct('', $defaultLifetime);
|
||||
$this->init($namespace, $directory);
|
||||
}
|
||||
@ -130,6 +130,40 @@ class FilesystemTagAwareAdapter extends AbstractTagAwareAdapter implements Prune
|
||||
return $failed;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function doFetchTags(array $ids): iterable
|
||||
{
|
||||
foreach ($ids as $id) {
|
||||
$file = $this->getFile($id);
|
||||
if (!file_exists($file) || !$h = @fopen($file, 'rb')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$meta = explode("\n", fread($h, 4096), 3)[2] ?? '';
|
||||
|
||||
// detect the compact format used in marshall() using magic numbers in the form 9D-..-..-..-..-00-..-..-..-5F
|
||||
if (13 < \strlen($meta) && "\x9D" === $meta[0] && "\0" === $meta[5] && "\x5F" === $meta[9]) {
|
||||
$meta[9] = "\0";
|
||||
$tagLen = unpack('Nlen', $meta, 9)['len'];
|
||||
$meta = substr($meta, 13, $tagLen);
|
||||
|
||||
if (0 < $tagLen -= \strlen($meta)) {
|
||||
$meta .= fread($h, $tagLen);
|
||||
}
|
||||
|
||||
try {
|
||||
yield $id => '' === $meta ? [] : $this->marshaller->unmarshall($meta);
|
||||
} catch (\Exception $e) {
|
||||
yield $id => [];
|
||||
}
|
||||
}
|
||||
|
||||
fclose($h);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
|
@ -16,6 +16,7 @@ use Predis\Connection\Aggregate\PredisCluster;
|
||||
use Predis\Response\Status;
|
||||
use Symfony\Component\Cache\Exception\InvalidArgumentException;
|
||||
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
|
||||
use Symfony\Component\Cache\Marshaller\TagAwareMarshaller;
|
||||
use Symfony\Component\Cache\Traits\RedisTrait;
|
||||
|
||||
/**
|
||||
@ -67,7 +68,7 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
|
||||
throw new InvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class, \get_class($redisClient->getConnection())));
|
||||
}
|
||||
|
||||
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
|
||||
$this->init($redisClient, $namespace, $defaultLifetime, new TagAwareMarshaller($marshaller));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -119,6 +120,42 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
|
||||
return $failed;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
protected function doFetchTags(array $ids): iterable
|
||||
{
|
||||
$lua = <<<'EOLUA'
|
||||
local v = redis.call('GET', KEYS[1])
|
||||
|
||||
if not v or v:len() <= 13 or v:byte(1) ~= 0x9D or v:byte(6) ~= 0 or v:byte(10) ~= 0x5F then
|
||||
return ''
|
||||
end
|
||||
|
||||
return v:sub(14, 13 + v:byte(13) + v:byte(12) * 256 + v:byte(11) * 65536)
|
||||
EOLUA;
|
||||
|
||||
if ($this->redis instanceof \Predis\ClientInterface) {
|
||||
$evalArgs = [$lua, 1, &$id];
|
||||
} else {
|
||||
$evalArgs = [$lua, [&$id], 1];
|
||||
}
|
||||
|
||||
$results = $this->pipeline(function () use ($ids, &$id, $evalArgs) {
|
||||
foreach ($ids as $id) {
|
||||
yield 'eval' => $evalArgs;
|
||||
}
|
||||
});
|
||||
|
||||
foreach ($results as $id => $result) {
|
||||
try {
|
||||
yield $id => !\is_string($result) || '' === $result ? [] : $this->marshaller->unmarshall($result);
|
||||
} catch (\Exception $e) {
|
||||
yield $id => [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
|
@ -7,6 +7,7 @@ CHANGELOG
|
||||
* added support for connecting to Redis Sentinel clusters
|
||||
* added argument `$prefix` to `AdapterInterface::clear()`
|
||||
* improved `RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
|
||||
* added `TagAwareMarshaller` for optimized data storage when using `AbstractTagAwareAdapter`
|
||||
* [BC BREAK] `RedisTagAwareAdapter` is not compatible with `RedisCluster` from `Predis` anymore, use `phpredis` instead
|
||||
|
||||
4.3.0
|
||||
|
@ -0,0 +1,89 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Symfony\Component\Cache\Marshaller;
|
||||
|
||||
/**
|
||||
* A marshaller optimized for data structures generated by AbstractTagAwareAdapter.
|
||||
*
|
||||
* @author Nicolas Grekas <p@tchwork.com>
|
||||
*/
|
||||
class TagAwareMarshaller implements MarshallerInterface
|
||||
{
|
||||
private $marshaller;
|
||||
|
||||
public function __construct(MarshallerInterface $marshaller = null)
|
||||
{
|
||||
$this->marshaller = $marshaller ?? new DefaultMarshaller();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function marshall(array $values, ?array &$failed): array
|
||||
{
|
||||
$failed = $notSerialized = $serialized = [];
|
||||
|
||||
foreach ($values as $id => $value) {
|
||||
if (\is_array($value) && \is_array($value['tags'] ?? null) && \array_key_exists('value', $value) && \count($value) === 2 + (\is_string($value['meta'] ?? null) && 8 === \strlen($value['meta']))) {
|
||||
// if the value is an array with keys "tags", "value" and "meta", use a compact serialization format
|
||||
// magic numbers in the form 9D-..-..-..-..-00-..-..-..-5F allow detecting this format quickly in unmarshall()
|
||||
|
||||
$v = $this->marshaller->marshall($value, $f);
|
||||
|
||||
if ($f) {
|
||||
$f = [];
|
||||
$failed[] = $id;
|
||||
} else {
|
||||
if ([] === $value['tags']) {
|
||||
$v['tags'] = '';
|
||||
}
|
||||
|
||||
$serialized[$id] = "\x9D".($value['meta'] ?? "\0\0\0\0\0\0\0\0").pack('N', \strlen($v['tags'])).$v['tags'].$v['value'];
|
||||
$serialized[$id][9] = "\x5F";
|
||||
}
|
||||
} else {
|
||||
// other arbitratry values are serialized using the decorated marshaller below
|
||||
$notSerialized[$id] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
if ($notSerialized) {
|
||||
$serialized += $this->marshaller->marshall($notSerialized, $f);
|
||||
$failed = array_merge($failed, $f);
|
||||
}
|
||||
|
||||
return $serialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritdoc}
|
||||
*/
|
||||
public function unmarshall(string $value)
|
||||
{
|
||||
// detect the compact format used in marshall() using magic numbers in the form 9D-..-..-..-..-00-..-..-..-5F
|
||||
if (13 >= \strlen($value) || "\x9D" !== $value[0] || "\0" !== $value[5] || "\x5F" !== $value[9]) {
|
||||
return $this->marshaller->unmarshall($value);
|
||||
}
|
||||
|
||||
// data consists of value, tags and metadata which we need to unpack
|
||||
$meta = substr($value, 1, 12);
|
||||
$meta[8] = "\0";
|
||||
$tagLen = unpack('Nlen', $meta, 8)['len'];
|
||||
$meta = substr($meta, 0, 8);
|
||||
|
||||
return [
|
||||
'value' => $this->marshaller->unmarshall(substr($value, 13 + $tagLen)),
|
||||
'tags' => $tagLen ? $this->marshaller->unmarshall(substr($value, 13, $tagLen)) : [],
|
||||
'meta' => "\0\0\0\0\0\0\0\0" === $meta ? null : $meta,
|
||||
];
|
||||
}
|
||||
}
|
@ -51,11 +51,13 @@ trait AbstractAdapterTrait
|
||||
foreach ($this->doFetch([$id]) as $value) {
|
||||
$isHit = true;
|
||||
}
|
||||
|
||||
return $f($key, $value, $isHit);
|
||||
} catch (\Exception $e) {
|
||||
CacheItem::log($this->logger, 'Failed to fetch key "{key}": '.$e->getMessage(), ['key' => $key, 'exception' => $e]);
|
||||
}
|
||||
|
||||
return $f($key, $value, $isHit);
|
||||
return $f($key, null, false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -445,25 +445,26 @@ trait RedisTrait
|
||||
$results = [];
|
||||
foreach ($generator() as $command => $args) {
|
||||
$results[] = $redis->{$command}(...$args);
|
||||
$ids[] = $args[0];
|
||||
$ids[] = 'eval' === $command ? ($redis instanceof \Predis\ClientInterface ? $args[2] : $args[1][0]) : $args[0];
|
||||
}
|
||||
} elseif ($redis instanceof \Predis\ClientInterface) {
|
||||
$results = $redis->pipeline(static function ($redis) use ($generator, &$ids) {
|
||||
foreach ($generator() as $command => $args) {
|
||||
$redis->{$command}(...$args);
|
||||
$ids[] = $args[0];
|
||||
$ids[] = 'eval' === $command ? $args[2] : $args[0];
|
||||
}
|
||||
});
|
||||
} elseif ($redis instanceof \RedisArray) {
|
||||
$connections = $results = $ids = [];
|
||||
foreach ($generator() as $command => $args) {
|
||||
if (!isset($connections[$h = $redis->_target($args[0])])) {
|
||||
$id = 'eval' === $command ? $args[1][0] : $args[0];
|
||||
if (!isset($connections[$h = $redis->_target($id)])) {
|
||||
$connections[$h] = [$redis->_instance($h), -1];
|
||||
$connections[$h][0]->multi(\Redis::PIPELINE);
|
||||
}
|
||||
$connections[$h][0]->{$command}(...$args);
|
||||
$results[] = [$h, ++$connections[$h][1]];
|
||||
$ids[] = $args[0];
|
||||
$ids[] = $id;
|
||||
}
|
||||
foreach ($connections as $h => $c) {
|
||||
$connections[$h] = $c[0]->exec();
|
||||
@ -475,7 +476,7 @@ trait RedisTrait
|
||||
$redis->multi(\Redis::PIPELINE);
|
||||
foreach ($generator() as $command => $args) {
|
||||
$redis->{$command}(...$args);
|
||||
$ids[] = $args[0];
|
||||
$ids[] = 'eval' === $command ? $args[1][0] : $args[0];
|
||||
}
|
||||
$results = $redis->exec();
|
||||
}
|
||||
|
Reference in New Issue
Block a user