From 1ae7dd5ec741be04c7acf3ff2f1fed0a840d4442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Jos=C3=A9=20Cerezo=20Aranda?= Date: Thu, 13 Jun 2019 12:07:42 +0200 Subject: [PATCH] [Cache] Add couchbase cache adapter --- .travis.yml | 17 +- phpunit.xml.dist | 3 + .../Cache/Adapter/AbstractAdapter.php | 3 + .../Cache/Adapter/CouchbaseBucketAdapter.php | 252 ++++++++++++++++++ src/Symfony/Component/Cache/CHANGELOG.md | 1 + src/Symfony/Component/Cache/LockRegistry.php | 1 + .../Adapter/CouchbaseBucketAdapterTest.php | 54 ++++ src/Symfony/Component/Cache/phpunit.xml.dist | 3 + 8 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 src/Symfony/Component/Cache/Adapter/CouchbaseBucketAdapter.php create mode 100644 src/Symfony/Component/Cache/Tests/Adapter/CouchbaseBucketAdapterTest.php diff --git a/.travis.yml b/.travis.yml index c58495b1f4..d1a316e09d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,13 +48,20 @@ services: before_install: - | - # Enable Sury ppa + # Enable extra ppa sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 6B05F25D762E3157 sudo add-apt-repository -y ppa:ondrej/php sudo rm /etc/apt/sources.list.d/google-chrome.list sudo rm /etc/apt/sources.list.d/mongodb-3.4.list + sudo wget -O - http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add - + echo "deb http://packages.couchbase.com/ubuntu xenial xenial/main" | sudo tee /etc/apt/sources.list.d/couchbase.list sudo apt update - sudo apt install -y librabbitmq-dev libsodium-dev + sudo apt install -y librabbitmq-dev libsodium-dev libcouchbase-dev zlib1g-dev + + - | + # Start Couchbase + docker pull couchbase:6.0.1 + docker run -d --name couchbase -p 8091-8094:8091-8094 -p 11210:11210 couchbase:6.0.1 - | # Start Redis cluster @@ -76,6 +83,11 @@ before_install: curl https://codeload.github.com/edenhill/librdkafka/tar.gz/v0.11.6 | tar xzf - -C /tmp/librdkafka (cd /tmp/librdkafka/librdkafka-0.11.6 && ./configure && make && sudo make install) + - | + # Create new Couchbase Cluster and Bucket ephemeral + docker exec couchbase /opt/couchbase/bin/couchbase-cli cluster-init -c localhost:8091 --cluster-username=Administrator --cluster-password=111111 --cluster-ramsize=256 + docker exec couchbase /opt/couchbase/bin/couchbase-cli bucket-create -c localhost:8091 --bucket=cache --bucket-type=ephemeral --bucket-ramsize=100 -u Administrator -p 111111 + - | # General configuration set -e @@ -191,6 +203,7 @@ before_install: tfold ext.amqp tpecl amqp-1.9.4 amqp.so $INI tfold ext.rdkafka tpecl rdkafka-4.0.2 rdkafka.so $INI tfold ext.redis tpecl redis-4.3.0 redis.so $INI "no" + tfold ext.couchbase tpecl couchbase-2.6.0 couchbase.so $INI done - | # List all php extensions with versions diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 7313d16d25..8aae634604 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -21,6 +21,9 @@ + + + diff --git a/src/Symfony/Component/Cache/Adapter/AbstractAdapter.php b/src/Symfony/Component/Cache/Adapter/AbstractAdapter.php index d12d8f6ae0..56e08f9607 100644 --- a/src/Symfony/Component/Cache/Adapter/AbstractAdapter.php +++ b/src/Symfony/Component/Cache/Adapter/AbstractAdapter.php @@ -130,6 +130,9 @@ abstract class AbstractAdapter implements AdapterInterface, CacheInterface, Logg if (0 === strpos($dsn, 'memcached:')) { return MemcachedAdapter::createConnection($dsn, $options); } + if (0 === strpos($dsn, 'couchbase:')) { + return CouchbaseBucketAdapter::createConnection($dsn, $options); + } throw new InvalidArgumentException(sprintf('Unsupported DSN: %s.', $dsn)); } diff --git a/src/Symfony/Component/Cache/Adapter/CouchbaseBucketAdapter.php b/src/Symfony/Component/Cache/Adapter/CouchbaseBucketAdapter.php new file mode 100644 index 0000000000..b3e6f16b19 --- /dev/null +++ b/src/Symfony/Component/Cache/Adapter/CouchbaseBucketAdapter.php @@ -0,0 +1,252 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Cache\Adapter; + +use Symfony\Component\Cache\Exception\CacheException; +use Symfony\Component\Cache\Exception\InvalidArgumentException; +use Symfony\Component\Cache\Marshaller\DefaultMarshaller; +use Symfony\Component\Cache\Marshaller\MarshallerInterface; + +/** + * @author Antonio Jose Cerezo Aranda + */ +class CouchbaseBucketAdapter extends AbstractAdapter +{ + private const THIRTY_DAYS_IN_SECONDS = 2592000; + private const MAX_KEY_LENGTH = 250; + private const KEY_NOT_FOUND = 13; + private const VALID_DSN_OPTIONS = [ + 'operationTimeout', + 'configTimeout', + 'configNodeTimeout', + 'n1qlTimeout', + 'httpTimeout', + 'configDelay', + 'htconfigIdleTimeout', + 'durabilityInterval', + 'durabilityTimeout', + ]; + + private $bucket; + private $marshaller; + + public function __construct(\CouchbaseBucket $bucket, string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null) + { + if (!static::isSupported()) { + throw new CacheException('Couchbase >= 2.6.0 is required.'); + } + + $this->maxIdLength = static::MAX_KEY_LENGTH; + + $this->bucket = $bucket; + + parent::__construct($namespace, $defaultLifetime); + $this->enableVersioning(); + $this->marshaller = $marshaller ?? new DefaultMarshaller(); + } + + /** + * @param array|string $servers + */ + public static function createConnection($servers, array $options = []): \CouchbaseBucket + { + if (\is_string($servers)) { + $servers = [$servers]; + } elseif (!\is_array($servers)) { + throw new \TypeError(sprintf('Argument 1 passed to %s() must be array or string, %s given.', __METHOD__, \gettype($servers))); + } + + if (!static::isSupported()) { + throw new CacheException('Couchbase >= 2.6.0 is required.'); + } + + set_error_handler(function ($type, $msg, $file, $line) { throw new \ErrorException($msg, 0, $type, $file, $line); }); + + $dsnPattern = '/^(?couchbase(?:s)?)\:\/\/(?:(?[^\:]+)\:(?[^\@]{6,})@)?' + .'(?[^\:]+(?:\:\d+)?)(?:\/(?[^\?]+))(?:\?(?.*))?$/i'; + + $newServers = []; + $protocol = 'couchbase'; + try { + $options = self::initOptions($options); + $username = $options['username']; + $password = $options['password']; + + foreach ($servers as $dsn) { + if (0 !== strpos($dsn, 'couchbase:')) { + throw new InvalidArgumentException(sprintf('Invalid Couchbase DSN: %s does not start with "couchbase:".', $dsn)); + } + + preg_match($dsnPattern, $dsn, $matches); + + $username = $matches['username'] ?: $username; + $password = $matches['password'] ?: $password; + $protocol = $matches['protocol'] ?: $protocol; + + if (isset($matches['options'])) { + $optionsInDsn = self::getOptions($matches['options']); + + foreach ($optionsInDsn as $parameter => $value) { + $options[$parameter] = $value; + } + } + + $newServers[] = $matches['host']; + } + + $connectionString = $protocol.'://'.implode(',', $newServers); + + $client = new \CouchbaseCluster($connectionString); + $client->authenticateAs($username, $password); + + $bucket = $client->openBucket($matches['bucketName']); + + unset($options['username'], $options['password']); + foreach ($options as $option => $value) { + if (!empty($value)) { + $bucket->$option = $value; + } + } + + return $bucket; + } finally { + restore_error_handler(); + } + } + + public static function isSupported(): bool + { + return \extension_loaded('couchbase') && version_compare(phpversion('couchbase'), '2.6.0', '>='); + } + + private static function getOptions(string $options): array + { + $results = []; + $optionsInArray = explode('&', $options); + + foreach ($optionsInArray as $option) { + list($key, $value) = explode('=', $option); + + if (\in_array($key, static::VALID_DSN_OPTIONS, true)) { + $results[$key] = $value; + } + } + + return $results; + } + + private static function initOptions(array $options): array + { + $options['username'] = $options['username'] ?? ''; + $options['password'] = $options['password'] ?? ''; + $options['operationTimeout'] = $options['operationTimeout'] ?? 0; + $options['configTimeout'] = $options['configTimeout'] ?? 0; + $options['configNodeTimeout'] = $options['configNodeTimeout'] ?? 0; + $options['n1qlTimeout'] = $options['n1qlTimeout'] ?? 0; + $options['httpTimeout'] = $options['httpTimeout'] ?? 0; + $options['configDelay'] = $options['configDelay'] ?? 0; + $options['htconfigIdleTimeout'] = $options['htconfigIdleTimeout'] ?? 0; + $options['durabilityInterval'] = $options['durabilityInterval'] ?? 0; + $options['durabilityTimeout'] = $options['durabilityTimeout'] ?? 0; + + return $options; + } + + /** + * {@inheritdoc} + */ + protected function doFetch(array $ids) + { + $resultsCouchbase = $this->bucket->get($ids); + + $results = []; + foreach ($resultsCouchbase as $key => $value) { + if (null !== $value->error) { + continue; + } + $results[$key] = $this->marshaller->unmarshall($value->value); + } + + return $results; + } + + /** + * {@inheritdoc} + */ + protected function doHave($id): bool + { + return false !== $this->bucket->get($id); + } + + /** + * {@inheritdoc} + */ + protected function doClear($namespace): bool + { + if ('' === $namespace) { + $this->bucket->manager()->flush(); + + return true; + } + + return false; + } + + /** + * {@inheritdoc} + */ + protected function doDelete(array $ids): bool + { + $results = $this->bucket->remove(array_values($ids)); + + foreach ($results as $key => $result) { + if (null !== $result->error && static::KEY_NOT_FOUND !== $result->error->getCode()) { + continue; + } + unset($results[$key]); + } + + return 0 === \count($results); + } + + /** + * {@inheritdoc} + */ + protected function doSave(array $values, $lifetime) + { + if (!$values = $this->marshaller->marshall($values, $failed)) { + return $failed; + } + + $lifetime = $this->normalizeExpiry($lifetime); + + $ko = []; + foreach ($values as $key => $value) { + $result = $this->bucket->upsert($key, $value, ['expiry' => $lifetime]); + + if (null !== $result->error) { + $ko[$key] = $result; + } + } + + return [] === $ko ? true : $ko; + } + + private function normalizeExpiry(int $expiry): int + { + if ($expiry && $expiry > static::THIRTY_DAYS_IN_SECONDS) { + $expiry += time(); + } + + return $expiry; + } +} diff --git a/src/Symfony/Component/Cache/CHANGELOG.md b/src/Symfony/Component/Cache/CHANGELOG.md index c7ed54ac91..f328b6729e 100644 --- a/src/Symfony/Component/Cache/CHANGELOG.md +++ b/src/Symfony/Component/Cache/CHANGELOG.md @@ -5,6 +5,7 @@ CHANGELOG ----- * added max-items + LRU + max-lifetime capabilities to `ArrayCache` + * added `CouchbaseBucketAdapter` 5.0.0 ----- diff --git a/src/Symfony/Component/Cache/LockRegistry.php b/src/Symfony/Component/Cache/LockRegistry.php index 6c0fbffc69..ac2670c231 100644 --- a/src/Symfony/Component/Cache/LockRegistry.php +++ b/src/Symfony/Component/Cache/LockRegistry.php @@ -39,6 +39,7 @@ final class LockRegistry __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'ApcuAdapter.php', __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'ArrayAdapter.php', __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'ChainAdapter.php', + __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'CouchbaseBucketAdapter.php', __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'DoctrineAdapter.php', __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'FilesystemAdapter.php', __DIR__.\DIRECTORY_SEPARATOR.'Adapter'.\DIRECTORY_SEPARATOR.'FilesystemTagAwareAdapter.php', diff --git a/src/Symfony/Component/Cache/Tests/Adapter/CouchbaseBucketAdapterTest.php b/src/Symfony/Component/Cache/Tests/Adapter/CouchbaseBucketAdapterTest.php new file mode 100644 index 0000000000..d93c74fc52 --- /dev/null +++ b/src/Symfony/Component/Cache/Tests/Adapter/CouchbaseBucketAdapterTest.php @@ -0,0 +1,54 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\Cache\Tests\Adapter; + +use Psr\Cache\CacheItemPoolInterface; +use Symfony\Component\Cache\Adapter\AbstractAdapter; +use Symfony\Component\Cache\Adapter\CouchbaseBucketAdapter; + +/** + * @requires extension couchbase 2.6.0 + * + * @author Antonio Jose Cerezo Aranda + */ +class CouchbaseBucketAdapterTest extends AdapterTestCase +{ + protected $skippedTests = [ + 'testClearPrefix' => 'Couchbase cannot clear by prefix', + ]; + + /** @var \CouchbaseBucket */ + protected static $client; + + public static function setupBeforeClass(): void + { + self::$client = AbstractAdapter::createConnection('couchbase://'.getenv('COUCHBASE_HOST').'/cache', + ['username' => getenv('COUCHBASE_USER'), 'password' => getenv('COUCHBASE_PASS')] + ); + } + + /** + * {@inheritdoc} + */ + public function createCachePool($defaultLifetime = 0): CacheItemPoolInterface + { + $client = $defaultLifetime + ? AbstractAdapter::createConnection('couchbase://' + .getenv('COUCHBASE_USER') + .':'.getenv('COUCHBASE_PASS') + .'@'.getenv('COUCHBASE_HOST') + .'/cache') + : self::$client; + + return new CouchbaseBucketAdapter($client, str_replace('\\', '.', __CLASS__), $defaultLifetime); + } +} diff --git a/src/Symfony/Component/Cache/phpunit.xml.dist b/src/Symfony/Component/Cache/phpunit.xml.dist index 591046cf1c..0ad6430f0b 100644 --- a/src/Symfony/Component/Cache/phpunit.xml.dist +++ b/src/Symfony/Component/Cache/phpunit.xml.dist @@ -12,6 +12,9 @@ + + +