Use specific command hash strategy for redis-cluster.

This is needed because redis-cluster does not support the same commands
or operations that can be performed with our client-side managed predis
cluster.

For example redis-cluster does not support key tagging (that is, parts
of a key enclosed by {...} to hash only that specific part of a key)
and multiple-key operations suchs as MGET, MSET, SDIFF, SUNION or SINTER.
Some multiple-key operations can be performed anyway if the command has
only one key (e.g. "MGET foo" and "MSET foo bar" will not fail).
This commit is contained in:
Daniele Alessandri
2012-07-29 13:34:26 +02:00
parent 024a31d571
commit 2ba54a0e1c
3 changed files with 608 additions and 10 deletions
@@ -0,0 +1,254 @@
<?php
/*
* This file is part of the Predis package.
*
* (c) Daniele Alessandri <suppakilla@gmail.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Predis\Command\Hash;
use Predis\Command\CommandInterface;
use Predis\Distribution\HashGeneratorInterface;
/**
* Default class used by Predis to calculate hashes out of keys of
* commands supported by redis-cluster.
*
* @author Daniele Alessandri <suppakilla@gmail.com>
*/
class RedisClusterHashStrategy implements CommandHashStrategyInterface
{
private $commands;
/**
*
*/
public function __construct()
{
$this->commands = $this->getDefaultCommands();
}
/**
* Returns the default map of supported commands with their handlers.
*
* @return array
*/
protected function getDefaultCommands()
{
$keyIsFirstArgument = array($this, 'getKeyFromFirstArgument');
return array(
/* commands operating on the key space */
'EXISTS' => $keyIsFirstArgument,
'DEL' => array($this, 'getKeyFromAllArguments'),
'TYPE' => $keyIsFirstArgument,
'EXPIRE' => $keyIsFirstArgument,
'EXPIREAT' => $keyIsFirstArgument,
'PERSIST' => $keyIsFirstArgument,
'PEXPIRE' => $keyIsFirstArgument,
'PEXPIREAT' => $keyIsFirstArgument,
'TTL' => $keyIsFirstArgument,
'PTTL' => $keyIsFirstArgument,
'SORT' => $keyIsFirstArgument, // TODO
/* commands operating on string values */
'APPEND' => $keyIsFirstArgument,
'DECR' => $keyIsFirstArgument,
'DECRBY' => $keyIsFirstArgument,
'GET' => $keyIsFirstArgument,
'GETBIT' => $keyIsFirstArgument,
'MGET' => array($this, 'getKeyFromAllArguments'),
'SET' => $keyIsFirstArgument,
'GETRANGE' => $keyIsFirstArgument,
'GETSET' => $keyIsFirstArgument,
'INCR' => $keyIsFirstArgument,
'INCRBY' => $keyIsFirstArgument,
'SETBIT' => $keyIsFirstArgument,
'SETEX' => $keyIsFirstArgument,
'MSET' => array($this, 'getKeyFromInterleavedArguments'),
'MSETNX' => array($this, 'getKeyFromInterleavedArguments'),
'SETNX' => $keyIsFirstArgument,
'SETRANGE' => $keyIsFirstArgument,
'STRLEN' => $keyIsFirstArgument,
'SUBSTR' => $keyIsFirstArgument,
'BITCOUNT' => $keyIsFirstArgument,
/* commands operating on lists */
'LINSERT' => $keyIsFirstArgument,
'LINDEX' => $keyIsFirstArgument,
'LLEN' => $keyIsFirstArgument,
'LPOP' => $keyIsFirstArgument,
'RPOP' => $keyIsFirstArgument,
'BLPOP' => array($this, 'getKeyFromBlockingListCommands'),
'BRPOP' => array($this, 'getKeyFromBlockingListCommands'),
'LPUSH' => $keyIsFirstArgument,
'LPUSHX' => $keyIsFirstArgument,
'RPUSH' => $keyIsFirstArgument,
'RPUSHX' => $keyIsFirstArgument,
'LRANGE' => $keyIsFirstArgument,
'LREM' => $keyIsFirstArgument,
'LSET' => $keyIsFirstArgument,
'LTRIM' => $keyIsFirstArgument,
/* commands operating on sets */
'SADD' => $keyIsFirstArgument,
'SCARD' => $keyIsFirstArgument,
'SISMEMBER' => $keyIsFirstArgument,
'SMEMBERS' => $keyIsFirstArgument,
'SPOP' => $keyIsFirstArgument,
'SRANDMEMBER' => $keyIsFirstArgument,
'SREM' => $keyIsFirstArgument,
/* commands operating on sorted sets */
'ZADD' => $keyIsFirstArgument,
'ZCARD' => $keyIsFirstArgument,
'ZCOUNT' => $keyIsFirstArgument,
'ZINCRBY' => $keyIsFirstArgument,
'ZRANGE' => $keyIsFirstArgument,
'ZRANGEBYSCORE' => $keyIsFirstArgument,
'ZRANK' => $keyIsFirstArgument,
'ZREM' => $keyIsFirstArgument,
'ZREMRANGEBYRANK' => $keyIsFirstArgument,
'ZREMRANGEBYSCORE' => $keyIsFirstArgument,
'ZREVRANGE' => $keyIsFirstArgument,
'ZREVRANGEBYSCORE' => $keyIsFirstArgument,
'ZREVRANK' => $keyIsFirstArgument,
'ZSCORE' => $keyIsFirstArgument,
/* commands operating on hashes */
'HDEL' => $keyIsFirstArgument,
'HEXISTS' => $keyIsFirstArgument,
'HGET' => $keyIsFirstArgument,
'HGETALL' => $keyIsFirstArgument,
'HMGET' => $keyIsFirstArgument,
'HINCRBY' => $keyIsFirstArgument,
'HINCRBYFLOAT' => $keyIsFirstArgument,
'HKEYS' => $keyIsFirstArgument,
'HLEN' => $keyIsFirstArgument,
'HSET' => $keyIsFirstArgument,
'HSETNX' => $keyIsFirstArgument,
'HVALS' => $keyIsFirstArgument,
);
}
/**
* Returns the list of IDs for the supported commands.
*
* @return array
*/
public function getSupportedCommands()
{
return array_keys($this->commands);
}
/**
* Sets an handler for the specified command ID.
*
* The signature of the callback must have a single parameter
* of type Predis\Command\CommandInterface.
*
* When the callback argument is omitted or NULL, the previously
* associated handler for the specified command ID is removed.
*
* @param string $commandId The ID of the command to be handled.
* @param mixed $callback A valid callable object or NULL.
*/
public function setCommandHandler($commandId, $callback = null)
{
$commandId = strtoupper($commandId);
if (!isset($callback)) {
unset($this->commands[$commandId]);
return;
}
if (!is_callable($callback)) {
throw new \InvalidArgumentException("Callback must be a valid callable object or NULL");
}
$this->commands[$commandId] = $callback;
}
/**
* Extracts the key from the first argument of a command instance.
*
* @param CommandInterface $command Command instance.
* @return string
*/
protected function getKeyFromFirstArgument(CommandInterface $command)
{
return $command->getArgument(0);
}
/**
* Extracts the key from a command that can accept multiple keys ensuring
* that only one key is actually specified to comply with redis-cluster.
*
* @param CommandInterface $command Command instance.
* @return string
*/
protected function getKeyFromAllArguments(CommandInterface $command)
{
$arguments = $command->getArguments();
if (count($arguments) === 1) {
return $arguments[0];
}
}
/**
* Extracts the key from a command that can accept multiple keys ensuring
* that only one key is actually specified to comply with redis-cluster.
*
* @param CommandInterface $command Command instance.
* @return string
*/
protected function getKeyFromInterleavedArguments(CommandInterface $command)
{
$arguments = $command->getArguments();
if (count($arguments) === 2) {
return $arguments[0];
}
}
/**
* Extracts the key from BLPOP and BRPOP commands ensuring that only one key
* is actually specified to comply with redis-cluster.
*
* @param CommandInterface $command Command instance.
* @return string
*/
protected function getKeyFromBlockingListCommands(CommandInterface $command)
{
$arguments = $command->getArguments();
if (count($arguments) === 2) {
return $arguments[0];
}
}
/**
* {@inheritdoc}
*/
public function getHash(HashGeneratorInterface $hasher, CommandInterface $command)
{
if (isset($this->commands[$cmdID = $command->getId()])) {
if ($key = call_user_func($this->commands[$cmdID], $command)) {
return $this->getKeyHash($hasher, $key);
}
}
}
/**
* {@inheritdoc}
*/
public function getKeyHash(HashGeneratorInterface $hasher, $key)
{
return $hasher->hash($key);
}
}
+29 -10
View File
@@ -15,6 +15,9 @@ use Predis\ClientException;
use Predis\NotSupportedException;
use Predis\ResponseErrorInterface;
use Predis\Command\CommandInterface;
use Predis\Command\Hash\RedisClusterHashStrategy;
use Predis\Connection\ConnectionFactory;
use Predis\Connection\ConnectionFactoryInterface;
use Predis\Distribution\CRC16HashGenerator;
/**
@@ -27,17 +30,19 @@ class RedisCluster implements ClusterConnectionInterface, \IteratorAggregate, \C
private $pool;
private $slots;
private $connections;
private $hashgenerator;
private $distributor;
private $cmdHasher;
/**
* @param FactoryInterface $connections Connection factory object.
* @param ConnectionFactoryInterface $connections Connection factory object.
*/
public function __construct(FactoryInterface $connections = null)
public function __construct(ConnectionFactoryInterface $connections = null)
{
$this->pool = array();
$this->slots = array();
$this->connections = $connections;
$this->hashgenerator = new CRC16HashGenerator();
$this->connections = $connections ?: new ConnectionFactory();
$this->distributor = new CRC16HashGenerator();
$this->cmdHasher = new RedisClusterHashStrategy();
}
/**
@@ -119,12 +124,16 @@ class RedisCluster implements ClusterConnectionInterface, \IteratorAggregate, \C
*/
public function getConnection(CommandInterface $command)
{
if ($hash = $command->getHash() === null) {
$hash = $this->hashgenerator->hash($command->getArgument(0));
$hash = $command->getHash();
if (!isset($hash)) {
$hash = $this->cmdHasher->getHash($this->distributor, $command);
if (!isset($hash)) {
throw new NotSupportedException("Cannot send {$command->getId()} commands to redis-cluster");
}
$command->setHash($hash);
}
$slot = $hash & 4095; // 0x0FFF
@@ -133,8 +142,7 @@ class RedisCluster implements ClusterConnectionInterface, \IteratorAggregate, \C
return $this->slots[$slot];
}
$connection = $this->pool[array_rand($this->pool)];
$this->slots[$slot] = $connection;
$this->slots[$slot] = $connection = $this->pool[array_rand($this->pool)];
return $connection;
}
@@ -174,7 +182,7 @@ class RedisCluster implements ClusterConnectionInterface, \IteratorAggregate, \C
switch ($request) {
case 'MOVED':
$this->add($connection);
$this->slots[$slot] = $connection;
$this->slots[(int) $slot] = $connection;
return $this->executeCommand($command);
case 'ASK':
@@ -185,6 +193,17 @@ class RedisCluster implements ClusterConnectionInterface, \IteratorAggregate, \C
}
}
/**
* Returns the underlying command hash strategy used to hash
* commands by their keys.
*
* @return CommandHashStrategy
*/
public function getCommandHashStrategy()
{
return $this->cmdHasher;
}
/**
* {@inheritdoc}
*/
@@ -0,0 +1,325 @@
<?php
/*
* This file is part of the Predis package.
*
* (c) Daniele Alessandri <suppakilla@gmail.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Predis\Command\Hash;
use \PHPUnit_Framework_TestCase as StandardTestCase;
use Predis\Distribution\CRC16HashGenerator;
use Predis\Profile\ServerProfile;
/**
*
*/
class RedisClusterHashStrategyTest extends StandardTestCase
{
/**
* @group disconnected
*/
public function testDoesNotSupportKeyTags()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$this->assertSame(35910, $hashstrategy->getKeyHash($distribution, '{foo}'));
$this->assertSame(60032, $hashstrategy->getKeyHash($distribution, '{foo}:bar'));
$this->assertSame(27528, $hashstrategy->getKeyHash($distribution, '{foo}:baz'));
$this->assertSame(34064, $hashstrategy->getKeyHash($distribution, 'bar:{foo}:bar'));
}
/**
* @group disconnected
*/
public function testSupportedCommands()
{
$hashstrategy = new RedisClusterHashStrategy();
$this->assertSame($this->getExpectedCommands(), $hashstrategy->getSupportedCommands());
}
/**
* @group disconnected
*/
public function testReturnsNullOnUnsupportedCommand()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$command = ServerProfile::getDevelopment()->createCommand('ping');
$this->assertNull($hashstrategy->getHash($distribution, $command));
}
/**
* @group disconnected
*/
public function testFirstKeyCommands()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key');
foreach ($this->getExpectedCommands('keys-first') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNotNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testAllKeysCommandsWithOneKey()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key');
foreach ($this->getExpectedCommands('keys-all') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNotNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testAllKeysCommandsWithMoreKeys()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key1', 'key2');
foreach ($this->getExpectedCommands('keys-all') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testInterleavedKeysCommandsWithOneKey()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key:1', 'value1');
foreach ($this->getExpectedCommands('keys-interleaved') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNotNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testInterleavedKeysCommandsWithMoreKeys()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key:1', 'value1', 'key:2', 'value2');
foreach ($this->getExpectedCommands('keys-interleaved') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testKeysForBlockingListCommandsWithOneKey()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key:1', 10);
foreach ($this->getExpectedCommands('keys-blockinglist') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNotNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testKeysForBlockingListCommandsWithMoreKeys()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$arguments = array('key:1', 'key:2', 10);
foreach ($this->getExpectedCommands('keys-blockinglist') as $commandID) {
$command = $profile->createCommand($commandID, $arguments);
$this->assertNull($hashstrategy->getHash($distribution, $command), $commandID);
}
}
/**
* @group disconnected
*/
public function testUnsettingCommandHandler()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$hashstrategy->setCommandHandler('set');
$hashstrategy->setCommandHandler('get', null);
$command = $profile->createCommand('set', array('key', 'value'));
$this->assertNull($hashstrategy->getHash($distribution, $command));
$command = $profile->createCommand('get', array('key'));
$this->assertNull($hashstrategy->getHash($distribution, $command));
}
/**
* @group disconnected
*/
public function testSettingCustomCommandHandler()
{
$distribution = new CRC16HashGenerator();
$hashstrategy = new RedisClusterHashStrategy();
$profile = ServerProfile::getDevelopment();
$callable = $this->getMock('stdClass', array('__invoke'));
$callable->expects($this->once())
->method('__invoke')
->with($this->isInstanceOf('Predis\Command\CommandInterface'))
->will($this->returnValue('key'));
$hashstrategy->setCommandHandler('get', $callable);
$command = $profile->createCommand('get', array('key'));
$this->assertNotNull($hashstrategy->getHash($distribution, $command));
}
// ******************************************************************** //
// ---- HELPER METHODS ------------------------------------------------ //
// ******************************************************************** //
/**
* Returns the list of expected supported commands.
*
* @param string $type Optional type of command (based on its keys)
* @return array
*/
protected function getExpectedCommands($type = null)
{
$commands = array(
/* commands operating on the key space */
'EXISTS' => 'keys-first',
'DEL' => 'keys-all',
'TYPE' => 'keys-first',
'EXPIRE' => 'keys-first',
'EXPIREAT' => 'keys-first',
'PERSIST' => 'keys-first',
'PEXPIRE' => 'keys-first',
'PEXPIREAT' => 'keys-first',
'TTL' => 'keys-first',
'PTTL' => 'keys-first',
'SORT' => 'keys-first', // TODO
/* commands operating on string values */
'APPEND' => 'keys-first',
'DECR' => 'keys-first',
'DECRBY' => 'keys-first',
'GET' => 'keys-first',
'GETBIT' => 'keys-first',
'MGET' => 'keys-all',
'SET' => 'keys-first',
'GETRANGE' => 'keys-first',
'GETSET' => 'keys-first',
'INCR' => 'keys-first',
'INCRBY' => 'keys-first',
'SETBIT' => 'keys-first',
'SETEX' => 'keys-first',
'MSET' => 'keys-interleaved',
'MSETNX' => 'keys-interleaved',
'SETNX' => 'keys-first',
'SETRANGE' => 'keys-first',
'STRLEN' => 'keys-first',
'SUBSTR' => 'keys-first',
'BITCOUNT' => 'keys-first',
/* commands operating on lists */
'LINSERT' => 'keys-first',
'LINDEX' => 'keys-first',
'LLEN' => 'keys-first',
'LPOP' => 'keys-first',
'RPOP' => 'keys-first',
'BLPOP' => 'keys-blockinglist',
'BRPOP' => 'keys-blockinglist',
'LPUSH' => 'keys-first',
'LPUSHX' => 'keys-first',
'RPUSH' => 'keys-first',
'RPUSHX' => 'keys-first',
'LRANGE' => 'keys-first',
'LREM' => 'keys-first',
'LSET' => 'keys-first',
'LTRIM' => 'keys-first',
/* commands operating on sets */
'SADD' => 'keys-first',
'SCARD' => 'keys-first',
'SISMEMBER' => 'keys-first',
'SMEMBERS' => 'keys-first',
'SPOP' => 'keys-first',
'SRANDMEMBER' => 'keys-first',
'SREM' => 'keys-first',
/* commands operating on sorted sets */
'ZADD' => 'keys-first',
'ZCARD' => 'keys-first',
'ZCOUNT' => 'keys-first',
'ZINCRBY' => 'keys-first',
'ZRANGE' => 'keys-first',
'ZRANGEBYSCORE' => 'keys-first',
'ZRANK' => 'keys-first',
'ZREM' => 'keys-first',
'ZREMRANGEBYRANK' => 'keys-first',
'ZREMRANGEBYSCORE' => 'keys-first',
'ZREVRANGE' => 'keys-first',
'ZREVRANGEBYSCORE' => 'keys-first',
'ZREVRANK' => 'keys-first',
'ZSCORE' => 'keys-first',
/* commands operating on hashes */
'HDEL' => 'keys-first',
'HEXISTS' => 'keys-first',
'HGET' => 'keys-first',
'HGETALL' => 'keys-first',
'HMGET' => 'keys-first',
'HINCRBY' => 'keys-first',
'HINCRBYFLOAT' => 'keys-first',
'HKEYS' => 'keys-first',
'HLEN' => 'keys-first',
'HSET' => 'keys-first',
'HSETNX' => 'keys-first',
'HVALS' => 'keys-first',
);
if (isset($type)) {
$commands = array_filter($commands, function ($expectedType) use ($type) {
return $expectedType === $type;
});
}
return array_keys($commands);
}
}