Issue #2430219 by alexpott: Implement a key value store to optimise config entity lookups

8.0.x
Nathaniel Catchpole 2015-05-08 15:02:19 +01:00
parent 1bc5359431
commit 2b49f7bb5c
11 changed files with 515 additions and 32 deletions

View File

@ -721,7 +721,9 @@ services:
- [setContainer, ['@service_container']]
entity.query.config:
class: Drupal\Core\Config\Entity\Query\QueryFactory
arguments: ['@config.factory']
arguments: ['@config.factory', '@keyvalue', '@config.manager']
tags:
- { name: event_subscriber }
entity.query.sql:
class: Drupal\Core\Entity\Query\Sql\QueryFactory
arguments: ['@database']

View File

@ -34,6 +34,13 @@ class ConfigEntityType extends EntityType implements ConfigEntityTypeInterface {
*/
protected $static_cache = FALSE;
/**
* Keys that are stored key value store for fast lookup.
*
* @var array
*/
protected $lookup_keys = [];
/**
* The list of configuration entity properties to export from the annotation.
*
@ -74,6 +81,7 @@ class ConfigEntityType extends EntityType implements ConfigEntityTypeInterface {
$this->handlers += array(
'storage' => 'Drupal\Core\Config\Entity\ConfigEntityStorage',
);
$this->lookup_keys[] = 'uuid';
}
/**
@ -190,4 +198,11 @@ class ConfigEntityType extends EntityType implements ConfigEntityTypeInterface {
return NULL;
}
/**
* {@inheritdoc}
*/
public function getLookupKeys() {
return $this->lookup_keys;
}
}

View File

@ -70,4 +70,12 @@ interface ConfigEntityTypeInterface extends EntityTypeInterface {
*/
public function getPropertiesToExport();
/**
* Gets the keys that are available for fast lookup.
*
* @return string[]
* The list of lookup keys.
*/
public function getLookupKeys();
}

View File

@ -0,0 +1,14 @@
<?php
/**
* @file
* Contains \Drupal\Core\Config\Entity\Query\InvalidLookupKeyException.
*/
namespace Drupal\Core\Config\Entity\Query;
/**
* Exception thrown when a config entity uses an invalid lookup key.
*/
class InvalidLookupKeyException extends \LogicException {
}

View File

@ -11,12 +11,20 @@ use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Entity\Query\QueryBase;
use Drupal\Core\Entity\Query\QueryInterface;
use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
/**
* Defines the entity query for configuration entities.
*/
class Query extends QueryBase implements QueryInterface {
/**
* Information about the entity type.
*
* @var \Drupal\Core\Config\Entity\ConfigEntityTypeInterface
*/
protected $entityType;
/**
* The config factory used by the config entity query.
*
@ -24,6 +32,13 @@ class Query extends QueryBase implements QueryInterface {
*/
protected $configFactory;
/**
* The key value factory.
*
* @var \Drupal\Core\KeyValueStore\KeyValueFactoryInterface
*/
protected $keyValueFactory;
/**
* Constructs a Query object.
*
@ -34,12 +49,15 @@ class Query extends QueryBase implements QueryInterface {
* - OR: at least one of the conditions on the query need to match.
* @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
* The config factory.
* @param \Drupal\Core\KeyValueStore\KeyValueFactoryInterface $key_value_factory
* The key value factory.
* @param array $namespaces
* List of potential namespaces of the classes belonging to this query.
*/
function __construct(EntityTypeInterface $entity_type, $conjunction, ConfigFactoryInterface $config_factory, array $namespaces) {
function __construct(EntityTypeInterface $entity_type, $conjunction, ConfigFactoryInterface $config_factory, KeyValueFactoryInterface $key_value_factory, array $namespaces) {
parent::__construct($entity_type, $conjunction, $namespaces);
$this->configFactory = $config_factory;
$this->keyValueFactory = $key_value_factory;
}
/**
@ -108,37 +126,47 @@ class Query extends QueryBase implements QueryInterface {
$prefix = $this->entityType->getConfigPrefix() . '.';
$prefix_length = strlen($prefix);
// Search the conditions for restrictions on entity IDs.
$ids = array();
// Search the conditions for restrictions on configuration object names.
$names = FALSE;
if ($this->condition->getConjunction() == 'AND') {
foreach ($this->condition->conditions() as $condition) {
if (is_string($condition['field']) && $condition['field'] == $this->entityType->getKey('id')) {
$operator = $condition['operator'] ?: (is_array($condition['value']) ? 'IN' : '=');
if ($operator == '=') {
$ids = array($condition['value']);
$lookup_keys = $this->entityType->getLookupKeys();
$conditions = $this->condition->conditions();
foreach ($conditions as $condition_key => $condition) {
$operator = $condition['operator'] ?: (is_array($condition['value']) ? 'IN' : '=');
if (is_string($condition['field']) && ($operator == 'IN' || $operator == '=')) {
// Special case ID lookups.
if ($condition['field'] == $this->entityType->getKey('id')) {
$ids = (array) $condition['value'];
$names = array_map(function ($id) use ($prefix) {
return $prefix . $id;
}, $ids);
}
elseif ($operator == 'IN') {
$ids = $condition['value'];
}
// We stop at the first restricting condition on ID. In the (weird)
// case where there are additional restricting conditions, results
// will be eliminated when the conditions are checked on the loaded
// records.
if ($ids) {
break;
elseif (in_array($condition['field'], $lookup_keys)) {
// If we don't find anything then there are no matches. No point in
// listing anything.
$names = array();
$keys = (array) $condition['value'];
$keys = array_map(function ($value) use ($condition) {
return $condition['field'] . ':' . $value;
}, $keys);
foreach ($this->getConfigKeyStore()->getMultiple($keys) as $list) {
$names = array_merge($names, $list);
}
}
}
// We stop at the first restricting condition on name. In the case where
// there are additional restricting conditions, results will be
// eliminated when the conditions are checked on the loaded records.
if ($names !== FALSE) {
// If the condition has been responsible for narrowing the list of
// configuration to check there is no point in checking it further.
unset($conditions[$condition_key]);
break;
}
}
}
// If there are conditions restricting config ID, we can narrow the list of
// records to load and parse.
if ($ids) {
$names = array_map(function ($id) use ($prefix) {
return $prefix . $id;
}, $ids);
}
// If no restrictions on IDs were found, we need to parse all records.
else {
if ($names === FALSE) {
$names = $this->configFactory->listAll($prefix);
}
@ -150,4 +178,14 @@ class Query extends QueryBase implements QueryInterface {
return $records;
}
/**
* Gets the key value store used to store fast lookups.
*
* @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
* The key value store used to store fast lookups.
*/
protected function getConfigKeyStore() {
return $this->keyValueFactory->get(QueryFactory::CONFIG_LOOKUP_PREFIX . $this->entityTypeId);
}
}

View File

@ -7,17 +7,28 @@
namespace Drupal\Core\Config\Entity\Query;
use Drupal\Core\Config\Config;
use Drupal\Core\Config\ConfigCrudEvent;
use Drupal\Core\Config\ConfigEvents;
use Drupal\Core\Config\ConfigFactoryInterface;
use Drupal\Core\Entity\EntityManagerInterface;
use Drupal\Core\Config\ConfigManagerInterface;
use Drupal\Core\Config\Entity\ConfigEntityTypeInterface;
use Drupal\Core\Entity\EntityTypeInterface;
use Drupal\Core\Entity\Query\QueryBase;
use Drupal\Core\Entity\Query\QueryException;
use Drupal\Core\Entity\Query\QueryFactoryInterface;
use Drupal\Core\KeyValueStore\KeyValueFactoryInterface;
use Symfony\Component\EventDispatcher\EventSubscriberInterface;
/**
* Provides a factory for creating entity query objects for the config backend.
*/
class QueryFactory implements QueryFactoryInterface {
class QueryFactory implements QueryFactoryInterface, EventSubscriberInterface {
/**
* The prefix for the key value collection for fast lookups.
*/
const CONFIG_LOOKUP_PREFIX = 'config.entity.key_store.';
/**
* The config factory used by the config entity query.
@ -36,13 +47,17 @@ class QueryFactory implements QueryFactoryInterface {
/**
* Constructs a QueryFactory object.
*
* @param \Drupal\Core\Config\StorageInterface $config_storage
* The config storage used by the config entity query.
* @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
* The config storage used by the config entity query.
* @param \Drupal\Core\KeyValueStore\KeyValueFactoryInterface $key_value
* The key value factory.
* @param \Drupal\Core\Config\ConfigManagerInterface $config_manager
* The configuration manager.
*/
public function __construct(ConfigFactoryInterface $config_factory) {
public function __construct(ConfigFactoryInterface $config_factory, KeyValueFactoryInterface $key_value, ConfigManagerInterface $config_manager) {
$this->configFactory = $config_factory;
$this->keyValueFactory = $key_value;
$this->configManager = $config_manager;
$this->namespaces = QueryBase::getNamespaces($this);
}
@ -50,7 +65,7 @@ class QueryFactory implements QueryFactoryInterface {
* {@inheritdoc}
*/
public function get(EntityTypeInterface $entity_type, $conjunction) {
return new Query($entity_type, $conjunction, $this->configFactory, $this->namespaces);
return new Query($entity_type, $conjunction, $this->configFactory, $this->keyValueFactory, $this->namespaces);
}
/**
@ -60,4 +75,189 @@ class QueryFactory implements QueryFactoryInterface {
throw new QueryException('Aggregation over configuration entities is not supported');
}
/**
* Gets the key value store used to store fast lookups.
*
* @param \Drupal\Core\Entity\EntityTypeInterface $entity_type
* The entity type.
*
* @return \Drupal\Core\KeyValueStore\KeyValueStoreInterface
* The key value store used to store fast lookups.
*/
protected function getConfigKeyStore(EntityTypeInterface $entity_type) {
return $this->keyValueFactory->get(static::CONFIG_LOOKUP_PREFIX . $entity_type->id());
}
/**
* Updates or adds lookup data.
*
* @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
* The entity type.
* @param \Drupal\Core\Config\Config $config
* The configuration object that is being saved.
*/
protected function updateConfigKeyStore(ConfigEntityTypeInterface $entity_type, Config $config) {
$config_key_store = $this->getConfigKeyStore($entity_type);
foreach ($entity_type->getLookupKeys() as $lookup_key) {
foreach ($this->getKeys($config, $lookup_key, 'get', $entity_type) as $key) {
$values = $config_key_store->get($key, []);
if (!in_array($config->getName(), $values, TRUE)) {
$values[] = $config->getName();
$config_key_store->set($key, $values);
}
}
}
}
/**
* Deletes lookup data.
*
* @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
* The entity type.
* @param \Drupal\Core\Config\Config $config
* The configuration object that is being deleted.
*/
protected function deleteConfigKeyStore(ConfigEntityTypeInterface $entity_type, Config $config) {
$config_key_store = $this->getConfigKeyStore($entity_type);
foreach ($entity_type->getLookupKeys() as $lookup_key) {
foreach ($this->getKeys($config, $lookup_key, 'getOriginal', $entity_type) as $key) {
$values = $config_key_store->get($key, []);
$pos = array_search($config->getName(), $values, TRUE);
if ($pos !== FALSE) {
unset($values[$pos]);
}
if (empty($values)) {
$config_key_store->delete($key);
}
else {
$config_key_store->set($key, $values);
}
}
}
}
/**
* Creates lookup keys for configuration data.
*
* @param \Drupal\Core\Config\Config $config
* The configuration object.
* @param string $key
* The configuration key to look for.
* @param string $get_method
* Which method on the config object to call to get the value. Either 'get'
* or 'getOriginal'.
* @param \Drupal\Core\Config\Entity\ConfigEntityTypeInterface $entity_type
* The configuration entity type.
*
* @return array
* An array of lookup keys concatenated to the configuration values.
*
* @throws \Drupal\Core\Config\Entity\Query\InvalidLookupKeyException
* The provided $key cannot end with a wildcard. This makes no sense since
* you cannot do fast lookups against this.
*/
protected function getKeys(Config $config, $key, $get_method, ConfigEntityTypeInterface $entity_type) {
if (substr($key, -1) == '*') {
throw new InvalidLookupKeyException(strtr('%entity_type lookup key %key ends with a wildcard this can not be used as a lookup', ['%entity_type' => $entity_type->id(), '%key' => $key]));
}
$parts = explode('.*', $key);
// Remove leading dots.
array_walk($parts, function (&$value) {
$value = trim($value, '.');
});
$values = (array) $this->getValues($config, $parts[0], $get_method, $parts);
$output = array();
// Flatten the array to a single dimension and add the key to all the
// values.
array_walk_recursive($values, function ($current) use (&$output, $key) {
if (is_scalar($current)) {
$current = $key . ':' . $current;
}
$output[] = $current;
});
return $output;
}
/**
* Finds all the values for a configuration key in a configuration object.
*
* @param \Drupal\Core\Config\Config $config
* The configuration object.
* @param string $key
* The current key being checked.
* @param string $get_method
* Which method on the config object to call to get the value.
* @param array $parts
* All the parts of a configuration key we are checking.
* @param int $start
* Which position of $parts we are processing. Defaults to 0.
*
* @return array|NULL
* The array of configuration values the match the provided key. NULL if
* the configuration object does not have a value that corresponds to the
* key.
*/
protected function getValues(Config $config, $key, $get_method, array $parts, $start = 0) {
$value = $config->$get_method($key);
if (is_array($value)) {
$new_value = [];
$start++;
if (!isset($parts[$start])) {
// The configuration object does not have a value that corresponds to
// the key.
return NULL;
}
foreach (array_keys($value) as $key_bit) {
$new_key = $key . '.' . $key_bit;
if (!empty($parts[$start])) {
$new_key .= '.' . $parts[$start];
}
$new_value[] = $this->getValues($config, $new_key, $get_method, $parts, $start);
}
$value = $new_value;
}
return $value;
}
/**
* Updates configuration entity in the key store.
*
* @param ConfigCrudEvent $event
* The configuration event.
*/
public function onConfigSave(ConfigCrudEvent $event) {
$saved_config = $event->getConfig();
$entity_type_id = $this->configManager->getEntityTypeIdByName($saved_config->getName());
if ($entity_type_id) {
$entity_type = $this->configManager->getEntityManager()->getDefinition($entity_type_id);
$this->updateConfigKeyStore($entity_type, $saved_config);
}
}
/**
* Removes configuration entity from key store.
*
* @param \Drupal\Core\Config\ConfigCrudEvent $event
* The configuration event.
*/
public function onConfigDelete(ConfigCrudEvent $event) {
$saved_config = $event->getConfig();
$entity_type_id = $this->configManager->getEntityTypeIdByName($saved_config->getName());
if ($entity_type_id) {
$entity_type = $this->configManager->getEntityManager()->getDefinition($entity_type_id);
$this->deleteConfigKeyStore($entity_type, $saved_config);
}
}
/**
* {@inheritdoc}
*/
static function getSubscribedEvents() {
$events[ConfigEvents::SAVE][] = array('onConfigSave', 128);
$events[ConfigEvents::DELETE][] = array('onConfigDelete', 128);
return $events;
}
}

View File

@ -48,6 +48,9 @@ use Drupal\Core\Entity\EntityStorageInterface;
* "plugin",
* "settings",
* "visibility",
* },
* lookup_keys = {
* "theme"
* }
* )
*/

View File

@ -58,4 +58,7 @@ function config_test_entity_type_alter(array &$entity_types) {
$config_test_no_status->set('id', 'config_test_no_status');
$config_test_no_status->set('entity_keys', $keys);
$config_test_no_status->set('config_prefix', 'no_status');
if (\Drupal::service('state')->get('config_test.lookup_keys', FALSE)) {
$entity_types['config_test']->set('lookup_keys', ['uuid', 'style']);
}
}

View File

@ -7,6 +7,7 @@
namespace Drupal\system\Tests\Entity;
use Drupal\Core\Config\Entity\Query\QueryFactory;
use Drupal\simpletest\KernelTestBase;
/**
@ -469,6 +470,64 @@ class ConfigEntityQueryTest extends KernelTestBase {
$this->assertResults(array('3', '4', '5'));
}
/**
* Tests lookup keys are added to the key value store.
*/
public function testLookupKeys() {
\Drupal::service('state')->set('config_test.lookup_keys', TRUE);
\Drupal::entityManager()->clearCachedDefinitions();
$key_value = $this->container->get('keyvalue')->get(QueryFactory::CONFIG_LOOKUP_PREFIX . 'config_test');
$test_entities = [];
$entity = entity_create('config_test', array(
'label' => $this->randomMachineName(),
'id' => '1',
'style' => 'test',
));
$test_entities[$entity->getConfigDependencyName()] = $entity;
$entity->enforceIsNew();
$entity->save();
$expected[] = $entity->getConfigDependencyName();
$this->assertEqual($expected, $key_value->get('style:test'));
$entity = entity_create('config_test', array(
'label' => $this->randomMachineName(),
'id' => '2',
'style' => 'test',
));
$test_entities[$entity->getConfigDependencyName()] = $entity;
$entity->enforceIsNew();
$entity->save();
$expected[] = $entity->getConfigDependencyName();
$this->assertEqual($expected, $key_value->get('style:test'));
$entity = entity_create('config_test', array(
'label' => $this->randomMachineName(),
'id' => '3',
'style' => 'blah',
));
$entity->enforceIsNew();
$entity->save();
// Do not add this entity to the list of expected result as it has a
// different value.
$this->assertEqual($expected, $key_value->get('style:test'));
$this->assertEqual([$entity->getConfigDependencyName()], $key_value->get('style:blah'));
// Ensure that a delete clears a key.
$entity->delete();
$this->assertEqual([], $key_value->get('style:blah'));
// Ensure that delete only clears one key.
$entity_id = array_pop($expected);
$test_entities[$entity_id]->delete();
$this->assertEqual($expected, $key_value->get('style:test'));
$entity_id = array_pop($expected);
$test_entities[$entity_id]->delete();
$this->assertEqual($expected, $key_value->get('style:test'));
}
/**
* Asserts the results as expected regardless of order.
*

View File

@ -31,6 +31,9 @@ use Drupal\tour\TourInterface;
* "module",
* "routes",
* "tips",
* },
* lookup_keys = {
* "routes.*.route_name"
* }
* )
*/

View File

@ -0,0 +1,138 @@
<?php
/**
* @file
* Contains \Drupal\Tests\Core\Config\Entity\Query\QueryFactoryTest.
*/
namespace Drupal\Tests\Core\Config\Entity\Query;
use Drupal\Core\Config\Config;
use Drupal\Core\Config\Entity\Query\QueryFactory;
use Drupal\Tests\UnitTestCase;
/**
* @coversDefaultClass \Drupal\Core\Config\Entity\Query\QueryFactory
* @group Config
*/
class QueryFactoryTest extends UnitTestCase {
/**
* @covers ::getKeys
* @covers ::getValues
*
* @dataProvider providerTestGetKeys
*/
public function testGetKeys(array $expected, $key, Config $config) {
$config_factory = $this->getMock('Drupal\Core\Config\ConfigFactoryInterface');
$key_value_factory = $this->getMock('Drupal\Core\KeyValueStore\KeyValueFactoryInterface');
$config_manager = $this->getMock('Drupal\Core\Config\ConfigManagerInterface');
$config_entity_type = $this->getMock('Drupal\Core\Config\Entity\ConfigEntityTypeInterface');
$query_factory = new QueryFactory($config_factory, $key_value_factory, $config_manager);
$method = new \ReflectionMethod($query_factory, 'getKeys');
$method->setAccessible(TRUE);
$actual = $method->invoke($query_factory, $config, $key, 'get', $config_entity_type);
$this->assertEquals($expected, $actual);
}
public function providerTestGetKeys() {
$tests = [];
$tests[] = [
['uuid:abc'],
'uuid',
$this->getConfigObject('test')->set('uuid', 'abc')
];
// Tests a lookup being set to a top level key when sub-keys exist.
$tests[] = [
[],
'uuid',
$this->getConfigObject('test')->set('uuid.blah', 'abc')
];
// Tests a non existent key.
$tests[] = [
[],
'uuid',
$this->getConfigObject('test')
];
// Tests a non existent sub key.
$tests[] = [
[],
'uuid.blah',
$this->getConfigObject('test')->set('uuid', 'abc')
];
// Tests a existent sub key.
$tests[] = [
['uuid.blah:abc'],
'uuid.blah',
$this->getConfigObject('test')->set('uuid.blah', 'abc')
];
// One wildcard.
$tests[] = [
['test.*.value:a', 'test.*.value:b'],
'test.*.value',
$this->getConfigObject('test')->set('test.a.value', 'a')->set('test.b.value', 'b')
];
// Three wildcards.
$tests[] = [
['test.*.sub2.*.sub4.*.value:aaa', 'test.*.sub2.*.sub4.*.value:aab', 'test.*.sub2.*.sub4.*.value:bab'],
'test.*.sub2.*.sub4.*.value',
$this->getConfigObject('test')
->set('test.a.sub2.a.sub4.a.value', 'aaa')
->set('test.a.sub2.a.sub4.b.value', 'aab')
->set('test.b.sub2.a.sub4.b.value', 'bab')
];
// Three wildcards in a row.
$tests[] = [
['test.*.*.*.value:abc', 'test.*.*.*.value:abd'],
'test.*.*.*.value',
$this->getConfigObject('test')->set('test.a.b.c.value', 'abc')->set('test.a.b.d.value', 'abd')
];
return $tests;
}
/**
* @expectedException \LogicException
* @expectedExceptionMessage test_config_entity_type lookup key test.* ends with a wildcard this can not be used as a lookup
*/
public function testGetKeysWildCardEnd() {
$config_factory = $this->getMock('Drupal\Core\Config\ConfigFactoryInterface');
$key_value_factory = $this->getMock('Drupal\Core\KeyValueStore\KeyValueFactoryInterface');
$config_manager = $this->getMock('Drupal\Core\Config\ConfigManagerInterface');
$config_entity_type = $this->getMock('Drupal\Core\Config\Entity\ConfigEntityTypeInterface');
$config_entity_type->expects($this->atLeastOnce())
->method('id')
->willReturn('test_config_entity_type');
$query_factory = new QueryFactory($config_factory, $key_value_factory, $config_manager);
$method = new \ReflectionMethod($query_factory, 'getKeys');
$method->setAccessible(TRUE);
$method->invoke($query_factory, $this->getConfigObject('test'), 'test.*', 'get', $config_entity_type);
}
/**
* Gets a test configuration object.
*
* @param string $name
* The config name.
*
* @return \Drupal\Core\Config\Config|\PHPUnit_Framework_MockObject_MockObject
* The test configuration object.
*/
protected function getConfigObject($name) {
$config = $this->getMockBuilder('Drupal\Core\Config\Config')
->disableOriginalConstructor()
->setMethods(['save', 'delete'])
->getMock();
return $config->setName($name);
}
}