Issue #2549821 by stefan.r, jhedstrom, plach: Langcode fields should be ASCII-only, but aren't

core/lib/Drupal/Core/Field/Plugin/Field/FieldType/LanguageItem.php

@@ -44,7 +44,6 @@ class LanguageItem extends FieldItemBase {
   public static function propertyDefinitions(FieldStorageDefinitionInterface $field_definition) {
     $properties['value'] = DataDefinition::create('string')
       ->setLabel(t('Language code'))
-      ->setSetting('is_ascii', TRUE)
       ->setRequired(TRUE);
 
     $properties['language'] = DataReferenceDefinition::create('language')
@@ -74,9 +73,8 @@ class LanguageItem extends FieldItemBase {
     return array(
       'columns' => array(
         'value' => array(
-          'type' => 'varchar',
+          'type' => 'varchar_ascii',
           'length' => 12,
-          'is_ascii' => TRUE,
         ),
       ),
     );

core/modules/system/src/Tests/Entity/Update/LangcodeToAsciiUpdateFilledTest.php

@@ -0,0 +1,26 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\system\Tests\Entity\Update\LangcodeToAsciiUpdateFilledTest.
+ */
+
+namespace Drupal\system\Tests\Entity\Update;
+
+/**
+ * Runs LangcodeToAsciiUpdateTest with a dump filled with content.
+ *
+ * @group Entity
+ */
+class LangcodeToAsciiUpdateFilledTest extends LangcodeToAsciiUpdateTest {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setDatabaseDumpFiles() {
+    $this->databaseDumpFiles = [
+      __DIR__ . '/../../../../tests/fixtures/update/drupal-8.filled.standard.php.gz',
+    ];
+  }
+
+}

core/modules/system/src/Tests/Entity/Update/LangcodeToAsciiUpdateTest.php

@@ -0,0 +1,79 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\system\Tests\Entity\Update\LangcodeToAsciiUpdateTest.
+ */
+
+namespace Drupal\system\Tests\Entity\Update;
+
+use Drupal\Core\Database\Database;
+use Drupal\system\Tests\Update\UpdatePathTestBase;
+
+/**
+ * Tests that the entity langcode fields have been updated to varchar_ascii.
+ *
+ * @group Entity
+ */
+class LangcodeToAsciiUpdateTest extends UpdatePathTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function setDatabaseDumpFiles() {
+    $this->databaseDumpFiles = [
+      __DIR__ . '/../../../../tests/fixtures/update/drupal-8.bare.standard.php.gz',
+    ];
+  }
+
+  /**
+   * Tests that the column collation has been updated on MySQL.
+   */
+  public function testLangcodeColumnCollation() {
+    // Only testable on MySQL.
+    // @see https://www.drupal.org/node/301038
+    if (Database::getConnection()->databaseType() !== 'mysql') {
+      $this->pass('This test can only run on MySQL');
+      return;
+    }
+
+    // Check a few different tables.
+    $tables = [
+      'node_field_data' => ['langcode'],
+      'users_field_data' => ['langcode', 'preferred_langcode', 'preferred_admin_langcode'],
+    ];
+    foreach ($tables as $table => $columns) {
+      foreach ($columns as $column) {
+        $this->assertEqual('utf8mb4_general_ci', $this->getColumnCollation($table, $column), 'Found correct starting collation for ' . $table . '.' . $column);
+      }
+    }
+
+    // Apply updates.
+    $this->runUpdates();
+
+    foreach ($tables as $table => $columns) {
+      foreach ($columns as $column) {
+        $this->assertEqual('ascii_general_ci', $this->getColumnCollation($table, $column), 'Found correct updated collation for ' . $table . '.' . $column);
+      }
+    }
+  }
+
+  /**
+   * Determine the column collation.
+   *
+   * @param string $table
+   *   The table name.
+   * @param string $column
+   *   The column name.
+   */
+  protected function getColumnCollation($table, $column) {
+    $query = Database::getConnection()->query("SHOW FULL COLUMNS FROM {" . $table . "}");
+    while ($row = $query->fetchAssoc()) {
+      if ($row['Field'] === $column) {
+        return $row['Collation'];
+      }
+    }
+    $this->fail('No collation found for ' . $table . '.' . $column);
+  }
+
+}

core/modules/system/system.install

@@ -1586,6 +1586,61 @@ function _system_update_create_block($name, $theme_name, array $values) {
   }
 }
 
+/**
+ * Set langcode fields to be ASCII-only.
+ */
+function system_update_8007() {
+  $database = \Drupal::database();
+  $database_schema = $database->schema();
+
+  $schema = \Drupal::keyValue('entity.storage_schema.sql')->getAll();
+  $schema_copy = $schema;
+  foreach ($schema as $item_name => $item) {
+    foreach ($item as $table_name => $table_schema) {
+      foreach ($table_schema as $schema_key => $schema_data) {
+        if ($schema_key == 'fields') {
+          foreach ($schema_data as $field_name => $field_data) {
+            foreach ($field_data as $field_data_property => $field_data_value) {
+              // Langcode fields have the property 'is_ascii' set, instead
+              // they should have set the type to 'varchar_ascii'.
+              if ($field_data_property == 'is_ascii') {
+                unset($schema_copy[$item_name][$table_name]['fields'][$field_name]['is_ascii']);
+                $schema_copy[$item_name][$table_name]['fields'][$field_name]['type'] = 'varchar_ascii';
+                if ($database->driver() == 'mysql') {
+                  $database_schema->changeField($table_name, $field_name, $field_name, $schema_copy[$item_name][$table_name]['fields'][$field_name]);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  \Drupal::keyValue('entity.storage_schema.sql')->setMultiple($schema_copy);
+
+  $definitions = \Drupal::keyValue('entity.definitions.installed')->getAll();
+  $definitions_copy = $definitions;
+  foreach ($definitions as $item_name => $item_value) {
+    $suffix = '.field_storage_definitions';
+    if (substr($item_name, -strlen($suffix)) == $suffix) {
+      foreach ($item_value as $field_name => $field_definition) {
+        $reflection = new \ReflectionObject($field_definition);
+        $schema_property = $reflection->getProperty('schema');
+        $schema_property->setAccessible(TRUE);
+        $schema = $schema_property->getValue($field_definition);
+        if (isset($schema['columns']['value']['is_ascii'])) {
+          $schema['columns']['value']['type'] = 'varchar_ascii';
+          unset($schema['columns']['value']['is_ascii']);
+        }
+        $schema_property->setValue($field_definition, $schema);
+        $schema_property->setAccessible(FALSE);
+        $definitions_copy[$item_name][$field_name] = $field_definition;
+      }
+    }
+  }
+  \Drupal::keyValue('entity.definitions.installed')->setMultiple($definitions_copy);
+}
+
 /**
  * @} End of "addtogroup updates-8.0.0-beta".
  */