Drupal
/
drupal
mirror of https://git.drupalcode.org/project/drupal.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Issue #2505283 by quietone, maxocub, Jo Fitzgerald, mikeryan, Pavan B S, penyaskito, chriscalip, phenaproxima: Handle import of private files

8.4.x
Nathaniel Catchpole 2017-06-28 11:46:57 +01:00
parent fa54442c29
commit 73bfe9122a
9 changed files with 505 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/modules/file/migration_templates/d7_file.yml
View File

@ -6,6 +6,7 @@ migration_tags:
- Drupal 7
source:
plugin: d7_file
scheme: public
constants:
# The tool configuring this migration must set source_base_path. It
# represents the fully qualified path relative to which URIs in the files

39
core/modules/file/migration_templates/d7_file_private.yml Normal file
View File

@ -0,0 +1,39 @@
id: d7_file_private
label: Files
migration_tags:
- Drupal 7
source:
plugin: d7_file
scheme: private
constants:
# source_base_path must be set by the tool configuring this migration.
# It represents the fully qualified path relative to which uris in the files
# table are specified, and must end with a /. See source_full_path
# configuration in this migration's process pipeline as an example.
source_base_path: ''
process:
# If you are using this file to build a custom migration consider removing
# the fid field to allow incremental migrations.
fid: fid
filename: filename
source_full_path:
-
plugin: concat
delimiter: /
source:
- constants/source_base_path
- filepath
uri:
plugin: file_copy
source:
- '@source_full_path'
- uri
filemime: filemime
status: status
# Drupal 7 didn't keep track of the file's creation or update time -- all it
# had was the vague "timestamp" column. So we'll use it for both.
created: timestamp
changed: timestamp
uid: uid
destination:
plugin: entity:file

98
core/modules/file/tests/src/Kernel/Migrate/d7/MigratePrivateFileTest.php Normal file
View File

@ -0,0 +1,98 @@
<?php
namespace Drupal\Tests\file\Kernel\Migrate\d7;
use Drupal\Core\DependencyInjection\ContainerBuilder;
use Drupal\file\Entity\File;
use Drupal\file\FileInterface;
use Drupal\Tests\migrate_drupal\Kernel\d7\MigrateDrupal7TestBase;
/**
* Tests private files migration.
*
* @group file
*/
class MigratePrivateFileTest extends MigrateDrupal7TestBase {
/**
* {@inheritdoc}
*/
public static $modules = ['file'];
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
$this->setSetting('file_private_path', $this->container->get('site.path') . '/private');
$this->installEntitySchema('file');
$fs = $this->container->get('file_system');
// Ensure that the private files directory exists.
$fs->mkdir('private://sites/default/private/', NULL, TRUE);
// Put test file in the source directory.
file_put_contents('private://sites/default/private/Babylon5.txt', str_repeat('*', 3));
/** @var \Drupal\migrate\Plugin\Migration $migration */
$migration = $this->getMigration('d7_file_private');
// Set the source plugin's source_file_private_path configuration value,
// which would normally be set by the user running the migration.
$source = $migration->getSourceConfiguration();
$source['constants']['source_base_path'] = $fs->realpath('private://');
$migration->set('source', $source);
$this->executeMigration($migration);
}
/**
* {@inheritdoc}
*/
public function register(ContainerBuilder $container) {
parent::register($container);
$container->register('stream_wrapper.private', 'Drupal\Core\StreamWrapper\PrivateStream')
->addTag('stream_wrapper', ['scheme' => 'private']);
}
/**
* Tests a single file entity.
*
* @param int $id
* The file ID.
* @param string $name
* The expected file name.
* @param string $uri
* The expected URI.
* @param string $mime
* The expected MIME type.
* @param int $size
* The expected file size.
* @param int $created
* The expected creation time.
* @param int $changed
* The expected modification time.
* @param int $uid
* The expected owner ID.
*/
protected function assertEntity($id, $name, $uri, $mime, $size, $created, $changed, $uid) {
/** @var \Drupal\file\FileInterface $file */
$file = File::load($id);
$this->assertInstanceOf(FileInterface::class, $file);
$this->assertSame($name, $file->getFilename());
$this->assertSame($uri, $file->getFileUri());
$this->assertFileExists($uri);
$this->assertSame($mime, $file->getMimeType());
$this->assertSame($size, $file->getSize());
// isPermanent(), isTemporary(), etc. are determined by the status column.
$this->assertTrue($file->isPermanent());
$this->assertSame($created, $file->getCreatedTime());
$this->assertSame($changed, $file->getChangedTime());
$this->assertSame($uid, $file->getOwnerId());
}
/**
* Tests that all expected files are migrated.
*/
public function testFileMigration() {
$this->assertEntity(3, 'Babylon5.txt', 'private://Babylon5.txt', 'text/plain', '3', '1486104045', '1486104045', '1');
}
}

306
core/modules/migrate_drupal/tests/fixtures/drupal7.php vendored
View File

@ -3426,6 +3426,21 @@ $connection->insert('field_config')
'translatable' => '0',
'deleted' => '0',
))
->values(array(
'id' => '25',
'field_name' => 'field_private_file',
'type' => 'file',
'module' => 'file',
'active' => '1',
'storage_type' => 'field_sql_storage',
'storage_module' => 'field_sql_storage',
'storage_active' => '1',
'locked' => '0',
'data' => 'a:7:{s:12:"translatable";s:1:"0";s:12:"entity_types";a:0:{}s:8:"settings";a:3:{s:13:"display_field";i:0;s:15:"display_default";i:0;s:10:"uri_scheme";s:7:"private";}s:7:"storage";a:5:{s:4:"type";s:17:"field_sql_storage";s:8:"settings";a:0:{}s:6:"module";s:17:"field_sql_storage";s:6:"active";s:1:"1";s:7:"details";a:1:{s:3:"sql";a:2:{s:18:"FIELD_LOAD_CURRENT";a:1:{s:29:"field_data_field_private_file";a:3:{s:3:"fid";s:22:"field_private_file_fid";s:7:"display";s:26:"field_private_file_display";s:11:"description";s:30:"field_private_file_description";}}s:19:"FIELD_LOAD_REVISION";a:1:{s:33:"field_revision_field_private_file";a:3:{s:3:"fid";s:22:"field_private_file_fid";s:7:"display";s:26:"field_private_file_display";s:11:"description";s:30:"field_private_file_description";}}}}}s:12:"foreign keys";a:1:{s:3:"fid";a:2:{s:5:"table";s:12:"file_managed";s:7:"columns";a:1:{s:3:"fid";s:3:"fid";}}}s:7:"indexes";a:1:{s:3:"fid";a:1:{i:0;s:3:"fid";}}s:2:"id";s:2:"25";}',
'cardinality' => '1',
'translatable' => '0',
'deleted' => '0',
))
->execute();
$connection->schema()->createTable('field_config_instance', array(
@ -3837,6 +3852,15 @@ $connection->insert('field_config_instance')
'data' => 'a:7:{s:5:"label";s:14:"Term Reference";s:6:"widget";a:5:{s:6:"weight";s:2:"14";s:4:"type";s:21:"taxonomy_autocomplete";s:6:"module";s:8:"taxonomy";s:6:"active";i:0;s:8:"settings";a:2:{s:4:"size";i:60;s:17:"autocomplete_path";s:21:"taxonomy/autocomplete";}}s:8:"settings";a:1:{s:18:"user_register_form";b:0;}s:7:"display";a:1:{s:7:"default";a:4:{s:5:"label";s:5:"above";s:4:"type";s:6:"hidden";s:6:"weight";s:2:"13";s:8:"settings";a:0:{}}}s:8:"required";i:0;s:11:"description";s:0:"";s:13:"default_value";N;}',
'deleted' => '0',
))
->values(array(
'id' => '42',
'field_id' => '25',
'field_name' => 'field_private_file',
'entity_type' => 'node',
'bundle' => 'test_content_type',
'data' => 'a:6:{s:5:"label";s:12:"Private file";s:6:"widget";a:5:{s:6:"weight";s:2:"19";s:4:"type";s:12:"file_generic";s:6:"module";s:4:"file";s:6:"active";i:1;s:8:"settings";a:1:{s:18:"progress_indicator";s:8:"throbber";}}s:8:"settings";a:5:{s:14:"file_directory";s:0:"";s:15:"file_extensions";s:3:"txt";s:12:"max_filesize";s:0:"";s:17:"description_field";i:0;s:18:"user_register_form";b:0;}s:7:"display";a:1:{s:7:"default";a:5:{s:5:"label";s:5:"above";s:4:"type";s:12:"file_default";s:8:"settings";a:0:{}s:6:"module";s:4:"file";s:6:"weight";i:18;}}s:8:"required";i:0;s:11:"description";s:0:"";}',
'deleted' => '0',
))
->execute();
$connection->schema()->createTable('field_data_body', array(
@ -5340,13 +5364,136 @@ $connection->insert('field_data_field_phone')
'bundle' => 'test_content_type',
'deleted' => '0',
'entity_id' => '1',
'revision_id' => '1',
'revision_id' => '6',
'language' => 'und',
'delta' => '0',
'field_phone_value' => '99-99-99-99',
))
->execute();
$connection->schema()->createTable('field_data_field_private_file', array(
'fields' => array(
'entity_type' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '128',
'default' => '',
),
'bundle' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '128',
'default' => '',
),
'deleted' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'tiny',
'default' => '0',
),
'entity_id' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'normal',
'unsigned' => TRUE,
),
'revision_id' => array(
'type' => 'int',
'not null' => FALSE,
'size' => 'normal',
'unsigned' => TRUE,
),
'language' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '32',
'default' => '',
),
'delta' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'normal',
'unsigned' => TRUE,
),
'field_private_file_fid' => array(
'type' => 'int',
'not null' => FALSE,
'size' => 'normal',
'unsigned' => TRUE,
),
'field_private_file_display' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'tiny',
'default' => '1',
'unsigned' => TRUE,
),
'field_private_file_description' => array(
'type' => 'text',
'not null' => FALSE,
'size' => 'normal',
),
),
'primary key' => array(
'entity_type',
'entity_id',
'deleted',
'delta',
'language',
),
'indexes' => array(
'entity_type' => array(
'entity_type',
),
'bundle' => array(
'bundle',
),
'deleted' => array(
'deleted',
),
'entity_id' => array(
'entity_id',
),
'revision_id' => array(
'revision_id',
),
'language' => array(
'language',
),
'field_private_file_fid' => array(
'field_private_file_fid',
),
),
'mysql_character_set' => 'utf8',
));
$connection->insert('field_data_field_private_file')
->fields(array(
'entity_type',
'bundle',
'deleted',
'entity_id',
'revision_id',
'language',
'delta',
'field_private_file_fid',
'field_private_file_display',
'field_private_file_description',
))
->values(array(
'entity_type' => 'node',
'bundle' => 'test_content_type',
'deleted' => '0',
'entity_id' => '1',
'revision_id' => '6',
'language' => 'und',
'delta' => '0',
'field_private_file_fid' => '4',
'field_private_file_display' => '1',
'field_private_file_description' => '',
))
->execute();
$connection->schema()->createTable('field_data_field_tags', array(
'fields' => array(
'entity_type' => array(
@ -7579,6 +7726,140 @@ $connection->insert('field_revision_field_phone')
'delta' => '0',
'field_phone_value' => '99-99-99-99',
))
->values(array(
'entity_type' => 'node',
'bundle' => 'test_content_type',
'deleted' => '0',
'entity_id' => '1',
'revision_id' => '6',
'language' => 'und',
'delta' => '0',
'field_phone_value' => '99-99-99-99',
))
->execute();
$connection->schema()->createTable('field_revision_field_private_file', array(
'fields' => array(
'entity_type' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '128',
'default' => '',
),
'bundle' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '128',
'default' => '',
),
'deleted' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'tiny',
'default' => '0',
),
'entity_id' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'normal',
'unsigned' => TRUE,
),
'revision_id' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'normal',
'unsigned' => TRUE,
),
'language' => array(
'type' => 'varchar',
'not null' => TRUE,
'length' => '32',
'default' => '',
),
'delta' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'normal',
'unsigned' => TRUE,
),
'field_private_file_fid' => array(
'type' => 'int',
'not null' => FALSE,
'size' => 'normal',
'unsigned' => TRUE,
),
'field_private_file_display' => array(
'type' => 'int',
'not null' => TRUE,
'size' => 'tiny',
'default' => '1',
'unsigned' => TRUE,
),
'field_private_file_description' => array(
'type' => 'text',
'not null' => FALSE,
'size' => 'normal',
),
),
'primary key' => array(
'entity_type',
'entity_id',
'revision_id',
'deleted',
'delta',
'language',
),
'indexes' => array(
'entity_type' => array(
'entity_type',
),
'bundle' => array(
'bundle',
),
'deleted' => array(
'deleted',
),
'entity_id' => array(
'entity_id',
),
'revision_id' => array(
'revision_id',
),
'language' => array(
'language',
),
'field_private_file_fid' => array(
'field_private_file_fid',
),
),
'mysql_character_set' => 'utf8',
));
$connection->insert('field_revision_field_private_file')
->fields(array(
'entity_type',
'bundle',
'deleted',
'entity_id',
'revision_id',
'language',
'delta',
'field_private_file_fid',
'field_private_file_display',
'field_private_file_description',
))
->values(array(
'entity_type' => 'node',
'bundle' => 'test_content_type',
'deleted' => '0',
'entity_id' => '1',
'revision_id' => '6',
'language' => 'und',
'delta' => '0',
'field_private_file_fid' => '4',
'field_private_file_display' => '1',
'field_private_file_description' => '',
))
->execute();
$connection->schema()->createTable('field_revision_field_tags', array(
@ -8365,6 +8646,16 @@ $connection->insert('file_managed')
'status' => '1',
'timestamp' => '1421727516',
))
->values(array(
'fid' => '3',
'uid' => '1',
'filename' => 'Babylon5.txt',
'uri' => 'private://Babylon5.txt',
'filemime' => 'text/plain',
'filesize' => '4',
'status' => '1',
'timestamp' => '1486104045',
))
->execute();
$connection->schema()->createTable('file_usage', array(
@ -8424,14 +8715,14 @@ $connection->insert('file_usage')
'module' => 'file',
'type' => 'node',
'id' => '1',
'count' => '2',
'count' => '3',
))
->values(array(
'fid' => '2',
'module' => 'file',
'type' => 'node',
'id' => '1',
'count' => '1',
'count' => '2',
))
->values(array(
'fid' => '2',
@ -8440,6 +8731,13 @@ $connection->insert('file_usage')
'id' => '2',
'count' => '1',
))
->values(array(
'fid' => '3',
'module' => 'file',
'type' => 'node',
'id' => '1',
'count' => '1',
))
->execute();
$connection->schema()->createTable('filter', array(
@ -43862,7 +44160,7 @@ $connection->insert('variable')
))
->values(array(
'name' => 'file_private_path',
'value' => 's:0:"";',
'value' => 's:21:"sites/default/private";',
))
->values(array(
'name' => 'file_public_path',

3
core/modules/migrate_drupal_ui/src/Batch/MigrateUpgradeImportBatch.php
View File

@ -104,6 +104,9 @@ class MigrateUpgradeImportBatch {
// @todo Find a way to avoid this in https://www.drupal.org/node/2804611.
if ($definition['destination']['plugin'] === 'entity:file') {
// Make sure we have a single trailing slash.
if ($definition['source']['plugin'] === 'd7_file_private') {
$configuration['source']['constants']['source_base_path'] = rtrim($config['source_private_file_path'], '/') . '/';
}
$configuration['source']['constants']['source_base_path'] = rtrim($config['source_base_path'], '/') . '/';
}

50
core/modules/migrate_drupal_ui/src/Form/MigrateUpgradeForm.php
View File

@ -250,6 +250,10 @@ class MigrateUpgradeForm extends ConfirmFormBase {
'source_module' => 'file',
'destination_module' => 'file',
],
'd7_file_private' => [
'source_module' => 'file',
'destination_module' => 'file',
],
'd6_filter_format' => [
'source_module' => 'filter',
'destination_module' => 'filter',
@ -885,6 +889,15 @@ class MigrateUpgradeForm extends ConfirmFormBase {
$default_options = [];
$form['version'] = [
'#type' => 'radios',
'#default_value' => 7,
'#title' => $this->t('Drupal version of the source site'),
'#options' => [6 => $this->t('Drupal 6'), 7 => $this->t('Drupal 7')],
'#required' => TRUE,
];
$form['database'] = [
'#type' => 'details',
'#title' => $this->t('Source database'),
@ -938,10 +951,38 @@ class MigrateUpgradeForm extends ConfirmFormBase {
'#title' => $this->t('Source files'),
'#open' => TRUE,
];
$form['source']['source_base_path'] = [
$form['source']['d6_source_base_path'] = [
'#type' => 'textfield',
'#title' => $this->t('Files directory'),
'#description' => $this->t('To import files from your current Drupal site, enter a local file directory containing your site (e.g. /var/www/docroot), or your site address (for example http://example.com).'),
'#states' => [
'visible' => [
':input[name="version"]' => ['value' => 6],
],
],
];
$form['source']['source_base_path'] = [
'#type' => 'textfield',
'#title' => $this->t('Public files directory'),
'#description' => $this->t('To import public files from your current Drupal site, enter a local file directory containing your site (e.g. /var/www/docroot), or your site address (for example http://example.com).'),
'#states' => [
'visible' => [
':input[name="version"]' => ['value' => 7],
],
],
];
$form['source']['source_private_file_path'] = [
'#type' => 'textfield',
'#title' => $this->t('Private file directory'),
'#default_value' => '',
'#description' => $this->t('To import private files from your current Drupal site, enter a local file directory containing your site (e.g. /var/www/docroot).'),
'#states' => [
'visible' => [
':input[name="version"]' => ['value' => 7],
],
],
];
$form['actions'] = ['#type' => 'actions'];
@ -992,6 +1033,12 @@ class MigrateUpgradeForm extends ConfirmFormBase {
if (!$version) {
$form_state->setErrorByName($database['driver'] . '][0', $this->t('Source database does not contain a recognizable Drupal version.'));
}
elseif ($version != $form_state->getValue('version')) {
$form_state->setErrorByName($database['driver'] . '][0', $this->t('Source database is Drupal version @version but version @selected was selected.', [
'@version' => $version,
'@selected' => $form_state->getValue('version'),
]));
}
else {
$this->createDatabaseStateSettings($database, $version);
$migrations = $this->getMigrations('migrate_drupal_' . $version, $version);
@ -1009,6 +1056,7 @@ class MigrateUpgradeForm extends ConfirmFormBase {
// Store the retrieved migration IDs in form storage.
$form_state->set('migrations', $migration_array);
$form_state->set('source_base_path', $form_state->getValue('source_base_path'));
$form_state->set('source_private_file_path', $form_state->getValue('source_private_file_path'));
// Store the retrived system data in form storage.
$form_state->set('system_data', $system_data);

6
core/modules/migrate_drupal_ui/tests/src/Functional/MigrateUpgradeTestBase.php
View File

@ -127,9 +127,12 @@ abstract class MigrateUpgradeTestBase extends BrowserTestBase {
$drivers = drupal_get_database_types();
$form = $drivers[$driver]->getFormOptions($connection_options);
$connection_options = array_intersect_key($connection_options, $form + $form['advanced_options']);
$version = $this->getLegacyDrupalVersion($this->sourceDatabase);
$edit = [
$driver => $connection_options,
'source_base_path' => $this->getSourceBasePath(),
'source_private_file_path' => $this->getSourceBasePath(),
'version' => $version,
];
if (count($drivers) !== 1) {
$edit['driver'] = $driver;
@ -159,10 +162,9 @@ abstract class MigrateUpgradeTestBase extends BrowserTestBase {
$this->assertEqual($expected_count, $real_count, "Found $real_count $entity_type entities, expected $expected_count.");
}
$version_tag = 'Drupal ' . $this->getLegacyDrupalVersion($this->sourceDatabase);
$plugin_manager = \Drupal::service('plugin.manager.migration');
/** @var \Drupal\migrate\Plugin\Migration[] $all_migrations */
$all_migrations = $plugin_manager->createInstancesByTag($version_tag);
$all_migrations = $plugin_manager->createInstancesByTag('Drupal ' . $version);
foreach ($all_migrations as $migration) {
$id_map = $migration->getIdMap();
foreach ($id_map as $source_id => $map) {

11
core/modules/migrate_drupal_ui/tests/src/Functional/d7/MigrateUpgrade7Test.php
View File

@ -14,6 +14,11 @@ use Drupal\user\Entity\User;
*/
class MigrateUpgrade7Test extends MigrateUpgradeTestBase {
/**
* {@inheritdoc}
*/
public static $modules = ['file'];
/**
* {@inheritdoc}
*/
@ -45,9 +50,9 @@ class MigrateUpgrade7Test extends MigrateUpgradeTestBase {
'configurable_language' => 4,
'contact_form' => 3,
'editor' => 2,
'field_config' => 52,
'field_storage_config' => 39,
'file' => 2,
'field_config' => 53,
'field_storage_config' => 40,
'file' => 3,
'filter_format' => 7,
'image_style' => 6,
'language_content_settings' => 2,

1
core/modules/migrate_drupal_ui/tests/src/Functional/d7/files/sites/default/private/Babylon5.txt Normal file
View File

@ -0,0 +1 @@
***