Issue #2503861 by phenaproxima, benjy, alexpott, mikeryan: Migrate file_managed table to file entities

core/modules/file/migration_templates/d7_file.yml

@@ -0,0 +1,25 @@
+# Every migration that references a file by fid should specify this migration
+# as an optional dependency.
+id: d7_file
+label: Drupal 7 files
+migration_tags:
+  - Drupal 7
+source:
+  plugin: d7_file
+process:
+  fid: fid
+  filename: filename
+  uri: uri
+  filemime: filemime
+  # filesize is dynamically computed when file entities are saved, so there is
+  # no point in migrating it.
+  # filesize: filesize
+  status: status
+  # Drupal 7 didn't keep track of the file's creation or update time -- all it
+  # had was the vague "timestamp" column. So we'll use it for both.
+  created: timestamp
+  changed: timestamp
+  uid: uid
+destination:
+  plugin: entity:file
+  source_path_property: filepath

core/modules/file/src/Plugin/migrate/source/d7/File.php

@@ -0,0 +1,99 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\file\Plugin\migrate\source\d7\File.
+ */
+
+namespace Drupal\file\Plugin\migrate\source\d7;
+
+use Drupal\migrate\Row;
+use Drupal\migrate_drupal\Plugin\migrate\source\DrupalSqlBase;
+
+/**
+ * Drupal 7 file source from database.
+ *
+ * @MigrateSource(
+ *   id = "d7_file"
+ * )
+ */
+class File extends DrupalSqlBase {
+
+  /**
+   * The public file directory path.
+   *
+   * @var string
+   */
+  protected $publicPath;
+
+  /**
+   * The private file directory path, if any.
+   *
+   * @var string
+   */
+  protected $privatePath;
+
+  /**
+   * The temporary file directory path.
+   *
+   * @var string
+   */
+  protected $temporaryPath;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function query() {
+    return $this->select('file_managed', 'f')
+      ->fields('f')
+      ->orderBy('timestamp');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function initializeIterator() {
+    $this->publicPath = $this->variableGet('file_public_path', 'sites/default/files');
+    $this->privatePath = $this->variableGet('file_private_path', NULL);
+    $this->temporaryPath = $this->variableGet('file_temporary_path', '/tmp');
+    return parent::initializeIterator();
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function prepareRow(Row $row) {
+    // Compute the filepath property, which is a physical representation of
+    // the URI relative to the Drupal root.
+    $path = str_replace(['public:/', 'private:/', 'temporary:/'], [$this->publicPath, $this->privatePath, $this->temporaryPath], $row->getSourceProperty('uri'));
+    // At this point, $path could be an absolute path or a relative path,
+    // depending on how the scheme's variable was set. So we need to shear out
+    // the source_base_path in order to make them all relative.
+    $path = str_replace($this->migration->get('destination.source_base_path'), NULL, $path);
+    $row->setSourceProperty('filepath', $path);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fields() {
+    return array(
+      'fid' => $this->t('File ID'),
+      'uid' => $this->t('The {users}.uid who added the file. If set to 0, this file was added by an anonymous user.'),
+      'filename' => $this->t('File name'),
+      'filepath' => $this->t('File path'),
+      'filemime' => $this->t('File Mime Type'),
+      'status' => $this->t('The published status of a file.'),
+      'timestamp' => $this->t('The time that the file was added.'),
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getIds() {
+    $ids['fid']['type'] = 'integer';
+    return $ids;
+  }
+
+}

core/modules/file/src/Tests/Migrate/d7/MigrateFileTest.php

@@ -0,0 +1,97 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\file\Tests\Migrate\d7\MigrateFileTest.
+ */
+
+namespace Drupal\file\Tests\Migrate\d7;
+
+use Drupal\file\Entity\File;
+use Drupal\file\FileInterface;
+use Drupal\migrate_drupal\Tests\d7\MigrateDrupal7TestBase;
+
+/**
+ * Migrates all files in the file_managed table.
+ *
+ * @group file
+ */
+class MigrateFileTest extends MigrateDrupal7TestBase {
+
+  static $modules = ['file'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp() {
+    parent::setUp();
+
+    $this->loadDumps(['FileManaged.php', 'Variable.php']);
+    $this->installEntitySchema('file');
+
+    $fs = \Drupal::service('file_system');
+    // The public file directory active during the test will serve as the
+    // root of the fictional Drupal 7 site we're migrating.
+    $fs->mkdir('public://sites/default/files', NULL, TRUE);
+    file_put_contents('public://sites/default/files/cube.jpeg', str_repeat('*', 3620));
+
+    /** @var \Drupal\migrate\Entity\MigrationInterface $migration */
+    $migration = entity_load('migration', 'd7_file');
+    // Set the destination plugin's source_base_path configuration value, which
+    // would normally be set by the user running the migration.
+    $migration->set('destination', [
+      'plugin' => 'entity:file',
+      // Note that source_base_path must include a trailing slash because it's
+      // prepended directly to the value of the source path property.
+      'source_base_path' => $fs->realpath($this->publicFilesDirectory) . '/',
+      // This is set in the migration's YAML file, but we need to repeat it
+      // here because all the destination configuration must be set at once.
+      'source_path_property' => 'filepath',
+    ]);
+    $this->executeMigration($migration);
+  }
+
+  /**
+   * Tests a single file entity.
+   *
+   * @param integer $id
+   *  The file ID.
+   * @param string $name
+   *  The expected file name.
+   * @param string $uri
+   *  The expected URI.
+   * @param string $mime
+   *  The expected MIME type.
+   * @param integer $size
+   *  The expected file size.
+   * @param integer $created
+   *  The expected creation time.
+   * @param integer $changed
+   *  The expected modification time.
+   * @param integer $uid
+   *  The expected owner ID.
+   */
+  protected function assertEntity($id, $name, $uri, $mime, $size, $created, $changed, $uid) {
+    /** @var \Drupal\file\FileInterface $file */
+    $file = File::load($id);
+    $this->assertTrue($file instanceof FileInterface);
+    $this->assertIdentical($name, $file->getFilename());
+    $this->assertIdentical($uri, $file->getFileUri());
+    $this->assertTrue(file_exists($uri));
+    $this->assertIdentical($mime, $file->getMimeType());
+    $this->assertIdentical($size, $file->getSize());
+    // isPermanent(), isTemporary(), etc. are determined by the status column.
+    $this->assertTrue($file->isPermanent());
+    $this->assertIdentical($created, $file->getCreatedTime());
+    $this->assertIdentical($changed, $file->getChangedTime());
+    $this->assertIdentical($uid, $file->getOwnerId());
+  }
+
+  /**
+   * Tests that all expected files are migrated.
+   */
+  public function testFileMigration() {
+    $this->assertEntity(1, 'cube.jpeg', 'public://cube.jpeg', 'image/jpeg', '3620', '1421727515', '1421727515', '1');
+  }
+
+}

core/modules/file/tests/src/Unit/Plugin/migrate/source/d7/FileTest.php

@@ -0,0 +1,117 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\Tests\file\Unit\Plugin\migrate\source\d7\FileTest.
+ */
+
+namespace Drupal\Tests\file\Unit\Plugin\migrate\source\d7;
+
+use Drupal\file\Plugin\migrate\source\d7\File;
+use Drupal\migrate\Row;
+use Drupal\Tests\migrate\Unit\MigrateSqlSourceTestCase;
+
+/**
+ * Tests D7 file source plugin.
+ *
+ * @group file
+ */
+class FileTest extends MigrateSqlSourceTestCase {
+
+  const PLUGIN_CLASS = 'Drupal\Tests\file\Unit\Plugin\migrate\source\d7\TestFile';
+
+  // The fake Migration configuration entity.
+  protected $migrationConfiguration = array(
+    // The ID of the entity, can be any string.
+    'id' => 'test',
+    'source' => array(
+      'plugin' => 'd7_file',
+    ),
+  );
+
+  protected $expectedResults = [
+    [
+      'fid' => '1',
+      'uid' => '1',
+      'filename' => 'cube.jpeg',
+      'uri' => 'public://cube.jpeg',
+      'filemime' => 'image/jpeg',
+      'filesize' => '3620',
+      'status' => '1',
+      'timestamp' => '1421727515',
+    ],
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp() {
+    $this->databaseContents['file_managed'] = $this->expectedResults;
+    parent::setUp();
+  }
+
+  /**
+   * Tests that public file URIs are properly transformed by prepareRow().
+   */
+  public function testPublicUri() {
+    $this->source->publicPath = 'sites/default/files';
+    $row = new Row(['uri' => 'public://foo.png'], ['uri' => []]);
+    $this->source->prepareRow($row);
+    $this->assertEquals('sites/default/files/foo.png',
+      $row->getSourceProperty('filepath'));
+  }
+
+  /**
+   * Tests that private file URIs are properly transformed by prepareRow().
+   */
+  public function testPrivateUri() {
+    $this->source->privatePath = '/path/to/private/files';
+    $row = new Row(['uri' => 'private://baz.jpeg'], ['uri' => []]);
+    $this->source->prepareRow($row);
+    $this->assertEquals('/path/to/private/files/baz.jpeg',
+      $row->getSourceProperty('filepath'));
+  }
+
+  /**
+   * Tests that temporary file URIs are property transformed by prepareRow().
+   */
+  public function testTemporaryUri() {
+    $this->source->temporaryPath = '/tmp';
+    $row = new Row(['uri' => 'temporary://camelot/lancelot.gif'],
+      ['uri' => []]);
+    $this->source->prepareRow($row);
+    $this->assertEquals('/tmp/camelot/lancelot.gif',
+      $row->getSourceProperty('filepath'));
+  }
+
+}
+
+/**
+ * Testing version of \Drupal\file\Plugin\migrate\source\d7\File.
+ *
+ * Exposes inaccessible properties.
+ */
+class TestFile extends File {
+
+  /**
+   * The public file directory path.
+   *
+   * @var string
+   */
+  public $publicPath;
+
+  /**
+   * The private file directory path, if any.
+   *
+   * @var string
+   */
+  public $privatePath;
+
+  /**
+   * The temporary file directory path.
+   *
+   * @var string
+   */
+  public $temporaryPath;
+
+}

core/modules/migrate_drupal/src/Plugin/migrate/source/d6/File.php

@@ -58,7 +58,6 @@ class File extends DrupalSqlBase {
     return $query;
   }
 
-
   /**
    * {@inheritdoc}
    */