Issue #2783075 by phenaproxima, claudiu.cristea, ohthehugemanatee, chx, mikeryan: Add a "download" process plugin, remove remote capability from FileCopy

8.3.x
Alex Pott 2016-10-26 22:58:17 -07:00
parent 21a86f3a46
commit 17c15fb6ff
5 changed files with 457 additions and 18 deletions

View File

@ -0,0 +1,117 @@
<?php
namespace Drupal\migrate\Plugin\migrate\process;
use Drupal\Core\File\FileSystemInterface;
use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
use GuzzleHttp\Client;
use Symfony\Component\DependencyInjection\ContainerInterface;
/**
* Downloads a file from a remote location into the local file system.
*
* @MigrateProcessPlugin(
* id = "download"
* )
*/
class Download extends ProcessPluginBase implements ContainerFactoryPluginInterface {
/**
* The file system service.
*
* @var \Drupal\Core\File\FileSystemInterface
*/
protected $fileSystem;
/**
* The Guzzle HTTP Client service.
*
* @var \GuzzleHttp\Client
*/
protected $httpClient;
/**
* Constructs a download process plugin.
*
* @param array $configuration
* The plugin configuration.
* @param string $plugin_id
* The plugin ID.
* @param mixed $plugin_definition
* The plugin definition.
* @param \Drupal\Core\File\FileSystemInterface $file_system
* The file system service.
* @param \GuzzleHttp\Client $http_client
* The HTTP client.
*/
public function __construct(array $configuration, $plugin_id, array $plugin_definition, FileSystemInterface $file_system, Client $http_client) {
$configuration += [
'rename' => FALSE,
'guzzle_options' => [],
];
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->fileSystem = $file_system;
$this->httpClient = $http_client;
}
/**
* {@inheritdoc}
*/
public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
return new static(
$configuration,
$plugin_id,
$plugin_definition,
$container->get('file_system'),
$container->get('http_client')
);
}
/**
* {@inheritdoc}
*/
public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
// If we're stubbing a file entity, return a uri of NULL so it will get
// stubbed by the general process.
if ($row->isStub()) {
return NULL;
}
list($source, $destination) = $value;
// Modify the destination filename if necessary.
$replace = !empty($this->configuration['rename']) ?
FILE_EXISTS_RENAME :
FILE_EXISTS_REPLACE;
$final_destination = file_destination($destination, $replace);
// Try opening the file first, to avoid calling file_prepare_directory()
// unnecessarily. We're suppressing fopen() errors because we want to try
// to prepare the directory before we give up and fail.
$destination_stream = @fopen($final_destination, 'w');
if (!$destination_stream) {
// If fopen didn't work, make sure there's a writable directory in place.
$dir = $this->fileSystem->dirname($final_destination);
if (!file_prepare_directory($dir, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) {
throw new MigrateException("Could not create or write to directory '$dir'");
}
// Let's try that fopen again.
$destination_stream = @fopen($final_destination, 'w');
if (!$destination_stream) {
throw new MigrateException("Could not write to file '$final_destination'");
}
}
// Stream the request body directly to the final destination stream.
$this->configuration['guzzle_options']['sink'] = $destination_stream;
// Make the request. Guzzle throws an exception for anything other than 200.
$this->httpClient->get($source, $this->configuration['guzzle_options']);
return $final_destination;
}
}

View File

@ -8,6 +8,7 @@ use Drupal\Core\StreamWrapper\LocalStream;
use Drupal\Core\StreamWrapper\StreamWrapperManagerInterface;
use Drupal\migrate\MigrateException;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\Plugin\MigrateProcessInterface;
use Drupal\migrate\ProcessPluginBase;
use Drupal\migrate\Row;
use Symfony\Component\DependencyInjection\ContainerInterface;
@ -35,6 +36,13 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
*/
protected $fileSystem;
/**
* An instance of the download process plugin.
*
* @var \Drupal\migrate\Plugin\MigrateProcessInterface
*/
protected $downloadPlugin;
/**
* Constructs a file_copy process plugin.
*
@ -48,8 +56,10 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
* The stream wrapper manager service.
* @param \Drupal\Core\File\FileSystemInterface $file_system
* The file system service.
* @param \Drupal\migrate\Plugin\MigrateProcessInterface $download_plugin
* An instance of the download plugin for handling remote URIs.
*/
public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system) {
public function __construct(array $configuration, $plugin_id, array $plugin_definition, StreamWrapperManagerInterface $stream_wrappers, FileSystemInterface $file_system, MigrateProcessInterface $download_plugin) {
$configuration += array(
'move' => FALSE,
'rename' => FALSE,
@ -58,6 +68,7 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
parent::__construct($configuration, $plugin_id, $plugin_definition);
$this->streamWrapperManager = $stream_wrappers;
$this->fileSystem = $file_system;
$this->downloadPlugin = $download_plugin;
}
/**
@ -69,7 +80,8 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
$plugin_id,
$plugin_definition,
$container->get('stream_wrapper_manager'),
$container->get('file_system')
$container->get('file_system'),
$container->get('plugin.manager.migrate.process')->createInstance('download')
);
}
@ -84,8 +96,14 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
}
list($source, $destination) = $value;
// If the source path or URI represents a remote resource, delegate to the
// download plugin.
if (!$this->isLocalUri($source)) {
return $this->downloadPlugin->transform($value, $migrate_executable, $row, $destination_property);
}
// Ensure the source file exists, if it's a local URI or path.
if ($this->isLocalUri($source) && !file_exists($source)) {
if (!file_exists($source)) {
throw new MigrateException("File '$source' does not exist");
}
@ -128,20 +146,14 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
* File destination on success, FALSE on failure.
*/
protected function writeFile($source, $destination, $replace = FILE_EXISTS_REPLACE) {
if ($this->configuration['move']) {
return file_unmanaged_move($source, $destination, $replace);
}
// Check if there is a destination available for copying. If there isn't,
// it already exists at the destination and the replace flag tells us to not
// replace it. In that case, return the original destination.
if (!($final_destination = file_destination($destination, $replace))) {
return $destination;
}
// We can't use file_unmanaged_copy because it will break with remote Urls.
if (@copy($source, $final_destination)) {
return $final_destination;
}
return FALSE;
$function = 'file_unmanaged_' . ($this->configuration['move'] ? 'move' : 'copy');
return $function($source, $destination, $replace);
}
/**
@ -187,8 +199,6 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
/**
* Determines if the source and destination URIs represent identical paths.
*
* If either URI is a remote stream, will return FALSE.
*
* @param string $source
* The source URI.
* @param string $destination
@ -199,10 +209,7 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
* otherwise FALSE.
*/
protected function isLocationUnchanged($source, $destination) {
if ($this->isLocalUri($source) && $this->isLocalUri($destination)) {
return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
}
return FALSE;
return $this->fileSystem->realpath($source) === $this->fileSystem->realpath($destination);
}
/**
@ -219,6 +226,13 @@ class FileCopy extends ProcessPluginBase implements ContainerFactoryPluginInterf
*/
protected function isLocalUri($uri) {
$scheme = $this->fileSystem->uriScheme($uri);
// The vfs scheme is vfsStream, which is used in testing. vfsStream is a
// simulated file system that exists only in memory, but should be treated
// as a local resource.
if ($scheme == 'vfs') {
$scheme = FALSE;
}
return $scheme === FALSE || $this->streamWrapperManager->getViaScheme($scheme) instanceof LocalStream;
}

View File

@ -18,7 +18,7 @@ class CopyFileTest extends FileTestBase {
/**
* {@inheritdoc}
*/
public static $modules = ['system'];
public static $modules = ['migrate', 'system'];
/**
* The file system service.

View File

@ -0,0 +1,180 @@
<?php
namespace Drupal\Tests\migrate\Kernel\process;
use Drupal\Core\StreamWrapper\StreamWrapperInterface;
use Drupal\KernelTests\Core\File\FileTestBase;
use Drupal\migrate\Plugin\migrate\process\FileCopy;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\Plugin\MigrateProcessInterface;
use Drupal\migrate\Row;
/**
* Tests the file_copy process plugin.
*
* @group migrate
*/
class FileCopyTest extends FileTestBase {
/**
* {@inheritdoc}
*/
public static $modules = ['migrate', 'system'];
/**
* The file system service.
*
* @var \Drupal\Core\File\FileSystemInterface
*/
protected $fileSystem;
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
$this->fileSystem = $this->container->get('file_system');
$this->container->get('stream_wrapper_manager')->registerWrapper('temporary', 'Drupal\Core\StreamWrapper\TemporaryStream', StreamWrapperInterface::LOCAL_NORMAL);
}
/**
* Test successful imports/copies.
*/
public function testSuccessfulCopies() {
$file = $this->createUri(NULL, NULL, 'temporary');
$file_absolute = $this->fileSystem->realpath($file);
$data_sets = [
// Test a local to local copy.
[
$this->root . '/core/modules/simpletest/files/image-test.jpg',
'public://file1.jpg'
],
// Test a temporary file using an absolute path.
[
$file_absolute,
'temporary://test.jpg'
],
// Test a temporary file using a relative path.
[
$file_absolute,
'temporary://core/modules/simpletest/files/test.jpg'
],
];
foreach ($data_sets as $data) {
list($source_path, $destination_path) = $data;
$actual_destination = $this->doTransform($source_path, $destination_path);
$message = sprintf('File %s exists', $destination_path);
$this->assertFileExists($destination_path, $message);
// Make sure we didn't accidentally do a move.
$this->assertFileExists($source_path, $message);
$this->assertSame($actual_destination, $destination_path, 'The import returned the copied filename.');
}
}
/**
* Test successful moves.
*/
public function testSuccessfulMoves() {
$file_1 = $this->createUri(NULL, NULL, 'temporary');
$file_1_absolute = $this->fileSystem->realpath($file_1);
$file_2 = $this->createUri(NULL, NULL, 'temporary');
$file_2_absolute = $this->fileSystem->realpath($file_2);
$local_file = $this->createUri(NULL, NULL, 'public');
$data_sets = [
// Test a local to local copy.
[
$local_file,
'public://file1.jpg'
],
// Test a temporary file using an absolute path.
[
$file_1_absolute,
'temporary://test.jpg'
],
// Test a temporary file using a relative path.
[
$file_2_absolute,
'temporary://core/modules/simpletest/files/test.jpg'
],
];
foreach ($data_sets as $data) {
list($source_path, $destination_path) = $data;
$actual_destination = $this->doTransform($source_path, $destination_path, ['move' => TRUE]);
$message = sprintf('File %s exists', $destination_path);
$this->assertFileExists($destination_path, $message);
$message = sprintf('File %s does not exist', $source_path);
$this->assertFileNotExists($source_path, $message);
$this->assertSame($actual_destination, $destination_path, 'The importer returned the moved filename.');
}
}
/**
* Test that non-existent files throw an exception.
*
* @expectedException \Drupal\migrate\MigrateException
*
* @expectedExceptionMessage File '/non/existent/file' does not exist
*/
public function testNonExistentSourceFile() {
$source = '/non/existent/file';
$this->doTransform($source, 'public://wontmatter.jpg');
}
/**
* Test the 'rename' overwrite mode.
*/
public function testRenameFile() {
$source = $this->createUri(NULL, NULL, 'temporary');
$destination = $this->createUri('foo.txt', NULL, 'public');
$expected_destination = 'public://foo_0.txt';
$actual_destination = $this->doTransform($source, $destination, ['rename' => TRUE]);
$this->assertFileExists($expected_destination, 'File was renamed on import');
$this->assertSame($actual_destination, $expected_destination, 'The importer returned the renamed filename.');
}
/**
* Tests that remote URIs are delegated to the download plugin.
*/
public function testDownloadRemoteUri() {
$download_plugin = $this->getMock(MigrateProcessInterface::class);
$download_plugin->expects($this->once())->method('transform');
$plugin = new FileCopy(
[],
$this->randomMachineName(),
[],
$this->container->get('stream_wrapper_manager'),
$this->container->get('file_system'),
$download_plugin
);
$plugin->transform(
['http://drupal.org/favicon.ico', '/destination/path'],
$this->getMock(MigrateExecutableInterface::class),
new Row([], []),
$this->randomMachineName()
);
}
/**
* Do an import using the destination.
*
* @param string $source_path
* Source path to copy from.
* @param string $destination_path
* The destination path to copy to.
* @param array $configuration
* Process plugin configuration settings.
*
* @return string
* The URI of the copied file.
*/
protected function doTransform($source_path, $destination_path, $configuration = []) {
$plugin = FileCopy::create($this->container, $configuration, 'file_copy', []);
$executable = $this->prophesize(MigrateExecutableInterface::class)->reveal();
$row = new Row([], []);
return $plugin->transform([$source_path, $destination_path], $executable, $row, 'foobaz');
}
}

View File

@ -0,0 +1,128 @@
<?php
namespace Drupal\Tests\migrate\Unit\process;
use Drupal\Core\StreamWrapper\StreamWrapperInterface;
use Drupal\KernelTests\Core\File\FileTestBase;
use Drupal\migrate\MigrateException;
use Drupal\migrate\Plugin\migrate\process\Download;
use Drupal\migrate\MigrateExecutableInterface;
use Drupal\migrate\Row;
use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Response;
/**
* Tests the download process plugin.
*
* @group migrate
*/
class DownloadTest extends FileTestBase {
/**
* {@inheritdoc}
*/
public static $modules = ['system'];
/**
* {@inheritdoc}
*/
protected function setUp() {
parent::setUp();
$this->container->get('stream_wrapper_manager')->registerWrapper('temporary', 'Drupal\Core\StreamWrapper\TemporaryStream', StreamWrapperInterface::LOCAL_NORMAL);
}
/**
* Tests a download that overwrites an existing local file.
*/
public function testOverwritingDownload() {
// Create a pre-existing file at the destination, to test overwrite behavior.
$destination_uri = $this->createUri('existing_file.txt');
// Test destructive download.
$actual_destination = $this->doTransform($destination_uri);
$this->assertSame($destination_uri, $actual_destination, 'Import returned a destination that was not renamed');
$this->assertFileNotExists('public://existing_file_0.txt', 'Import did not rename the file');
}
/**
* Tests a download that renames the downloaded file if there's a collision.
*/
public function testNonDestructiveDownload() {
// Create a pre-existing file at the destination, to test overwrite behavior.
$destination_uri = $this->createUri('another_existing_file.txt');
// Test non-destructive download.
$actual_destination = $this->doTransform($destination_uri, ['rename' => TRUE]);
$this->assertSame('public://another_existing_file_0.txt', $actual_destination, 'Import returned a renamed destination');
$this->assertFileExists($actual_destination, 'Downloaded file was created');
}
/**
* Tests that an exception is thrown if the destination URI is not writable.
*/
public function testWriteProectedDestination() {
// Create a pre-existing file at the destination, to test overwrite behavior.
$destination_uri = $this->createUri('not-writable.txt');
// Make the destination non-writable.
$this->container
->get('file_system')
->chmod($destination_uri, 0444);
// Pass or fail, we'll need to make the file writable again so the test
// can clean up after itself.
$fix_permissions = function () use ($destination_uri) {
$this->container
->get('file_system')
->chmod($destination_uri, 0755);
};
try {
$this->doTransform($destination_uri);
$fix_permissions();
$this->fail('MigrateException was not thrown for non-writable destination URI.');
}
catch (MigrateException $e) {
$this->assertTrue(TRUE, 'MigrateException was thrown for non-writable destination URI.');
$fix_permissions();
}
}
/**
* Runs an input value through the download plugin.
*
* @param string $destination_uri
* The destination URI to download to.
* @param array $configuration
* (optional) Configuration for the download plugin.
*
* @return string
* The local URI of the downloaded file.
*/
protected function doTransform($destination_uri, $configuration = []) {
// The HTTP client will return a file with contents 'It worked!'
$body = fopen('data://text/plain;base64,SXQgd29ya2VkIQ==', 'r');
// Prepare a mock HTTP client.
$this->container->set('http_client', $this->getMock(Client::class));
$this->container->get('http_client')
->method('get')
->willReturn(new Response(200, [], $body));
// Instantiate the plugin statically so it can pull dependencies out of
// the container.
$plugin = Download::create($this->container, $configuration, 'download', []);
// Execute the transformation.
$executable = $this->getMock(MigrateExecutableInterface::class);
$row = new Row([], []);
// Return the downloaded file's local URI.
$value = [
'http://drupal.org/favicon.ico',
$destination_uri,
];
return $plugin->transform($value, $executable, $row, 'foobaz');
}
}