Issue #2361093 by mikeryan, dawehner, Devin Carlson, benjy, phenaproxima: Add a rollback functionality to migrate

core/modules/migrate/src/Event/MigrateEvents.php

@@ -15,6 +15,8 @@ namespace Drupal\migrate\Event;
  * @see \Drupal\migrate\Event\MigrateImportEvent
  * @see \Drupal\migrate\Event\MigratePreRowSaveEvent
  * @see \Drupal\migrate\Event\MigratePostRowSaveEvent
+ * @see \Drupal\migrate\Event\MigrateRollbackEvent
+ * @see \Drupal\migrate\Event\MigrateRowDeleteEvent
  */
 final class MigrateEvents {
 
@@ -108,4 +110,64 @@ final class MigrateEvents {
    */
   const POST_ROW_SAVE = 'migrate.post_row_save';
 
+  /**
+   * Name of the event fired when beginning a migration rollback operation.
+   *
+   * This event allows modules to perform an action whenever a migration
+   * rollback operation is about to begin. The event listener method receives a
+   * \Drupal\migrate\Event\MigrateRollbackEvent instance.
+   *
+   * @Event
+   *
+   * @see \Drupal\migrate\Event\MigrateRollbackEvent
+   *
+   * @var string
+   */
+  const PRE_ROLLBACK = 'migrate.pre_rollback';
+
+  /**
+   * Name of the event fired when finishing a migration rollback operation.
+   *
+   * This event allows modules to perform an action whenever a migration
+   * rollback operation is completing. The event listener method receives a
+   * \Drupal\migrate\Event\MigrateRollbackEvent instance.
+   *
+   * @Event
+   *
+   * @see \Drupal\migrate\Event\MigrateRollbackEvent
+   *
+   * @var string
+   */
+  const POST_ROLLBACK = 'migrate.post_rollback';
+
+  /**
+   * Name of the event fired when about to delete a single item.
+   *
+   * This event allows modules to perform an action whenever a specific item
+   * is about to be deleted by the destination plugin. The event listener method
+   * receives a \Drupal\migrate\Event\MigrateRowDeleteEvent instance.
+   *
+   * @Event
+   *
+   * @see \Drupal\migrate\Event\MigrateRowDeleteEvent
+   *
+   * @var string
+   */
+  const PRE_ROW_DELETE = 'migrate.pre_row_delete';
+
+  /**
+   * Name of the event fired just after a single item has been deleted.
+   *
+   * This event allows modules to perform an action whenever a specific item
+   * has been deleted by the destination plugin. The event listener method
+   * receives a \Drupal\migrate\Event\MigrateRowDeleteEvent instance.
+   *
+   * @Event
+   *
+   * @see \Drupal\migrate\Event\MigrateRowDeleteEvent
+   *
+   * @var string
+   */
+  const POST_ROW_DELETE = 'migrate.post_row_delete';
+
 }

core/modules/migrate/src/Event/MigrateRollbackEvent.php

@@ -0,0 +1,45 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\migrate\Event\MigrateRollbackEvent.
+ */
+
+namespace Drupal\migrate\Event;
+
+use Drupal\migrate\Entity\MigrationInterface;
+use Symfony\Component\EventDispatcher\Event;
+
+/**
+ * Wraps a pre- or post-rollback event for event listeners.
+ */
+class MigrateRollbackEvent extends Event {
+
+  /**
+   * Migration entity.
+   *
+   * @var \Drupal\migrate\Entity\MigrationInterface
+   */
+  protected $migration;
+
+  /**
+   * Constructs an rollback event object.
+   *
+   * @param \Drupal\migrate\Entity\MigrationInterface $migration
+   *   Migration entity.
+   */
+  public function __construct(MigrationInterface $migration) {
+    $this->migration = $migration;
+  }
+
+  /**
+   * Gets the migration entity.
+   *
+   * @return \Drupal\migrate\Entity\MigrationInterface
+   *   The migration entity involved.
+   */
+  public function getMigration() {
+    return $this->migration;
+  }
+
+}

core/modules/migrate/src/Event/MigrateRowDeleteEvent.php

@@ -0,0 +1,65 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\migrate\Event\MigrateRowDeleteEvent.
+ */
+
+namespace Drupal\migrate\Event;
+
+use Drupal\migrate\Entity\MigrationInterface;
+use Symfony\Component\EventDispatcher\Event;
+
+/**
+ * Wraps a row deletion event for event listeners.
+ */
+class MigrateRowDeleteEvent extends Event {
+
+  /**
+   * Migration entity.
+   *
+   * @var \Drupal\migrate\Entity\MigrationInterface
+   */
+  protected $migration;
+
+  /**
+   * Values representing the destination ID.
+   *
+   * @var array
+   */
+  protected $destinationIdValues;
+
+  /**
+   * Constructs a row deletion event object.
+   *
+   * @param \Drupal\migrate\Entity\MigrationInterface $migration
+   *   Migration entity.
+   * @param array $destination_id_values
+   *   Values represent the destination ID.
+   */
+  public function __construct(MigrationInterface $migration, $destination_id_values) {
+    $this->migration = $migration;
+    $this->destinationIdValues = $destination_id_values;
+  }
+
+  /**
+   * Gets the migration entity.
+   *
+   * @return \Drupal\migrate\Entity\MigrationInterface
+   *   The migration being rolled back.
+   */
+  public function getMigration() {
+    return $this->migration;
+  }
+
+  /**
+   * Gets the destination ID values.
+   *
+   * @return array
+   *   The destination ID as an array.
+   */
+  public function getDestinationIdValues() {
+    return $this->destinationIdValues;
+  }
+
+}

core/modules/migrate/src/MigrateExecutable.php

@@ -14,6 +14,8 @@ use Drupal\migrate\Event\MigrateEvents;
 use Drupal\migrate\Event\MigrateImportEvent;
 use Drupal\migrate\Event\MigratePostRowSaveEvent;
 use Drupal\migrate\Event\MigratePreRowSaveEvent;
+use Drupal\migrate\Event\MigrateRollbackEvent;
+use Drupal\migrate\Event\MigrateRowDeleteEvent;
 use Drupal\migrate\Exception\RequirementsException;
 use Drupal\migrate\Plugin\MigrateIdMapInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
@@ -76,15 +78,6 @@ class MigrateExecutable implements MigrateExecutableInterface {
    */
   protected $counts = array();
 
-  /**
-   * The maximum number of items to pass in a single call during a rollback.
-   *
-   * For use in bulkRollback(). Can be overridden in derived class constructor.
-   *
-   * @var int
-   */
-  protected $rollbackBatchSize = 50;
-
   /**
    * The object currently being constructed.
    *
@@ -312,6 +305,64 @@ class MigrateExecutable implements MigrateExecutableInterface {
     return $return;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function rollback() {
+    // Only begin the rollback operation if the migration is currently idle.
+    if ($this->migration->getStatus() !== MigrationInterface::STATUS_IDLE) {
+      $this->message->display($this->t('Migration @id is busy with another operation: @status', ['@id' => $this->migration->id(), '@status' => $this->t($this->migration->getStatusLabel())]), 'error');
+      return MigrationInterface::RESULT_FAILED;
+    }
+
+    // Announce that rollback is about to happen.
+    $this->getEventDispatcher()->dispatch(MigrateEvents::PRE_ROLLBACK, new MigrateRollbackEvent($this->migration));
+
+    // Optimistically assume things are going to work out; if not, $return will be
+    // updated to some other status.
+    $return = MigrationInterface::RESULT_COMPLETED;
+
+    $this->migration->setStatus(MigrationInterface::STATUS_ROLLING_BACK);
+    $id_map = $this->migration->getIdMap();
+    $destination = $this->migration->getDestinationPlugin();
+
+    // Loop through each row in the map, and try to roll it back.
+    foreach ($id_map as $serialized_key => $map_row) {
+      $destination_key = $id_map->currentDestination();
+      if ($destination_key) {
+        $this->getEventDispatcher()
+          ->dispatch(MigrateEvents::PRE_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key));
+        $destination->rollback($destination_key);
+        $this->getEventDispatcher()
+          ->dispatch(MigrateEvents::POST_ROW_DELETE, new MigrateRowDeleteEvent($this->migration, $destination_key));
+        // We're now done with this row, so remove it from the map.
+        $id_map->delete(unserialize($serialized_key));
+      }
+
+      // Check for memory exhaustion.
+      if (($return = $this->checkStatus()) != MigrationInterface::RESULT_COMPLETED) {
+        break;
+      }
+
+      // If anyone has requested we stop, return the requested result.
+      if ($this->migration->getStatus() == MigrationInterface::STATUS_STOPPING) {
+        $return = $this->migration->getMigrationResult();
+        break;
+      }
+    }
+    // If rollback completed successfully, reset the high water mark.
+    if ($return == MigrationInterface::RESULT_COMPLETED) {
+      $this->migration->saveHighWater(NULL);
+    }
+
+    // Notify modules that rollback attempt was complete.
+    $this->migration->setMigrationResult($return);
+    $this->getEventDispatcher()->dispatch(MigrateEvents::POST_ROLLBACK, new MigrateRollbackEvent($this->migration));
+    $this->migration->setStatus(MigrationInterface::STATUS_IDLE);
+
+    return $return;
+  }
+
   /**
    * {@inheritdoc}
    */

core/modules/migrate/src/MigrateExecutableInterface.php

@@ -16,6 +16,11 @@ interface MigrateExecutableInterface {
    */
   public function import();
 
+  /**
+   * Performs a rollback operation - remove previously-imported items.
+   */
+  public function rollback();
+
   /**
    * Processes a row.
    *

core/modules/migrate/src/Plugin/MigrateDestinationInterface.php

@@ -59,7 +59,7 @@ interface MigrateDestinationInterface extends PluginInspectionInterface {
    * Import the row.
    *
    * Derived classes must implement import(), to construct one new object
-   * (pre-populated) using ID mappings in the Migration).
+   * (pre-populated) using ID mappings in the Migration.
    *
    * @param \Drupal\migrate\Row $row
    *   The row object.
@@ -72,11 +72,15 @@ interface MigrateDestinationInterface extends PluginInspectionInterface {
   public function import(Row $row, array $old_destination_id_values = array());
 
   /**
-   * Delete the specified IDs from the target Drupal.
+   * Delete the specified destination object from the target Drupal.
    *
-   * @param array $destination_identifiers
-   *   The destination ids to delete.
+   * @param array $destination_identifier
+   *   The ID of the destination object to delete.
    */
-  public function rollbackMultiple(array $destination_identifiers);
+  public function rollback(array $destination_identifier);
 
+  /**
+   * @return bool
+   */
+  public function supportsRollback();
 }

core/modules/migrate/src/Plugin/MigrateIdMapInterface.php

@@ -210,7 +210,7 @@ interface MigrateIdMapInterface extends \Iterator, PluginInspectionInterface {
   public function lookupSourceID(array $destination_id_values);
 
   /**
-   * Looks up the destination identifier.
+   * Looks up the destination identifier corresponding to a source key.
    *
    * Given a (possibly multi-field) source identifier value, return the
    * (possibly multi-field) destination identifier value it is mapped to.
@@ -223,6 +223,14 @@ interface MigrateIdMapInterface extends \Iterator, PluginInspectionInterface {
    */
   public function lookupDestinationId(array $source_id_values);
 
+  /**
+   * Looks up the destination identifier currently being iterated.
+   *
+   * @return array
+   *   The destination identifier values of the record, or NULL on failure.
+   */
+  public function currentDestination();
+
   /**
    * Removes any persistent storage used by this map.
    *

core/modules/migrate/src/Plugin/migrate/destination/Config.php

@@ -13,10 +13,8 @@ use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Entity\DependencyTrait;
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
 use Drupal\migrate\Entity\MigrationInterface;
-use Drupal\migrate\MigrateException;
 use Drupal\migrate\Row;
 use Symfony\Component\DependencyInjection\ContainerInterface;
-use Drupal\Core\Config\Config as ConfigObject;
 
 /**
  * Persist data to the config system.
@@ -83,18 +81,6 @@ class Config extends DestinationBase implements ContainerFactoryPluginInterface,
     return TRUE;
   }
 
-  /**
-   * Throw an exception because config can not be rolled back.
-   *
-   * @param array $destination_keys
-   *   The array of destination ids to roll back.
-   *
-   * @throws \Drupal\migrate\MigrateException
-   */
-  public function rollbackMultiple(array $destination_keys) {
-    throw new MigrateException('Configuration can not be rolled back');
-  }
-
   /**
    * {@inheritdoc}
    */

core/modules/migrate/src/Plugin/migrate/destination/DestinationBase.php

@@ -26,6 +26,13 @@ use Drupal\migrate\Plugin\RequirementsInterface;
  */
 abstract class DestinationBase extends PluginBase implements MigrateDestinationInterface, RequirementsInterface {
 
+  /**
+   * Indicates whether the destination can be rolled back.
+   *
+   * @var bool
+   */
+  protected $supportsRollback = FALSE;
+
   /**
    * The migration.
    *
@@ -62,8 +69,14 @@ abstract class DestinationBase extends PluginBase implements MigrateDestinationI
   /**
    * {@inheritdoc}
    */
-  public function rollbackMultiple(array $destination_identifiers) {
+  public function rollback(array $destination_identifier) {
     // By default we do nothing.
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function supportsRollback() {
+    return $this->supportsRollback;
+  }
 }

core/modules/migrate/src/Plugin/migrate/destination/Entity.php

@@ -58,6 +58,7 @@ abstract class Entity extends DestinationBase implements ContainerFactoryPluginI
     parent::__construct($configuration, $plugin_id, $plugin_definition, $migration);
     $this->storage = $storage;
     $this->bundles = $bundles;
+    $this->supportsRollback = TRUE;
   }
 
   /**
@@ -163,6 +164,17 @@ abstract class Entity extends DestinationBase implements ContainerFactoryPluginI
     return $this->storage->getEntityType()->getKey($key);
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function rollback(array $destination_identifier) {
+    // Delete the specified entity from Drupal if it exists.
+    $entity = $this->storage->load(reset($destination_identifier));
+    if ($entity) {
+      $entity->delete();
+    }
+  }
+
   /**
    * {@inheritdoc}
    */

core/modules/migrate/src/Plugin/migrate/id_map/Sql.php

@@ -805,6 +805,22 @@ class Sql extends PluginBase implements MigrateIdMapInterface, ContainerFactoryP
     return serialize($this->currentKey);
   }
 
+  /**
+   * @inheritdoc
+   */
+  public function currentDestination() {
+    if ($this->valid()) {
+      $result = array();
+      foreach ($this->destinationIdFields() as $field_name) {
+        $result[$field_name] = $this->currentRow[$field_name];
+      }
+      return $result;
+    }
+    else {
+      return NULL;
+    }
+  }
+
   /**
    * Implementation of Iterator::next().
    *

core/modules/migrate/src/Tests/MigrateRollbackTest.php

@@ -0,0 +1,167 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\migrate\Tests\MigrateRollbackTest.
+ */
+
+namespace Drupal\migrate\Tests;
+
+use Drupal\migrate\Entity\Migration;
+use Drupal\migrate\MigrateExecutable;
+use Drupal\taxonomy\Entity\Term;
+use Drupal\taxonomy\Entity\Vocabulary;
+
+/**
+ * Tests rolling back of imports.
+ *
+ * @group migrate
+ */
+class MigrateRollbackTest extends MigrateTestBase {
+
+  /**
+   * Modules to enable.
+   *
+   * @var array
+   */
+  public static $modules = ['field', 'taxonomy', 'text'];
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp() {
+    parent::setUp();
+    $this->installEntitySchema('taxonomy_vocabulary');
+    $this->installEntitySchema('taxonomy_term');
+    $this->installConfig(['taxonomy']);
+  }
+
+  /**
+   * Tests rolling back configuration and content entities.
+   */
+  public function testRollback() {
+    // We use vocabularies to demonstrate importing and rolling back
+    // configuration entities.
+    $vocabulary_data_rows = [
+      ['id' => '1', 'name' => 'categories', 'weight' => '2'],
+      ['id' => '2', 'name' => 'tags', 'weight' => '1'],
+    ];
+    $ids = ['id' => ['type' => 'integer']];
+    $config = [
+      'id' => 'vocabularies',
+      'migration_tags' => ['Import and rollback test'],
+      'source' => [
+        'plugin' => 'embedded_data',
+        'data_rows' => $vocabulary_data_rows,
+        'ids' => $ids,
+      ],
+      'process' => [
+        'vid' => 'id',
+        'name' => 'name',
+        'weight' => 'weight',
+      ],
+      'destination' => ['plugin' => 'entity:taxonomy_vocabulary'],
+    ];
+
+    $vocabulary_migration = Migration::create($config);
+    $vocabulary_id_map = $vocabulary_migration->getIdMap();
+
+    $this->assertTrue($vocabulary_migration->getDestinationPlugin()->supportsRollback());
+
+    // Import and validate vocabulary config entities were created.
+    $vocabulary_executable = new MigrateExecutable($vocabulary_migration, $this);
+    $vocabulary_executable->import();
+    foreach ($vocabulary_data_rows as $row) {
+      /** @var Vocabulary $vocabulary */
+      $vocabulary = Vocabulary::load($row['id']);
+      $this->assertTrue($vocabulary);
+      $map_row = $vocabulary_id_map->getRowBySource([$row['id']]);
+      $this->assertNotNull($map_row['destid1']);
+    }
+
+    // We use taxonomy terms to demonstrate importing and rolling back
+    // content entities.
+    $term_data_rows = [
+      ['id' => '1', 'vocab' => '1', 'name' => 'music'],
+      ['id' => '2', 'vocab' => '2', 'name' => 'Bach'],
+      ['id' => '3', 'vocab' => '2', 'name' => 'Beethoven'],
+    ];
+    $ids = ['id' => ['type' => 'integer']];
+    $config = [
+      'id' => 'terms',
+      'migration_tags' => ['Import and rollback test'],
+      'source' => [
+        'plugin' => 'embedded_data',
+        'data_rows' => $term_data_rows,
+        'ids' => $ids,
+      ],
+      'process' => [
+        'tid' => 'id',
+        'vid' => 'vocab',
+        'name' => 'name',
+      ],
+      'destination' => ['plugin' => 'entity:taxonomy_term'],
+      'migration_dependencies' => ['required' => ['vocabularies']],
+    ];
+
+    $term_migration = Migration::create($config);
+    $term_id_map = $term_migration->getIdMap();
+
+    $this->assertTrue($term_migration->getDestinationPlugin()->supportsRollback());
+
+    // Import and validate term entities were created.
+    $term_executable = new MigrateExecutable($term_migration, $this);
+    $term_executable->import();
+    foreach ($term_data_rows as $row) {
+      /** @var Term $term */
+      $term = Term::load($row['id']);
+      $this->assertTrue($term);
+      $map_row = $term_id_map->getRowBySource([$row['id']]);
+      $this->assertNotNull($map_row['destid1']);
+    }
+
+    // Rollback and verify the entities are gone.
+    $term_executable->rollback();
+    foreach ($term_data_rows as $row) {
+      $term = Term::load($row['id']);
+      $this->assertNull($term);
+      $map_row = $term_id_map->getRowBySource([$row['id']]);
+      $this->assertFalse($map_row);
+    }
+    $vocabulary_executable->rollback();
+    foreach ($vocabulary_data_rows as $row) {
+      $term = Vocabulary::load($row['id']);
+      $this->assertNull($term);
+      $map_row = $vocabulary_id_map->getRowBySource([$row['id']]);
+      $this->assertFalse($map_row);
+    }
+
+    // Test that simple configuration is not rollbackable.
+    $term_setting_rows = [
+      ['id' => 1, 'override_selector' => '0', 'terms_per_page_admin' => '10'],
+    ];
+    $ids = ['id' => ['type' => 'integer']];
+    $config = [
+      'id' => 'taxonomy_settings',
+      'migration_tags' => ['Import and rollback test'],
+      'source' => [
+        'plugin' => 'embedded_data',
+        'data_rows' => $term_setting_rows,
+        'ids' => $ids,
+      ],
+      'process' => [
+        'override_selector' => 'override_selector',
+        'terms_per_page_admin' => 'terms_per_page_admin',
+      ],
+      'destination' => [
+        'plugin' => 'config',
+        'config_name' => 'taxonomy.settings',
+      ],
+      'migration_dependencies' => ['required' => ['vocabularies']],
+    ];
+
+    $settings_migration = Migration::create($config);
+    $this->assertFalse($settings_migration->getDestinationPlugin()->supportsRollback());
+  }
+
+}