Issue #3232668: Perform entity validation during 1.x->2.x migration

composer.json

@@ -52,6 +52,7 @@
         },
         "patches": {
             "drupal/core": {
+                "Issue #3134470: Switch to entity owner in EntityContentBase during validation": "https://www.drupal.org/files/issues/2021-07-12/3134470-78.patch",
                 "Issue #2339235: Remove taxonomy hard dependency on node module": "https://www.drupal.org/files/issues/2020-10-12/2339235_60.patch"
             },
             "drupal/entity": {

docs/hosting/migration.md

@@ -123,6 +123,36 @@ responsibility to update the modules for 2.x and provide migration logic.
 
 ## Troubleshooting
 
+### Validation
+
+Validation is performed on all areas, assets, logs, plans, and taxonomy terms
+as they are migrated. This will check things like required fields, allowed
+values, etc. In some cases the data in a 1.x database will not pass validation,
+either because it was not properly validated originally, or due to legacy bugs
+in the farmOS 1.x code. If any entities fail validation, migration will stop
+and an error like the following will be displayed:
+
+    farm_migrate_asset_plant Migration - 1 failed.
+
+You can view validation messages for individual migrations by running
+`drush migrate:messages [migration-id]`, which will provide more details. For
+example:
+
+    $ drush migrate:messages farm_migrate_asset_plant
+     -------------- ------------------- ------- ---------------------------------------------------------
+      Source ID(s)   Destination ID(s)   Level   Message
+     -------------- ------------------- ------- ---------------------------------------------------------
+      432                                1       [asset: 432]: plant_type=This value should not be null.
+     -------------- ------------------- ------- ---------------------------------------------------------
+
+This gives you the opportunity to fix the data in your 1.x database. Then you
+can rollback and re-run the migration, like so:
+
+    drush migrate:rollback farm_migrate_asset_plant
+    drush migrate:import farm_migrate_asset_plant
+
+### Reset status
+
 If an error occurs during migration, the status of the broken migration may be
 stuck as "Importing". In order to rerun the migration, first reset the status
 and then roll back the migration. Replace `[migration_id]` with ID of the

modules/core/migrate/config/install/migrate_plus.migration.farm_migrate_file.yml

@@ -50,14 +50,14 @@ process:
     plugin: get
     source: timestamp
   uid:
-    -
-      plugin: skip_on_empty
-      method: process
-      source: uid
     -
       plugin: migration_lookup
       migration: farm_migrate_user
       source: uid
+      no_stub: true
+    -
+      plugin: default_value
+      default_value: 1
 migration_dependencies:
   required:
     - farm_migrate_user

modules/core/migrate/config/install/migrate_plus.migration.farm_migrate_file_private.yml

@@ -50,14 +50,14 @@ process:
     plugin: get
     source: timestamp
   uid:
-    -
-      plugin: skip_on_empty
-      method: process
-      source: uid
     -
       plugin: migration_lookup
       migration: farm_migrate_user
       source: uid
+      no_stub: true
+    -
+      plugin: default_value
+      default_value: 1
 migration_dependencies:
   required:
     - farm_migrate_user

modules/core/migrate/config/install/migrate_plus.migration_group.farm_migrate_area.yml

@@ -10,10 +10,15 @@ description: 'Migrates areas from farmOS 1.x to farmOS 2.x'
 source_type: 'farmOS 1.x'
 module: null
 shared_configuration:
+  destination:
+    validate: true
   process:
     name:
       plugin: get
       source: name
+    uid:
+      plugin: default_value
+      default_value: 1
     status:
       plugin: default_value
       default_value: active

modules/core/migrate/config/install/migrate_plus.migration_group.farm_migrate_asset.yml

@@ -10,19 +10,21 @@ description: 'Migrates assets from farmOS 1.x to farmOS 2.x'
 source_type: 'farmOS 1.x'
 module: null
 shared_configuration:
+  destination:
+    validate: true
   process:
     name:
       plugin: get
       source: name
     uid:
-      -
-        plugin: skip_on_empty
-        method: process
-        source: uid
       -
         plugin: migration_lookup
         migration: farm_migrate_user
         source: uid
+        no_stub: true
+      -
+        plugin: default_value
+        default_value: 1
     created:
       plugin: get
       source: created

modules/core/migrate/config/install/migrate_plus.migration_group.farm_migrate_log.yml

@@ -10,19 +10,21 @@ description: 'Migrates logs from farmOS 1.x to farmOS 2.x'
 source_type: 'farmOS 1.x'
 module: null
 shared_configuration:
+  destination:
+    validate: true
   process:
     name:
       plugin: get
       source: name
     uid:
-      -
-        plugin: skip_on_empty
-        method: process
-        source: uid
       -
         plugin: migration_lookup
         migration: farm_migrate_user
         source: uid
+        no_stub: true
+      -
+        plugin: default_value
+        default_value: 1
     timestamp:
       plugin: get
       source: timestamp

modules/core/migrate/config/install/migrate_plus.migration_group.farm_migrate_plan.yml

@@ -10,19 +10,21 @@ description: 'Migrates plans from farmOS 1.x to farmOS 2.x'
 source_type: 'farmOS 1.x'
 module: null
 shared_configuration:
+  destination:
+    validate: true
   process:
     name:
       plugin: get
       source: name
     uid:
-      -
-        plugin: skip_on_empty
-        method: process
-        source: uid
       -
         plugin: migration_lookup
         migration: farm_migrate_user
         source: uid
+        no_stub: true
+      -
+        plugin: default_value
+        default_value: 1
     created:
       plugin: get
       source: created

modules/core/migrate/config/install/migrate_plus.migration_group.farm_migrate_taxonomy.yml

@@ -10,6 +10,8 @@ description: 'Migrates taxonomy terms from farmOS 1.x to farmOS 2.x'
 source_type: 'farmOS 1.x'
 module: null
 shared_configuration:
+  destination:
+    validate: true
   process:
     name:
       plugin: get

modules/core/migrate/config/optional/migrate_plus.migration.farm_migrate_inventory.yml

@@ -35,9 +35,14 @@ process:
     source: value_numerator
   value/denominator: value_denominator
   uid:
-    plugin: migration_lookup
-    migration: farm_migrate_user
-    source: uid
+    -
+      plugin: migration_lookup
+      migration: farm_migrate_user
+      source: uid
+      no_stub: true
+    -
+      plugin: default_value
+      default_value: 1
   inventory_asset:
     plugin: farm_migration_group_lookup
     migration_group: farm_migrate_asset

modules/core/migrate/config/optional/migrate_plus.migration.farm_migrate_quantity_standard.yml

@@ -32,9 +32,14 @@ process:
     source: units
   label: label
   uid:
-    plugin: migration_lookup
-    migration: farm_migrate_user
-    source: uid
+    -
+      plugin: migration_lookup
+      migration: farm_migrate_user
+      source: uid
+      no_stub: true
+    -
+      plugin: default_value
+      default_value: 1
 migration_dependencies:
   required:
     - farm_migrate_user

modules/core/migrate/farm_migrate.module

@@ -0,0 +1,22 @@
+<?php
+
+/**
+ * @file
+ * Hooks and customizations for the farm_migrate module.
+ */
+
+use Drupal\Core\Access\AccessResult;
+use Drupal\Core\Entity\EntityInterface;
+use Drupal\Core\Session\AccountInterface;
+
+/**
+ * Implements hook_ENTITY_TYPE_access().
+ */
+function farm_migrate_file_access(EntityInterface $entity, $operation, AccountInterface $account) {
+
+  // Allow access to private file referencing during migration.
+  // @see FarmMigrationSubscriber::allowPrivateFileReferencing()
+  if (\Drupal::state()->get('farm_migrate_allow_file_referencing')) {
+    return AccessResult::allowed();
+  }
+}

modules/core/migrate/farm_migrate.services.yml

@@ -1,7 +1,7 @@
 services:
-  post_migration_subscriber:
-    class: Drupal\farm_migrate\EventSubscriber\PostMigrationSubscriber
+  farm_migrate_event_subscriber:
+    class: Drupal\farm_migrate\EventSubscriber\FarmMigrationSubscriber
     arguments:
-      [ '@database', '@datetime.time' ]
+      [ '@database', '@datetime.time', '@entity_type.manager', '@state' ]
     tags:
       - { name: 'event_subscriber' }

modules/core/migrate/src/EventSubscriber/FarmMigrationSubscriber.php

@@ -0,0 +1,244 @@
+<?php
+
+namespace Drupal\farm_migrate\EventSubscriber;
+
+use Drupal\Component\Datetime\TimeInterface;
+use Drupal\Core\Database\Connection;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\Core\State\StateInterface;
+use Drupal\migrate\Event\MigrateEvents;
+use Drupal\migrate\Event\MigrateImportEvent;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Farm migration event subscriber.
+ */
+class FarmMigrationSubscriber implements EventSubscriberInterface {
+
+  /**
+   * The database service.
+   *
+   * @var \Drupal\Core\Database\Connection
+   */
+  protected $database;
+
+  /**
+   * The time service.
+   *
+   * @var \Drupal\Component\Datetime\TimeInterface
+   */
+  protected $time;
+
+  /**
+   * The entity type manager service.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * The role storage.
+   *
+   * @var \Drupal\user\RoleStorageInterface
+   */
+  protected $roleStorage;
+
+  /**
+   * The state key/value store.
+   *
+   * @var \Drupal\Core\State\StateInterface
+   */
+  protected $state;
+
+  /**
+   * FarmMigrationSubscriber Constructor.
+   *
+   * @param \Drupal\Core\Database\Connection $database
+   *   The database service.
+   * @param \Drupal\Component\Datetime\TimeInterface $time
+   *   The time service.
+   * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
+   *   The entity type manager service.
+   * @param \Drupal\Core\State\StateInterface $state
+   *   The state key/value store.
+   */
+  public function __construct(Connection $database, TimeInterface $time, EntityTypeManagerInterface $entity_type_manager, StateInterface $state) {
+    $this->database = $database;
+    $this->time = $time;
+    $this->entityTypeManager = $entity_type_manager;
+    $this->roleStorage = $entity_type_manager->getStorage('user_role');
+    $this->state = $state;
+  }
+
+  /**
+   * Get subscribed events.
+   *
+   * @inheritdoc
+   */
+  public static function getSubscribedEvents() {
+    $events[MigrateEvents::PRE_IMPORT][] = ['onMigratePreImport'];
+    $events[MigrateEvents::POST_IMPORT][] = ['onMigratePostImport'];
+    return $events;
+  }
+
+  /**
+   * Run pre-migration logic.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function onMigratePreImport(MigrateImportEvent $event) {
+    $this->grantTextFormatPermission($event);
+    $this->allowPrivateFileReferencing($event);
+  }
+
+  /**
+   * Run post-migration logic.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function onMigratePostImport(MigrateImportEvent $event) {
+    $this->revokeTextFormatPermission($event);
+    $this->preventPrivateFileReferencing($event);
+    $this->addRevisionLogMessage($event);
+  }
+
+  /**
+   * Grant default text format permission to anonymous role.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function grantTextFormatPermission(MigrateImportEvent $event) {
+
+    // If the migration is in the farm_migrate_taxonomy migration group,
+    // grant the 'use text format default' permission to anonymous role.
+    // This allows entity validation to pass even when the migration is run
+    // via Drush (which runs as the anonymous user). The permission is revoked
+    // in post-migration.
+    // @see revokeTextFormatPermission()
+    $migration = $event->getMigration();
+    if (isset($migration->migration_group) && $migration->migration_group == 'farm_migrate_taxonomy') {
+      $anonymous = $this->roleStorage->load('anonymous');
+      $anonymous->grantPermission('use text format default');
+      $anonymous->save();
+    }
+  }
+
+  /**
+   * Revoke default text format permission from anonymous role.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function revokeTextFormatPermission(MigrateImportEvent $event) {
+
+    // If the migration is in the farm_migrate_taxonomy migration group,
+    // revoke the 'use text format default' permission to anonymous role.
+    // This permission was added in pre-migration.
+    // @see grantTextFormatPermission()
+    $migration = $event->getMigration();
+    if (isset($migration->migration_group) && $migration->migration_group == 'farm_migrate_taxonomy') {
+      $anonymous = $this->roleStorage->load('anonymous');
+      $anonymous->revokePermission('use text format default');
+      $anonymous->save();
+    }
+  }
+
+  /**
+   * Temporarily allow private files to be referenced by entities.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function allowPrivateFileReferencing(MigrateImportEvent $event) {
+
+    // During farmOS 1.x -> 2.x migrations, Drupal's FileAccessControlHandler
+    // will not allow file entities to be referenced unless they were originally
+    // uploaded by the same user that created the entity that references them.
+    // In farmOS, it is common for an entity to be created by one user, and
+    // photos to be uploaded to it later by a different user. With entity
+    // validation enabled on the migration, this throws a validation error and
+    // doesn't allow the file to be referenced.
+    // We work around this by setting a Drupal state variable during our
+    // migrations, and check for it in hook_ENTITY_TYPE_access(), so we can
+    // explicitly grant access to the files.
+    // This state is removed post-migration.
+    // @see \Drupal\file\FileAccessControlHandler
+    // @see farm_migrate_file_access()
+    // @see preventPrivateFileReferencing()
+    $migration_groups = [
+      'farm_migrate_area',
+      'farm_migrate_asset',
+      'farm_migrate_log',
+      'farm_migrate_plan',
+      'farm_migrate_taxonomy',
+    ];
+    $migration = $event->getMigration();
+    if (isset($migration->migration_group) && in_array($migration->migration_group, $migration_groups)) {
+      $this->state->set('farm_migrate_allow_file_referencing', TRUE);
+    }
+  }
+
+  /**
+   * Prevent private files from being referenced by entities.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function preventPrivateFileReferencing(MigrateImportEvent $event) {
+
+    // Unset the Drupal state variable that was set to temporarily allow private
+    // files to be referenced by entities.
+    // @see farm_migrate_file_access()
+    // @see allowPrivateFileReferencing()
+    $this->state->delete('farm_migrate_allow_file_referencing');
+  }
+
+  /**
+   * Add a revision log message to imported entities.
+   *
+   * @param \Drupal\migrate\Event\MigrateImportEvent $event
+   *   The import event object.
+   */
+  public function addRevisionLogMessage(MigrateImportEvent $event) {
+
+    // Define the migration groups that we will post-process and their
+    // corresponding entity revision tables.
+    $groups_revision_tables = [
+      'farm_migrate_asset' => 'asset_revision',
+      'farm_migrate_area' => 'asset_revision',
+      'farm_migrate_log' => 'log_revision',
+      'farm_migrate_plan' => 'plan_revision',
+      'farm_migrate_quantity' => 'quantity_revision',
+      'farm_migrate_taxonomy' => 'taxonomy_term_revision',
+    ];
+    $migration = $event->getMigration();
+    if (isset($migration->migration_group) && array_key_exists($migration->migration_group, $groups_revision_tables)) {
+
+      // Define the entity id column name. This will be "id" in all cases
+      // except taxonomy_terms, which use "tid".
+      $id_column = 'id';
+      if ($migration->migration_group == 'farm_migrate_taxonomy') {
+        $id_column = 'tid';
+      }
+
+      // Build a query to set the revision log message.
+      $revision_table = $groups_revision_tables[$migration->migration_group];
+      $migration_id = $migration->id();
+      $query = "UPDATE {$revision_table}
+        SET revision_log_message = :revision_log_message
+        WHERE revision_id IN (
+          SELECT r.revision_id
+          FROM {migrate_map_$migration_id} mm
+          INNER JOIN {$revision_table} r ON mm.destid1 = r.$id_column
+        )";
+      $args = [
+        ':revision_log_message' => 'Migrated from farmOS 1.x on ' . date('Y-m-d', $this->time->getRequestTime()),
+      ];
+      $this->database->query($query, $args);
+    }
+  }
+
+}

modules/core/migrate/src/EventSubscriber/PostMigrationSubscriber.php

@@ -1,100 +0,0 @@
-<?php
-
-namespace Drupal\farm_migrate\EventSubscriber;
-
-use Drupal\Component\Datetime\TimeInterface;
-use Drupal\Core\Database\Connection;
-use Drupal\migrate\Event\MigrateEvents;
-use Drupal\migrate\Event\MigrateImportEvent;
-use Symfony\Component\EventDispatcher\EventSubscriberInterface;
-
-/**
- * Class PostMigrationSubscriber.
- *
- * Run our user flagging after the last node migration is run.
- */
-class PostMigrationSubscriber implements EventSubscriberInterface {
-
-  /**
-   * The database service.
-   *
-   * @var \Drupal\Core\Database\Connection
-   */
-  protected $database;
-
-  /**
-   * The time service.
-   *
-   * @var \Drupal\Component\Datetime\TimeInterface
-   */
-  protected $time;
-
-  /**
-   * PostMigrationSubscriber Constructor.
-   *
-   * @param \Drupal\Core\Database\Connection $database
-   *   The database service.
-   * @param \Drupal\Component\Datetime\TimeInterface $time
-   *   The time service.
-   */
-  public function __construct(Connection $database, TimeInterface $time) {
-    $this->database = $database;
-    $this->time = $time;
-  }
-
-  /**
-   * Get subscribed events.
-   *
-   * @inheritdoc
-   */
-  public static function getSubscribedEvents() {
-    $events[MigrateEvents::POST_IMPORT][] = ['onMigratePostImport'];
-    return $events;
-  }
-
-  /**
-   * Run post migration logic.
-   *
-   * @param \Drupal\migrate\Event\MigrateImportEvent $event
-   *   The import event object.
-   */
-  public function onMigratePostImport(MigrateImportEvent $event) {
-
-    // Define the migration groups that we will post-process and their
-    // corresponding entity revision tables.
-    $groups_revision_tables = [
-      'farm_migrate_asset' => 'asset_revision',
-      'farm_migrate_area' => 'asset_revision',
-      'farm_migrate_log' => 'log_revision',
-      'farm_migrate_plan' => 'plan_revision',
-      'farm_migrate_quantity' => 'quantity_revision',
-      'farm_migrate_taxonomy' => 'taxonomy_term_revision',
-    ];
-    $migration = $event->getMigration();
-    if (isset($migration->migration_group) && array_key_exists($migration->migration_group, $groups_revision_tables)) {
-
-      // Define the entity id column name. This will be "id" in all cases
-      // except taxonomy_terms, which use "tid".
-      $id_column = 'id';
-      if ($migration->migration_group == 'farm_migrate_taxonomy') {
-        $id_column = 'tid';
-      }
-
-      // Build a query to set the revision log message.
-      $revision_table = $groups_revision_tables[$migration->migration_group];
-      $migration_id = $migration->id();
-      $query = "UPDATE {$revision_table}
-        SET revision_log_message = :revision_log_message
-        WHERE revision_id IN (
-          SELECT r.revision_id
-          FROM {migrate_map_$migration_id} mm
-          INNER JOIN {$revision_table} r ON mm.destid1 = r.$id_column
-        )";
-      $args = [
-        ':revision_log_message' => 'Migrated from farmOS 1.x on ' . date('Y-m-d', $this->time->getRequestTime()),
-      ];
-      $this->database->query($query, $args);
-    }
-  }
-
-}