6 * This file contains a fork of the Drupal Batch API that has been drastically
7 * simplified and tailored to Drush's unique use case.
9 * The existing API is very targeted towards environments that are web accessible,
10 * and would frequently attempt to redirect the user which would result in the
11 * drush process being completely destroyed with no hope of recovery.
13 * While the original API does offer a 'non progressive' mode which simply
14 * calls each operation in sequence within the current process, in most
15 * implementations (D6), it would still attempt to redirect
16 * unless very specific conditions were met.
18 * When operating in 'non progressive' mode, Drush would experience the problems
19 * that the API was written to solve in the first place, specifically that processes
20 * would exceed the available memory and exit with an error.
22 * Each major release of Drupal has also had slightly different implementations
23 * of the batch API, and this provides a uniform interface to all of these
27 use Drush\Log\LogLevel;
30 * Class extending ArrayObject to allow the batch API to perform logging when
31 * some keys of the array change.
33 * It is used to wrap batch's $context array and set log messages when values
34 * are assigned to keys 'message' or 'error_message'.
36 * @see _drush_batch_worker().
38 class DrushBatchContext extends ArrayObject {
39 function offsetSet($name, $value) {
40 if ($name == 'message') {
41 drush_log(strip_tags($value), LogLevel::OK);
43 elseif ($name == 'error_message') {
44 drush_set_error('DRUSH_BATCH_ERROR', strip_tags($value));
46 parent::offsetSet($name, $value);
51 * Process a Drupal batch by spawning multiple Drush processes.
53 * This function will include the correct batch engine for the current
54 * major version of Drupal, and will make use of the drush_backend_invoke
55 * system to spawn multiple worker threads to handle the processing of
56 * the current batch, while keeping track of available memory.
58 * The batch system will process as many batch sets as possible until
59 * the entire batch has been completed or half of the available memory
62 * This function is a drop in replacement for the existing batch_process()
65 * @param string $command
66 * (optional) The command to call for the back end process. By default this will be
67 * the 'batch-process' command, but some commands will
68 * have special initialization requirements, and will need to define and
69 * use their own command.
72 * @param array $options
75 function drush_backend_batch_process($command = 'batch-process', $args = [], $options = []) {
76 // Command line options to pass to the command.
77 $options['u'] = \Drupal::currentUser()->id();
78 return _drush_backend_batch_process($command, $args, $options);
82 * Process sets from the specified batch.
84 * This function is called by the worker process that is spawned by the
85 * drush_backend_batch_process function.
87 * The command called needs to call this function after it's special bootstrap
88 * requirements have been taken care of.
91 * The batch ID of the batch being processed.
93 function drush_batch_command($id) {
94 include_once(DRUSH_DRUPAL_CORE . '/includes/batch.inc');
95 return _drush_batch_command($id);
99 * Main loop for the Drush batch API.
101 * Saves a record of the batch into the database, and progressively call $command to
102 * process the operations.
105 * The command to call to process the batch.
108 function _drush_backend_batch_process($command = 'batch-process', $args, $options) {
111 $batch =& batch_get();
117 $batch += $process_info;
119 // The batch is now completely built. Allow other modules to make changes
120 // to the batch so that it is easier to reuse batch processes in other
122 \Drupal::moduleHandler()->alter('batch', $batch);
124 // Assign an arbitrary id: don't rely on a serial column in the 'batch'
125 // table, since non-progressive batches skip database storage completely.
126 $batch['id'] = db_next_id();
127 $args[] = $batch['id'];
129 $batch['progressive'] = TRUE;
131 // Move operations to a job queue. Non-progressive batches will use a
132 // memory-based queue.
133 foreach ($batch['sets'] as $key => $batch_set) {
134 _batch_populate_queue($batch, $key);
138 /** @var \Drupal\Core\Batch\BatchStorage $batch_storage */
139 $batch_storage = \Drupal::service('batch.storage');
140 $batch_storage->create($batch);
144 $result = drush_invoke_process('@self', $command, $args);
145 $finished = drush_get_error() || !$result || (isset($result['context']['drush_batch_process_finished']) && $result['context']['drush_batch_process_finished'] == TRUE);
154 * Initialize the batch command and call the worker function.
156 * Loads the batch record from the database and sets up the requirements
157 * for the worker, such as registering the shutdown function.
160 * The batch id of the batch being processed.
162 function _drush_batch_command($id) {
163 $batch =& batch_get();
165 $data = db_query("SELECT batch FROM {batch} WHERE bid = :bid", [
170 $batch = unserialize($data);
176 if (!isset($batch['running'])) {
177 $batch['running'] = TRUE;
180 // Register database update for end of processing.
181 register_shutdown_function('_drush_batch_shutdown');
183 if (_drush_batch_worker()) {
184 return _drush_batch_finished();
190 * Process batch operations
192 * Using the current $batch process each of the operations until the batch
193 * has been completed or half of the available memory for the process has been
196 function _drush_batch_worker() {
197 $batch =& batch_get();
198 $current_set =& _batch_current_set();
201 if (empty($current_set['start'])) {
202 $current_set['start'] = microtime(TRUE);
204 $queue = _batch_queue($current_set);
205 while (!$current_set['success']) {
206 // If this is the first time we iterate this batch set in the current
207 // request, we check if it requires an additional file for functions
209 if ($set_changed && isset($current_set['file']) && is_file($current_set['file'])) {
210 include_once DRUPAL_ROOT . '/' . $current_set['file'];
214 // Assume a single pass operation and set the completion level to 1 by
218 if ($item = $queue->claimItem()) {
219 list($function, $args) = $item->data;
221 // Build the 'context' array and execute the function call.
223 'sandbox' => &$current_set['sandbox'],
224 'results' => &$current_set['results'],
225 'finished' => &$finished,
226 'message' => &$task_message,
228 // Magic wrap to catch changes to 'message' key.
229 $batch_context = new DrushBatchContext($batch_context);
231 // Tolerate recoverable errors.
232 // See https://github.com/drush-ops/drush/issues/1930
233 $halt_on_error = \Drush\Drush::config()->get('runtime.php.halt-on-error', TRUE);
234 \Drush\Drush::config()->set('runtime.php.halt-on-error', FALSE);
235 $message = call_user_func_array($function, array_merge($args, [&$batch_context]));
236 if (!empty($message)) {
237 drush_print(strip_tags($message), 2);
239 \Drush\Drush::config()->set('runtime.php.halt-on-error', $halt_on_error);
241 $finished = $batch_context['finished'];
242 if ($finished >= 1) {
243 // Make sure this step is not counted twice when computing $current.
245 // Remove the processed operation and clear the sandbox.
246 $queue->deleteItem($item);
247 $current_set['count']--;
248 $current_set['sandbox'] = [];
252 // When all operations in the current batch set are completed, browse
253 // through the remaining sets, marking them 'successfully processed'
254 // along the way, until we find a set that contains operations.
255 // _batch_next_set() executes form submit handlers stored in 'control'
256 // sets (see form_execute_handlers()), which can in turn add new sets to
258 $set_changed = FALSE;
259 $old_set = $current_set;
260 while (empty($current_set['count']) && ($current_set['success'] = TRUE) && _batch_next_set()) {
261 $current_set = &_batch_current_set();
262 $current_set['start'] = microtime(TRUE);
266 // At this point, either $current_set contains operations that need to be
267 // processed or all sets have been completed.
268 $queue = _batch_queue($current_set);
270 // If we are in progressive mode, break processing after 1 second.
271 if (drush_memory_limit() > 0 && (memory_get_usage() * 2) >= drush_memory_limit()) {
272 drush_log(dt("Batch process has consumed in excess of 50% of available memory. Starting new thread"), LogLevel::BATCH);
273 // Record elapsed wall clock time.
274 $current_set['elapsed'] = round((microtime(TRUE) - $current_set['start']) * 1000, 2);
279 // Reporting 100% progress will cause the whole batch to be considered
280 // processed. If processing was paused right after moving to a new set,
281 // we have to use the info from the new (unprocessed) set.
282 if ($set_changed && isset($current_set['queue'])) {
283 // Processing will continue with a fresh batch set.
284 $remaining = $current_set['count'];
285 $total = $current_set['total'];
286 $progress_message = $current_set['init_message'];
290 // Processing will continue with the current batch set.
291 $remaining = $old_set['count'];
292 $total = $old_set['total'];
293 $progress_message = $old_set['progress_message'];
296 $current = $total - $remaining + $finished;
297 $percentage = _batch_api_percentage($total, $current);
298 return ($percentage == 100);
302 * End the batch processing:
303 * Call the 'finished' callbacks to allow custom handling of results,
304 * and resolve page redirection.
306 function _drush_batch_finished() {
309 $batch = &batch_get();
311 // Execute the 'finished' callbacks for each batch set, if defined.
312 foreach ($batch['sets'] as $id => $batch_set) {
313 if (isset($batch_set['finished'])) {
314 // Check if the set requires an additional file for function definitions.
315 if (isset($batch_set['file']) && is_file($batch_set['file'])) {
316 include_once DRUPAL_ROOT . '/' . $batch_set['file'];
318 if (is_callable($batch_set['finished'])) {
319 $queue = _batch_queue($batch_set);
320 $operations = $queue->getAllItems();
321 $elapsed = $batch_set['elapsed'] / 1000;
322 $elapsed = drush_drupal_major_version() >=8 ? \Drupal::service('date.formatter')->formatInterval($elapsed) : format_interval($elapsed);
323 call_user_func_array($batch_set['finished'], [$batch_set['success'], $batch_set['results'], $operations, $elapsed]);
324 $results[$id] = $batch_set['results'];
329 // Clean up the batch table and unset the static $batch variable.
330 if (drush_drupal_major_version() >= 8) {
331 /** @var \Drupal\Core\Batch\BatchStorage $batch_storage */
332 $batch_storage = \Drupal::service('batch.storage');
333 $batch_storage->delete($batch['id']);
337 ->condition('bid', $batch['id'])
341 foreach ($batch['sets'] as $batch_set) {
342 if ($queue = _batch_queue($batch_set)) {
343 $queue->deleteQueue();
348 drush_set_option('drush_batch_process_finished', TRUE);
354 * Shutdown function: store the batch data for next request,
355 * or clear the table if the batch is finished.
357 function _drush_batch_shutdown() {
358 if ($batch = batch_get()) {
359 if (drush_drupal_major_version() >= 8) {
360 /** @var \Drupal\Core\Batch\BatchStorage $batch_storage */
361 $batch_storage = \Drupal::service('batch.storage');
362 $batch_storage->update($batch);
366 ->fields(['batch' => serialize($batch)])
367 ->condition('bid', $batch['id'])