Skip to content

Commit 7ffc573

Browse files
schlesseraschlessera
authored
Merge pull request #162 from wp-cli/fix/chunking
Fix offset handling when doing chunked replaces
2 parents 08ba95f + bb6335b commit 7ffc573

File tree

2 files changed

+101
-44
lines changed

2 files changed

+101
-44
lines changed

‎features/search-replace.feature‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1142,3 +1142,45 @@ Feature: Do global search/replace
11421142
"""
11431143
Success:
11441144
"""
1145+
1146+
Scenario: Chunking works without skipping lines
1147+
Given a WP install
1148+
And a create_sql_file.sh file:
1149+
"""
1150+
#!/bin/bash
1151+
echo "CREATE TABLE \`wp_123_test\` (\`key\` INT(5) UNSIGNED NOT NULL AUTO_INCREMENT, \`text\` TEXT, PRIMARY KEY (\`key\`) );" > test_db.sql
1152+
echo "INSERT INTO \`wp_123_test\` (\`text\`) VALUES" >> test_db.sql
1153+
index=1
1154+
while [[ $index -le 199 ]];
1155+
do
1156+
echo "('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc')," >> test_db.sql
1157+
index=`expr $index + 1`
1158+
done
1159+
echo "('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc'),('abc');" >> test_db.sql
1160+
"""
1161+
And I run `bash create_sql_file.sh`
1162+
And I run `wp db query "SOURCE test_db.sql;"`
1163+
1164+
When I run `wp search-replace --dry-run 'abc' 'def' --all-tables-with-prefix --skip-columns=guid,domain --precise`
1165+
Then STDOUT should contain:
1166+
"""
1167+
Success: 2000 replacements to be made.
1168+
"""
1169+
1170+
When I run `wp search-replace 'abc' 'def' --all-tables-with-prefix --skip-columns=guid,domain --precise`
1171+
Then STDOUT should contain:
1172+
"""
1173+
Success: Made 2000 replacements.
1174+
"""
1175+
1176+
When I run `wp search-replace --dry-run 'abc' 'def' --all-tables-with-prefix --skip-columns=guid,domain --precise`
1177+
Then STDOUT should contain:
1178+
"""
1179+
Success: 0 replacements to be made.
1180+
"""
1181+
1182+
When I run `wp search-replace 'abc' 'def' --all-tables-with-prefix --skip-columns=guid,domain --precise`
1183+
Then STDOUT should contain:
1184+
"""
1185+
Success: Made 0 replacements.
1186+
"""

‎src/Search_Replace_Command.php‎

Lines changed: 59 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
<?php
22

3+
use cli\Colors;
4+
use cli\Table;
5+
use WP_CLI\Iterators;
6+
use WP_CLI\SearchReplacer;
7+
use WP_CLI\Utils;
8+
use function cli\safe_substr;
9+
310
class Search_Replace_Command extends WP_CLI_Command {
411

512
private $dry_run;
@@ -15,8 +22,9 @@ class Search_Replace_Command extends WP_CLI_Command {
1522
private $include_columns;
1623
private $format;
1724
private $report;
18-
private $report_changed_only;
25+
private $verbose;
1926

27+
private $report_changed_only;
2028
private $log_handle = null;
2129
private $log_before_context = 40;
2230
private $log_after_context = 40;
@@ -167,24 +175,24 @@ public function __invoke( $args, $assoc_args ) {
167175
$new = array_shift( $args );
168176
$total = 0;
169177
$report = array();
170-
$this->dry_run = \WP_CLI\Utils\get_flag_value( $assoc_args, 'dry-run' );
171-
$php_only = \WP_CLI\Utils\get_flag_value( $assoc_args, 'precise' );
172-
$this->recurse_objects = \WP_CLI\Utils\get_flag_value( $assoc_args, 'recurse-objects', true );
173-
$this->verbose = \WP_CLI\Utils\get_flag_value( $assoc_args, 'verbose' );
174-
$this->format = \WP_CLI\Utils\get_flag_value( $assoc_args, 'format' );
175-
$this->regex = \WP_CLI\Utils\get_flag_value( $assoc_args, 'regex', false );
178+
$this->dry_run = Utils\get_flag_value( $assoc_args, 'dry-run' );
179+
$php_only = Utils\get_flag_value( $assoc_args, 'precise' );
180+
$this->recurse_objects = Utils\get_flag_value( $assoc_args, 'recurse-objects', true );
181+
$this->verbose = Utils\get_flag_value( $assoc_args, 'verbose' );
182+
$this->format = Utils\get_flag_value( $assoc_args, 'format' );
183+
$this->regex = Utils\get_flag_value( $assoc_args, 'regex', false );
176184

177185
if ( null !== $this->regex ) {
178186
$default_regex_delimiter = false;
179-
$this->regex_flags = \WP_CLI\Utils\get_flag_value( $assoc_args, 'regex-flags', false );
180-
$this->regex_delimiter = \WP_CLI\Utils\get_flag_value( $assoc_args, 'regex-delimiter', '' );
187+
$this->regex_flags = Utils\get_flag_value( $assoc_args, 'regex-flags', false );
188+
$this->regex_delimiter = Utils\get_flag_value( $assoc_args, 'regex-delimiter', '' );
181189
if ( '' === $this->regex_delimiter ) {
182190
$this->regex_delimiter = chr( 1 );
183191
$default_regex_delimiter = true;
184192
}
185193
}
186194

187-
$regex_limit = \WP_CLI\Utils\get_flag_value( $assoc_args, 'regex-limit' );
195+
$regex_limit = Utils\get_flag_value( $assoc_args, 'regex-limit' );
188196
if ( null !== $regex_limit ) {
189197
if ( ! preg_match( '/^(?:[0-9]+|-1)$/', $regex_limit ) || 0 === (int) $regex_limit ) {
190198
WP_CLI::error( '`--regex-limit` expects a non-zero positive integer or -1.' );
@@ -215,16 +223,16 @@ public function __invoke( $args, $assoc_args ) {
215223
}
216224
}
217225

218-
$this->skip_columns = explode( ',', \WP_CLI\Utils\get_flag_value( $assoc_args, 'skip-columns' ) );
219-
$this->skip_tables = explode( ',', \WP_CLI\Utils\get_flag_value( $assoc_args, 'skip-tables' ) );
220-
$this->include_columns = array_filter( explode( ',', \WP_CLI\Utils\get_flag_value( $assoc_args, 'include-columns' ) ) );
226+
$this->skip_columns = explode( ',', Utils\get_flag_value( $assoc_args, 'skip-columns' ) );
227+
$this->skip_tables = explode( ',', Utils\get_flag_value( $assoc_args, 'skip-tables' ) );
228+
$this->include_columns = array_filter( explode( ',', Utils\get_flag_value( $assoc_args, 'include-columns' ) ) );
221229

222230
if ( $old === $new && ! $this->regex ) {
223231
WP_CLI::warning( "Replacement value '{$old}' is identical to search value '{$new}'. Skipping operation." );
224232
exit;
225233
}
226234

227-
$export = \WP_CLI\Utils\get_flag_value( $assoc_args, 'export' );
235+
$export = Utils\get_flag_value( $assoc_args, 'export' );
228236
if ( null !== $export ) {
229237
if ( $this->dry_run ) {
230238
WP_CLI::error( 'You cannot supply --dry-run and --export at the same time.' );
@@ -239,15 +247,15 @@ public function __invoke( $args, $assoc_args ) {
239247
WP_CLI::error( sprintf( 'Unable to open export file "%s" for writing: %s.', $assoc_args['export'], $error['message'] ) );
240248
}
241249
}
242-
$export_insert_size = WP_CLI\Utils\get_flag_value( $assoc_args, 'export_insert_size', 50 );
250+
$export_insert_size = Utils\get_flag_value( $assoc_args, 'export_insert_size', 50 );
243251
// phpcs:ignore WordPress.PHP.StrictComparisons.LooseComparison -- See the code, this is deliberate.
244252
if ( (int) $export_insert_size == $export_insert_size && $export_insert_size > 0 ) {
245253
$this->export_insert_size = $export_insert_size;
246254
}
247255
$php_only = true;
248256
}
249257

250-
$log = \WP_CLI\Utils\get_flag_value( $assoc_args, 'log' );
258+
$log = Utils\get_flag_value( $assoc_args, 'log' );
251259
if ( null !== $log ) {
252260
if ( true === $log || '-' === $log ) {
253261
$this->log_handle = STDOUT;
@@ -259,12 +267,12 @@ public function __invoke( $args, $assoc_args ) {
259267
}
260268
}
261269
if ( $this->log_handle ) {
262-
$before_context = \WP_CLI\Utils\get_flag_value( $assoc_args, 'before_context' );
270+
$before_context = Utils\get_flag_value( $assoc_args, 'before_context' );
263271
if ( null !== $before_context && preg_match( '/^[0-9]+$/', $before_context ) ) {
264272
$this->log_before_context = (int) $before_context;
265273
}
266274

267-
$after_context = \WP_CLI\Utils\get_flag_value( $assoc_args, 'after_context' );
275+
$after_context = Utils\get_flag_value( $assoc_args, 'after_context' );
268276
if ( null !== $after_context && preg_match( '/^[0-9]+$/', $after_context ) ) {
269277
$this->log_after_context = (int) $after_context;
270278
}
@@ -297,14 +305,14 @@ public function __invoke( $args, $assoc_args ) {
297305
);
298306
}
299307

300-
$this->log_colors = self::get_colors( $assoc_args, $default_log_colors );
308+
$this->log_colors = $this->get_colors( $assoc_args, $default_log_colors );
301309
$this->log_encoding = 0 === strpos( $wpdb->charset, 'utf8' ) ? 'UTF-8' : false;
302310
}
303311
}
304312

305-
$this->report = \WP_CLI\Utils\get_flag_value( $assoc_args, 'report', true );
313+
$this->report = Utils\get_flag_value( $assoc_args, 'report', true );
306314
// Defaults to true if logging, else defaults to false.
307-
$this->report_changed_only = \WP_CLI\Utils\get_flag_value( $assoc_args, 'report-changed-only', null !== $this->log_handle );
315+
$this->report_changed_only = Utils\get_flag_value( $assoc_args, 'report-changed-only', null !== $this->log_handle );
308316

309317
if ( $this->regex_flags ) {
310318
$php_only = true;
@@ -314,7 +322,7 @@ public function __invoke( $args, $assoc_args ) {
314322
$this->skip_columns[] = 'user_pass';
315323

316324
// Get table names based on leftover $args or supplied $assoc_args
317-
$tables = \WP_CLI\Utils\wp_get_table_names( $args, $assoc_args );
325+
$tables = Utils\wp_get_table_names( $args, $assoc_args );
318326

319327
foreach ( $tables as $table ) {
320328

@@ -418,7 +426,7 @@ public function __invoke( $args, $assoc_args ) {
418426
}
419427

420428
if ( $this->report && ! empty( $report ) ) {
421-
$table = new \cli\Table();
429+
$table = new Table();
422430
$table->setHeaders( array( 'Table', 'Column', 'Replacements', 'Type' ) );
423431
$table->setRows( $report );
424432
$table->display();
@@ -429,7 +437,7 @@ public function __invoke( $args, $assoc_args ) {
429437
$success_message = 1 === $total ? "Made 1 replacement and exported to {$assoc_args['export']}." : "Made {$total} replacements and exported to {$assoc_args['export']}.";
430438
} else {
431439
$success_message = 1 === $total ? 'Made 1 replacement.' : "Made $total replacements.";
432-
if ( $total && 'Default' !== WP_CLI\Utils\wp_get_cache_type() ) {
440+
if ( $total && 'Default' !== Utils\wp_get_cache_type() ) {
433441
$success_message .= ' Please remember to flush your persistent object cache with `wp cache flush`.';
434442
}
435443
}
@@ -450,15 +458,15 @@ private function php_export_table( $table, $old, $new ) {
450458
'chunk_size' => $chunk_size,
451459
);
452460

453-
$replacer = new \WP_CLI\SearchReplacer( $old, $new, $this->recurse_objects, $this->regex, $this->regex_flags, $this->regex_delimiter, false, $this->regex_limit );
461+
$replacer = new SearchReplacer( $old, $new, $this->recurse_objects, $this->regex, $this->regex_flags, $this->regex_delimiter, false, $this->regex_limit );
454462
$col_counts = array_fill_keys( $all_columns, 0 );
455463
if ( $this->verbose && 'table' === $this->format ) {
456464
$this->start_time = microtime( true );
457465
WP_CLI::log( sprintf( 'Checking: %s', $table ) );
458466
}
459467

460468
$rows = array();
461-
foreach ( new \WP_CLI\Iterators\Table( $args ) as $i => $row ) {
469+
foreach ( new Iterators\Table( $args ) as $i => $row ) {
462470
$row_fields = array();
463471
foreach ( $all_columns as $col ) {
464472
$value = $row->$col;
@@ -527,15 +535,15 @@ private function php_handle_col( $col, $primary_keys, $table, $old, $new ) {
527535
global $wpdb;
528536

529537
$count = 0;
530-
$replacer = new \WP_CLI\SearchReplacer( $old, $new, $this->recurse_objects, $this->regex, $this->regex_flags, $this->regex_delimiter, null !== $this->log_handle, $this->regex_limit );
538+
$replacer = new SearchReplacer( $old, $new, $this->recurse_objects, $this->regex, $this->regex_flags, $this->regex_delimiter, null !== $this->log_handle, $this->regex_limit );
531539

532540
$table_sql = self::esc_sql_ident( $table );
533541
$col_sql = self::esc_sql_ident( $col );
534542
$where = $this->regex ? '' : " WHERE $col_sql" . $wpdb->prepare( ' LIKE BINARY %s', '%' . self::esc_like( $old ) . '%' );
535543
$escaped_primary_keys = self::esc_sql_ident( $primary_keys );
536544
$primary_keys_sql = implode( ',', $escaped_primary_keys );
537545
$order_by_keys = array_map(
538-
function( $key ) {
546+
static function ( $key ) {
539547
return "{$key} ASC";
540548
},
541549
$escaped_primary_keys
@@ -544,6 +552,10 @@ function( $key ) {
544552
$limit = 1000;
545553
$offset = 0;
546554

555+
// Updates have to be deferred to after the chunking is completed, as
556+
// the offset will otherwise not work correctly.
557+
$updates = [];
558+
547559
// 2 errors:
548560
// - WordPress.DB.PreparedSQL.InterpolatedNotPrepared -- escaped through self::esc_sql_ident
549561
// - WordPress.CodeAnalysis.AssignmentInCondition -- no reason to do copy-paste for a single valid assignment in while
@@ -552,7 +564,7 @@ function( $key ) {
552564
foreach ( $rows as $keys ) {
553565
$where_sql = '';
554566
foreach ( (array) $keys as $k => $v ) {
555-
if ( strlen( $where_sql ) ) {
567+
if ( '' !== $where_sql ) {
556568
$where_sql .= ' AND ';
557569
}
558570
$where_sql .= self::esc_sql_ident( $k ) . ' = ' . self::esc_sql_value( $v );
@@ -576,21 +588,24 @@ function( $key ) {
576588
$replacer->clear_log_data();
577589
}
578590

579-
if ( $this->dry_run ) {
580-
$count++;
581-
} else {
591+
$count++;
592+
if ( ! $this->dry_run ) {
582593
$update_where = array();
583594
foreach ( (array) $keys as $k => $v ) {
584595
$update_where[ $k ] = $v;
585596
}
586597

587-
$count += $wpdb->update( $table, array( $col => $value ), $update_where );
598+
$updates[] = [ $table, array( $col => $value ), $update_where ];
588599
}
589600
}
590601

591602
$offset += $limit;
592603
}
593604

605+
foreach ( $updates as $update ) {
606+
$wpdb->update( ...$update );
607+
}
608+
594609
if ( $this->verbose && 'table' === $this->format ) {
595610
$time = round( microtime( true ) - $this->start_time, 3 );
596611
WP_CLI::log( sprintf( '%d rows affected using PHP (in %ss).', $count, $time ) );
@@ -728,7 +743,7 @@ private static function esc_like( $old ) {
728743
* @return string|array An escaped string if given a string, or an array of escaped strings if given an array of strings.
729744
*/
730745
private static function esc_sql_ident( $idents ) {
731-
$backtick = function ( $v ) {
746+
$backtick = static function ( $v ) {
732747
// Escape any backticks in the identifier by doubling.
733748
return '`' . str_replace( '`', '``', $v ) . '`';
734749
};
@@ -745,7 +760,7 @@ private static function esc_sql_ident( $idents ) {
745760
* @return string|array A quoted string if given a string, or an array of quoted strings if given an array of strings.
746761
*/
747762
private static function esc_sql_value( $values ) {
748-
$quote = function ( $v ) {
763+
$quote = static function ( $v ) {
749764
// Don't quote integer values to avoid MySQL's implicit type conversion.
750765
if ( preg_match( '/^[+-]?[0-9]{1,20}$/', $v ) ) { // MySQL BIGINT UNSIGNED max 18446744073709551615 (20 digits).
751766
return esc_sql( $v );
@@ -772,18 +787,18 @@ private static function esc_sql_value( $values ) {
772787
private function get_colors( $assoc_args, $colors ) {
773788
$color_reset = WP_CLI::colorize( '%n' );
774789

775-
$color_code_callback = function ( $v ) {
790+
$color_code_callback = static function ( $v ) {
776791
return substr( $v, 1 );
777792
};
778793

779-
$color_codes = array_keys( \cli\Colors::getColors() );
794+
$color_codes = array_keys( Colors::getColors() );
780795
$color_codes = array_map( $color_code_callback, $color_codes );
781796
$color_codes = implode( '', $color_codes );
782797

783798
$color_codes_regex = '/^(?:%[' . $color_codes . '])*$/';
784799

785800
foreach ( array_keys( $colors ) as $color_col ) {
786-
$col_color_flag = \WP_CLI\Utils\get_flag_value( $assoc_args, $color_col . '_color' );
801+
$col_color_flag = Utils\get_flag_value( $assoc_args, $color_col . '_color' );
787802
if ( null !== $col_color_flag ) {
788803
if ( ! preg_match( $color_codes_regex, $col_color_flag, $matches ) ) {
789804
WP_CLI::warning( "Unrecognized percent color code '$col_color_flag' for '{$color_col}_color'." );
@@ -891,12 +906,12 @@ private function log_bits( $search_regex, $old_data, $old_matches, $new ) {
891906
$new_matches = array();
892907
$new_data = preg_replace_callback(
893908
$search_regex,
894-
function ( $matches ) use ( $old_matches, $new, $is_regex, &$new_matches, &$i, &$diff ) {
909+
static function ( $matches ) use ( $old_matches, $new, $is_regex, &$new_matches, &$i, &$diff ) {
895910
if ( $is_regex ) {
896911
// Sub in any back references, "$1", "\2" etc, in the replacement string.
897912
$new = preg_replace_callback(
898913
'/(?<!\\\\)(?:\\\\\\\\)*((?:\\\\|\\$)[0-9]{1,2}|\\${[0-9]{1,2}\\})/',
899-
function ( $m ) use ( $matches ) {
914+
static function ( $m ) use ( $matches ) {
900915
$idx = (int) str_replace( array( '\\', '$', '{', '}' ), '', $m[0] );
901916
return isset( $matches[ $idx ] ) ? $matches[ $idx ] : '';
902917
},
@@ -939,14 +954,14 @@ function ( $m ) use ( $matches ) {
939954

940955
// Offsets are in bytes, so need to use `strlen()` and `substr()` before using `safe_substr()`.
941956
if ( $this->log_before_context && $old_offset && ! $append_next ) {
942-
$old_before = \cli\safe_substr( substr( $old_data, $last_old_offset, $old_offset - $last_old_offset ), -$this->log_before_context, null /*length*/, false /*is_width*/, $encoding );
943-
$new_before = \cli\safe_substr( substr( $new_data, $last_new_offset, $new_offset - $last_new_offset ), -$this->log_before_context, null /*length*/, false /*is_width*/, $encoding );
957+
$old_before = safe_substr( substr( $old_data, $last_old_offset, $old_offset - $last_old_offset ), -$this->log_before_context, null /*length*/, false /*is_width*/, $encoding );
958+
$new_before = safe_substr( substr( $new_data, $last_new_offset, $new_offset - $last_new_offset ), -$this->log_before_context, null /*length*/, false /*is_width*/, $encoding );
944959
}
945960
if ( $this->log_after_context ) {
946961
$old_end_offset = $old_offset + strlen( $old_match );
947962
$new_end_offset = $new_offset + strlen( $new_match );
948-
$old_after = \cli\safe_substr( substr( $old_data, $old_end_offset ), 0, $this->log_after_context, false /*is_width*/, $encoding );
949-
$new_after = \cli\safe_substr( substr( $new_data, $new_end_offset ), 0, $this->log_after_context, false /*is_width*/, $encoding );
963+
$old_after = safe_substr( substr( $old_data, $old_end_offset ), 0, $this->log_after_context, false /*is_width*/, $encoding );
964+
$new_after = safe_substr( substr( $new_data, $new_end_offset ), 0, $this->log_after_context, false /*is_width*/, $encoding );
950965
// To lessen context duplication in output, shorten the after context if it overlaps with the next match.
951966
if ( $i + 1 < $match_cnt && $old_end_offset + strlen( $old_after ) > $old_matches[0][ $i + 1 ][1] ) {
952967
$old_after = substr( $old_after, 0, $old_matches[0][ $i + 1 ][1] - $old_end_offset );

0 commit comments

Comments
 (0)