From 5a75531f19834073e8ebaf143f25a13fdf5817d8 Mon Sep 17 00:00:00 2001 From: Scott Taylor Date: Tue, 10 Sep 2013 03:17:51 +0000 Subject: [PATCH] Replace the ancient `phpfreaks.com` RegEx to extract urls to ping with a more robust matcher. URLs with commas and things like `&` were not being pinged. The new matcher even works for most IDN URLs. Adds unit tests. Fixes #9064. git-svn-id: https://develop.svn.wordpress.org/trunk@25313 602fd350-edb4-49c9-b593-d223f7449a82 --- src/wp-includes/comment.php | 12 +-- src/wp-includes/functions.php | 31 ++++-- tests/phpunit/tests/functions.php | 167 +++++++++++++++++++++++++++++- 3 files changed, 187 insertions(+), 23 deletions(-) diff --git a/src/wp-includes/comment.php b/src/wp-includes/comment.php index 10bf57d9b0..cc772ea719 100644 --- a/src/wp-includes/comment.php +++ b/src/wp-includes/comment.php @@ -1827,17 +1827,9 @@ function pingback($content, $post_ID) { $pung = get_pung($post_ID); - // Variables - $ltrs = '\w'; - $gunk = '/#~:.?+=&%@!\-'; - $punc = '.:?\-'; - $any = $ltrs . $gunk . $punc; - // Step 1 // Parsing the post, external links (if any) are stored in the $post_links array - // This regexp comes straight from phpfreaks.com - // http://www.phpfreaks.com/quickcode/Extract_All_URLs_on_a_Page/15.php - preg_match_all("{\b http : [$any] +? (?= [$punc] * [^$any] | $)}x", $content, $post_links_temp); + $post_links_temp = wp_extract_urls( $content ); // Step 2. // Walking thru the links array @@ -1848,7 +1840,7 @@ function pingback($content, $post_ID) { // http://dummy-weblog.org/post.php // We don't wanna ping first and second types, even if they have a valid - foreach ( (array) $post_links_temp[0] as $link_test ) : + foreach ( (array) $post_links_temp as $link_test ) : if ( !in_array($link_test, $pung) && (url_to_postid($link_test) != $post_ID) // If we haven't pung it already and it isn't a link to itself && !is_local_attachment($link_test) ) : // Also, let's never ping local attachments. if ( $test = @parse_url($link_test) ) { diff --git a/src/wp-includes/functions.php b/src/wp-includes/functions.php index 73b9de8d5f..966cf811d7 100644 --- a/src/wp-includes/functions.php +++ b/src/wp-includes/functions.php @@ -392,6 +392,26 @@ function xmlrpc_removepostdata( $content ) { return $content; } +/** + * Use RegEx to extract URLs from arbitrary content + * + * @since 3.7.0 + * + * @param string $content + * @return array URLs found in passed string + */ +function wp_extract_urls( $content ) { + preg_match_all( + "#((?:[\w-]+://?|[\w\d]+[.])[^\s()<>]+[.](?:\([\w\d]+\)|(?:[^`!()\[\]{};:'\".,<>?«»“”‘’\s]|(?:[:]\d+)?/?)+))#", + $content, + $post_links + ); + + $post_links = array_unique( array_map( 'html_entity_decode', $post_links[0] ) ); + + return array_values( $post_links ); +} + /** * Check content for video and audio links to add as enclosures. * @@ -417,22 +437,17 @@ function do_enclose( $content, $post_ID ) { $pung = get_enclosed( $post_ID ); - $ltrs = '\w'; - $gunk = '/#~:.?+=&%@!\-'; - $punc = '.:?\-'; - $any = $ltrs . $gunk . $punc; - - preg_match_all( "{\b https? : [$any] +? (?= [$punc] * [^$any] | $)}x", $content, $post_links_temp ); + $post_links_temp = wp_extract_urls( $content ); foreach ( $pung as $link_test ) { - if ( !in_array( $link_test, $post_links_temp[0] ) ) { // link no longer in post + if ( ! in_array( $link_test, $post_links_temp ) ) { // link no longer in post $mids = $wpdb->get_col( $wpdb->prepare("SELECT meta_id FROM $wpdb->postmeta WHERE post_id = %d AND meta_key = 'enclosure' AND meta_value LIKE (%s)", $post_ID, like_escape( $link_test ) . '%') ); foreach ( $mids as $mid ) delete_metadata_by_mid( 'post', $mid ); } } - foreach ( (array) $post_links_temp[0] as $link_test ) { + foreach ( (array) $post_links_temp as $link_test ) { if ( !in_array( $link_test, $pung ) ) { // If we haven't pung it already $test = @parse_url( $link_test ); if ( false === $test ) diff --git a/tests/phpunit/tests/functions.php b/tests/phpunit/tests/functions.php index 8bf892fed4..a223cf439a 100644 --- a/tests/phpunit/tests/functions.php +++ b/tests/phpunit/tests/functions.php @@ -99,9 +99,9 @@ class Tests_Functions extends WP_UnitTestCase { } function test_wp_unique_filename() { - + $testdir = DIR_TESTDATA . '/images/'; - + // sanity check $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcdefg.png' ), 'Sanitiy check failed' ); @@ -112,10 +112,10 @@ class Tests_Functions extends WP_UnitTestCase { // check special chars $this->assertEquals( 'testtést-imagé.png', wp_unique_filename( $testdir, 'testtést-imagé.png' ), 'Filename with special chars failed' ); - + // check special chars with potential conflicting name $this->assertEquals( 'tést-imagé.png', wp_unique_filename( $testdir, 'tést-imagé.png' ), 'Filename with special chars failed' ); - + // check with single quotes in name (somehow) $this->assertEquals( "abcdefgh.png", wp_unique_filename( $testdir, "abcdefg'h.png" ), 'File with quote failed' ); @@ -124,7 +124,7 @@ class Tests_Functions extends WP_UnitTestCase { // test crazy name (useful for regression tests) $this->assertEquals( '12%af34567890@..%^_+qwerty-fghjkl-zx.png', wp_unique_filename( $testdir, '12%af34567890#~!@#$..%^&*()|_+qwerty fgh`jkl zx<>?:"{}[]="\'/?.png' ), 'Failed crazy file name' ); - + // test slashes in names $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcde\fg.png' ), 'Slash not removed' ); $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcde\\fg.png' ), 'Double slashed not removed' ); @@ -362,4 +362,161 @@ class Tests_Functions extends WP_UnitTestCase { function test_data_is_not_an_allowed_protocol() { $this->assertNotContains( 'data', wp_allowed_protocols() ); } + + /** + * @ticket 9064 + */ + function test_wp_extract_urls() { + $original_urls = array( + 'http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html', + 'http://this.com', + 'http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437', + 'http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html', + 'http://wordpress-core.com:8080/', + 'http://www.website.com:5000', + 'http://wordpress-core/?346236346326&2134362574863.437', + 'http://افغانستا.icom.museum', + 'http://الجزائر.icom.museum', + 'http://österreich.icom.museum', + 'http://বাংলাদেশ.icom.museum', + 'http://беларусь.icom.museum', + 'http://belgië.icom.museum', + 'http://българия.icom.museum', + 'http://تشادر.icom.museum', + 'http://中国.icom.museum', + #'http://القمر.icom.museum', // Comoros http://القمر.icom.museum + #'http://κυπρος.icom.museum', Cyprus http://κυπρος.icom.museum + 'http://českárepublika.icom.museum', + #'http://مصر.icom.museum', // Egypt http://مصر.icom.museum + 'http://ελλάδα.icom.museum', + 'http://magyarország.icom.museum', + 'http://ísland.icom.museum', + 'http://भारत.icom.museum', + 'http://ايران.icom.museum', + 'http://éire.icom.museum', + 'http://איקו״ם.ישראל.museum', + 'http://日本.icom.museum', + 'http://الأردن.icom.museum', + 'http://қазақстан.icom.museum', + 'http://한국.icom.museum', + 'http://кыргызстан.icom.museum', + 'http://ລາວ.icom.museum', + 'http://لبنان.icom.museum', + 'http://македонија.icom.museum', + #'http://موريتانيا.icom.museum', // Mauritania http://موريتانيا.icom.museum + 'http://méxico.icom.museum', + 'http://монголулс.icom.museum', + #'http://المغرب.icom.museum', // Morocco http://المغرب.icom.museum + 'http://नेपाल.icom.museum', + #'http://عمان.icom.museum', // Oman http://عمان.icom.museum + 'http://قطر.icom.museum', + 'http://românia.icom.museum', + 'http://россия.иком.museum', + 'http://србијаицрнагора.иком.museum', + 'http://இலங்கை.icom.museum', + 'http://españa.icom.museum', + 'http://ไทย.icom.museum', + 'http://تونس.icom.museum', + 'http://türkiye.icom.museum', + 'http://украина.icom.museum', + 'http://việtnam.icom.museum' + ); + + $blob =" + http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html + + http://this.com + + http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437 + + http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html + + http://wordpress-core.com:8080/ + + http://www.website.com:5000 + + http://wordpress-core/?346236346326&2134362574863.437 + + http://افغانستا.icom.museum + http://الجزائر.icom.museum + http://österreich.icom.museum + http://বাংলাদেশ.icom.museum + http://беларусь.icom.museum + http://belgië.icom.museum + http://българия.icom.museum + http://تشادر.icom.museum + http://中国.icom.museum + http://českárepublika.icom.museum + http://ελλάδα.icom.museum + http://magyarország.icom.museum + http://ísland.icom.museum + http://भारत.icom.museum + http://ايران.icom.museum + http://éire.icom.museum + http://איקו״ם.ישראל.museum + http://日本.icom.museum + http://الأردن.icom.museum + http://қазақстан.icom.museum + http://한국.icom.museum + http://кыргызстан.icom.museum + http://ລາວ.icom.museum + http://لبنان.icom.museum + http://македонија.icom.museum + http://méxico.icom.museum + http://монголулс.icom.museum + http://नेपाल.icom.museum + http://قطر.icom.museum + http://românia.icom.museum + http://россия.иком.museum + http://србијаицрнагора.иком.museum + http://இலங்கை.icom.museum + http://españa.icom.museum + http://ไทย.icom.museum + http://تونس.icom.museum + http://türkiye.icom.museum + http://украина.icom.museum + http://việtnam.icom.museum + "; + + $urls = wp_extract_urls( $blob ); + $this->assertNotEmpty( $urls ); + $this->assertInternalType( 'array', $urls ); + $this->assertCount( count( $original_urls ), $urls ); + $this->assertEquals( $original_urls, $urls ); + + $exploded = array_values( array_filter( array_map( 'trim', explode( "\n", $blob ) ) ) ); + // wp_extract_urls calls html_entity_decode + $decoded = array_map( 'html_entity_decode', $exploded ); + + $this->assertEquals( $decoded, $urls ); + $this->assertEquals( $original_urls, $decoded ); + + $blob ="Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor + incididunt ut labore http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html et dolore magna aliqua. + Ut http://this.com enim ad minim veniam, quis nostrud exercitation ullamco + laboris nisi ut aliquip ex http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437 ea + commodo consequat. http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html Duis aute irure dolor in reprehenderit in voluptate + velit esse http://wordpress-core.com:8080/ cillum dolore eu fugiat nulla http://www.website.com:5000 pariatur. Excepteur sint occaecat cupidatat non proident, + sunt in culpa qui officia deserunt mollit http://wordpress-core/?346236346326&2134362574863.437 anim id est laborum."; + + $urls = wp_extract_urls( $blob ); + $this->assertNotEmpty( $urls ); + $this->assertInternalType( 'array', $urls ); + $this->assertCount( 7, $urls ); + $this->assertEquals( array_slice( $original_urls, 0, 7 ), $urls ); + + $blob = 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor + incididunt ut labore 343462^ et dolore magna aliqua. + Ut &3640i6p1yi499 enim ad minim veniam, quis nostrud exercitation ullamco + laboris nisi ut aliquip ex 343462^ ea + commodo consequat. 343462^ Duis aute irure dolor in reprehenderit in voluptate + velit esse -3-4--321-64-4@#!$^$!@^@^ cillum dolore eu http://www.website.com:5000 fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, + sunt in culpa qui officia deserunt mollit )(*&^%$ anim id est laborum.'; + + $urls = wp_extract_urls( $blob ); + $this->assertNotEmpty( $urls ); + $this->assertInternalType( 'array', $urls ); + $this->assertCount( 7, $urls ); + $this->assertEquals( array_slice( $original_urls, 0, 7 ), $urls ); + } }