diff --git a/src/js/media/controllers/featured-image.js b/src/js/media/controllers/featured-image.js
index 83c39f9936..e05b377be3 100644
--- a/src/js/media/controllers/featured-image.js
+++ b/src/js/media/controllers/featured-image.js
@@ -117,7 +117,7 @@ FeaturedImage = Library.extend(/** @lends wp.media.controller.FeaturedImage.prot
 
 		selection.reset( attachment ? [ attachment ] : [] );
 
-		if ( library.hasMore() ) {
+		if ( ! infinite_scrolling && library.hasMore() ) {
 			library.more();
 		}
 	}
diff --git a/src/js/media/controllers/replace-image.js b/src/js/media/controllers/replace-image.js
index d042ec3c07..858e84c2d5 100644
--- a/src/js/media/controllers/replace-image.js
+++ b/src/js/media/controllers/replace-image.js
@@ -104,7 +104,7 @@ ReplaceImage = Library.extend(/** @lends wp.media.controller.ReplaceImage.protot
 
 		selection.reset( attachment ? [ attachment ] : [] );
 
-		if ( library.hasMore() ) {
+		if ( ! infinite_scrolling && library.getTotalAttachments() == 0 && library.hasMore() ) {
 			library.more();
 		}
 	}