diff --git a/src/wp-includes/class-wp-user.php b/src/wp-includes/class-wp-user.php
index b2f8687828..c6c7ef4eaf 100644
--- a/src/wp-includes/class-wp-user.php
+++ b/src/wp-includes/class-wp-user.php
@@ -542,6 +542,10 @@ class WP_User {
 			return;
 		}
 
+		if ( in_array( $role, $this->roles, true ) ) {
+			return;
+		}
+
 		$this->caps[ $role ] = true;
 		update_user_meta( $this->ID, $this->cap_key, $this->caps );
 		$this->get_role_caps();
@@ -569,6 +573,7 @@ class WP_User {
 		if ( ! in_array( $role, $this->roles, true ) ) {
 			return;
 		}
+
 		unset( $this->caps[ $role ] );
 		update_user_meta( $this->ID, $this->cap_key, $this->caps );
 		$this->get_role_caps();
@@ -606,16 +611,32 @@ class WP_User {
 		}
 
 		$old_roles = $this->roles;
+
 		if ( ! empty( $role ) ) {
 			$this->caps[ $role ] = true;
 			$this->roles         = array( $role => true );
 		} else {
-			$this->roles = false;
+			$this->roles = array();
 		}
+
 		update_user_meta( $this->ID, $this->cap_key, $this->caps );
 		$this->get_role_caps();
 		$this->update_user_level_from_caps();
 
+		foreach ( $old_roles as $old_role ) {
+			if ( ! $old_role || $old_role === $role ) {
+				continue;
+			}
+
+			/** This action is documented in wp-includes/class-wp-user.php */
+			do_action( 'remove_user_role', $this->ID, $old_role );
+		}
+
+		if ( $role && ! in_array( $role, $old_roles, true ) ) {
+			/** This action is documented in wp-includes/class-wp-user.php */
+			do_action( 'add_user_role', $this->ID, $role );
+		}
+
 		/**
 		 * Fires after the user's role has changed.
 		 *
diff --git a/tests/phpunit/tests/user/capabilities.php b/tests/phpunit/tests/user/capabilities.php
index 496de9487a..5f838ac27d 100644
--- a/tests/phpunit/tests/user/capabilities.php
+++ b/tests/phpunit/tests/user/capabilities.php
@@ -1623,11 +1623,28 @@ class Tests_User_Capabilities extends WP_UnitTestCase {
 		$user = self::$users['administrator'];
 		$caps = $user->caps;
 		$this->assertNotEmpty( $user->caps );
+
 		$user->set_role( 'administrator' );
 		$this->assertNotEmpty( $user->caps );
 		$this->assertSame( $caps, $user->caps );
 	}
 
+	/**
+	 * @ticket 54164
+	 */
+	public function test_set_role_fires_remove_user_role_and_add_user_role_hooks() {
+		$user = self::$users['administrator'];
+
+		$remove_user_role = new MockAction();
+		$add_user_role    = new MockAction();
+		add_action( 'remove_user_role', array( $remove_user_role, 'action' ) );
+		add_action( 'add_user_role', array( $add_user_role, 'action' ) );
+
+		$user->set_role( 'editor' );
+		$this->assertSame( 1, $remove_user_role->get_call_count() );
+		$this->assertSame( 1, $add_user_role->get_call_count() );
+	}
+
 	public function test_current_user_can_for_blog() {
 		global $wpdb;