diff --git a/src/hooks/useSortBy.js b/src/hooks/useSortBy.js
index 4e0f905..90a66b1 100755
--- a/src/hooks/useSortBy.js
+++ b/src/hooks/useSortBy.js
@@ -82,7 +82,8 @@ export const useSortBy = props => {
 
       if (!multi) {
         if (sortBy.length <= 1 && existingSortBy) {
-          if (existingSortBy.desc) {
+          if ((existingSortBy.desc && !resolvedDefaultSortDesc) ||
+            (!existingSortBy.desc && resolvedDefaultSortDesc)) {
             action = 'remove'
           } else {
             action = 'toggle'