diff --git a/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/aad/implementation/AzureActiveDirectoryOAuth2UserService.java b/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/aad/implementation/AzureActiveDirectoryOAuth2UserService.java
index 826400c46695..8aa6810b5b1a 100644
--- a/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/aad/implementation/AzureActiveDirectoryOAuth2UserService.java
+++ b/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/aad/implementation/AzureActiveDirectoryOAuth2UserService.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static com.azure.spring.autoconfigure.aad.Constants.DEFAULT_AUTHORITY_SET;
 import static com.azure.spring.autoconfigure.aad.Constants.ROLE_PREFIX;
 
 /**
@@ -56,14 +57,12 @@ public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2Authenticatio
                                        .filter(properties::isAllowedGroup)
                                        .map(group -> ROLE_PREFIX + group)
                                        .collect(Collectors.toSet());
-        Set<String> allRoles = oidcUser.getAuthorities()
-                                       .stream()
-                                       .map(GrantedAuthority::getAuthority)
-                                       .collect(Collectors.toSet());
-        allRoles.addAll(groupRoles);
-        Set<SimpleGrantedAuthority> authorities = allRoles.stream()
-                                                          .map(SimpleGrantedAuthority::new)
-                                                          .collect(Collectors.toSet());
+        Set<SimpleGrantedAuthority> authorities = groupRoles.stream()
+                                                            .map(SimpleGrantedAuthority::new)
+                                                            .collect(Collectors.toSet());
+        if (authorities.isEmpty()) {
+            authorities = DEFAULT_AUTHORITY_SET;
+        }
         String nameAttributeKey =
             Optional.of(userRequest)
                     .map(OAuth2UserRequest::getClientRegistration)
diff --git a/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/autoconfigure/aad/AADOAuth2UserService.java b/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/autoconfigure/aad/AADOAuth2UserService.java
index d5548a5645b8..e6533724fef5 100644
--- a/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/autoconfigure/aad/AADOAuth2UserService.java
+++ b/sdk/spring/azure-spring-boot/src/main/java/com/azure/spring/autoconfigure/aad/AADOAuth2UserService.java
@@ -27,6 +27,7 @@
 import static com.azure.spring.autoconfigure.aad.AADOAuth2ErrorCode.CONDITIONAL_ACCESS_POLICY;
 import static com.azure.spring.autoconfigure.aad.AADOAuth2ErrorCode.INVALID_REQUEST;
 import static com.azure.spring.autoconfigure.aad.AADOAuth2ErrorCode.SERVER_SERVER;
+import static com.azure.spring.autoconfigure.aad.Constants.DEFAULT_AUTHORITY_SET;
 import static com.azure.spring.autoconfigure.aad.Constants.ROLE_PREFIX;
 
 /**
@@ -49,7 +50,7 @@ public AADOAuth2UserService(AADAuthenticationProperties aadAuthenticationPropert
     public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationException {
         // Delegate to the default implementation for loading a user
         OidcUser oidcUser = oidcUserService.loadUser(userRequest);
-        final Set<SimpleGrantedAuthority> authorities;
+        Set<SimpleGrantedAuthority> authorities;
         try {
             // https://github.com/MicrosoftDocs/azure-docs/issues/8121#issuecomment-387090099
             // In AAD App Registration configure oauth2AllowImplicitFlow to true
@@ -71,14 +72,12 @@ public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2Authenticatio
                                            .filter(aadAuthenticationProperties::isAllowedGroup)
                                            .map(group -> ROLE_PREFIX + group)
                                            .collect(Collectors.toSet());
-            Set<String> allRoles = oidcUser.getAuthorities()
-                                           .stream()
-                                           .map(GrantedAuthority::getAuthority)
-                                           .collect(Collectors.toSet());
-            allRoles.addAll(groupRoles);
-            authorities = allRoles.stream()
-                                  .map(SimpleGrantedAuthority::new)
-                                  .collect(Collectors.toSet());
+            authorities = groupRoles.stream()
+                                    .map(SimpleGrantedAuthority::new)
+                                    .collect(Collectors.toSet());
+            if (authorities.isEmpty()) {
+                authorities = DEFAULT_AUTHORITY_SET;
+            }
         } catch (MalformedURLException e) {
             throw toOAuth2AuthenticationException(INVALID_REQUEST, "Failed to acquire token for Graph API.", e);
         } catch (ServiceUnavailableException e) {