diff --git a/plexpy/users.py b/plexpy/users.py
index 974d30cd..67293800 100644
--- a/plexpy/users.py
+++ b/plexpy/users.py
@@ -63,13 +63,15 @@ def refresh_users():
         new_users = []
 
         for item in result:
-            user_ids.append(helpers.cast_to_int(item['user_id']))
-
             if item.get('shared_libraries'):
                 item['shared_libraries'] = ';'.join(item['shared_libraries'])
+                # Only add user if libraries are shared
+                user_ids.add(helpers.cast_to_int(item['user_id']))
             elif item.get('server_token'):
                 libs = libraries.Libraries().get_sections()
                 item['shared_libraries'] = ';'.join([str(l['section_id']) for l in libs])
+                # Only add user if libraries are shared
+                user_ids.add(helpers.cast_to_int(item['user_id']))
 
             keys_dict = {"user_id": item.pop('user_id')}
 
@@ -101,7 +103,7 @@ def refresh_users():
         query = "UPDATE users SET is_active = 0 WHERE user_id NOT IN ({})".format(", ".join(["?"] * len(user_ids)))
         monitor_db.action(query=query, args=user_ids)
 
-        # Add new users to loger username filter
+        # Add new users to logger username filter
         logger.filter_usernames(new_users)
 
         logger.info("Tautulli Users :: Users list refreshed.")