diff --git a/src/lib.rs b/src/lib.rs
index 2ba9d86..015ad29 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -310,8 +310,11 @@ fn accept_packet(
             let server = &mut *server;
             let id = u.get_session();
 
-            let state = server.users.entry(id).or_default();
-            if u.has_channel_id() {
+            let state_entry = server.users.entry(id);
+            let new = matches!(state_entry, std::collections::hash_map::Entry::Vacant(_));
+            let state = state_entry.or_default();
+            // the server might now send a channel_id if the user is in channel=0
+            if u.has_channel_id() || new {
                 if let Some(parent) = server.channels.get_mut(&state.channel) {
                     parent.users.remove(&id);
                 }