Jitsi breaks on enabling Lobby with JWT

Hi,

I am using Jitsi version stable-7439 with docker setup on custom kubernetes cluster.

I have enabled JWT + Lobby feature on jitsi.

Below is my prosody config file

admins = { "focus@auth.meet.jitsi", "jvb@auth.meet.jitsi" }

unlimited_jids = { "focus@auth.meet.jitsi", "jvb@auth.meet.jitsi"}

plugin_paths = { "/prosody-plugins/", "/prosody-plugins-custom" }

muc_mapper_domain_base = "meet.jitsi";
muc_mapper_domain_prefix = "muc";
http_default_host = "meet.jitsi"

asap_accepted_issuers = { "jitsi-authentication-adapter" }
asap_accepted_audiences = { "Jitsi Meet" }

consider_bosh_secure = true;
consider_websocket_secure = true;

VirtualHost "meet.jitsi"
  authentication = "token";
  app_id = "jitsi-internal-portal";
  app_secret = "<Secret>";
  allow_empty_token = false;

  -- enable_domain_verification = true;

  ssl = { key = "/config/certs/meet.jitsi.key"; certificate = "/config/certs/meet.jitsi.crt"; }

  modules_enabled = {
    "bosh";
    "websocket";
    "smacks"; -- XEP-0198: Stream Management
    "pubsub";
    "ping";
    "speakerstats";
    "conference_duration";
    "external_services";
    "muc_lobby_rooms";
    "av_moderation";
    "reservations";
    "presence_identity";
  }

  main_muc = "muc.meet.jitsi"
  lobby_muc = "lobby.meet.jitsi"
  speakerstats_component = "speakerstats.meet.jitsi"
  conference_duration_component = "conferenceduration.meet.jitsi"
  av_moderation_component = "avmoderation.meet.jitsi"
  c2s_require_encryption = false

  reservations_api_prefix = "https://<host>"
  reservations_enable_max_occupants = true
  reservations_api_timeout = 60000
  reservations_api_headers = {
    ["Authorization"] = "Basic <Token>";
  }

VirtualHost "auth.meet.jitsi"
  ssl = { key = "/config/certs/auth.meet.jitsi.key"; certificate = "/config/certs/auth.meet.jitsi.crt"; }
  modules_enabled = {
    "limits_exception";
  }
  authentication = "internal_hashed"

Component "internal-muc.meet.jitsi" "muc"
  modules_enabled = {
    "ping";
  }
  restrict_room_creation = true
  muc_room_locking = false
  muc_room_default_public_jids = true

Component "muc.meet.jitsi" "muc"
  modules_enabled = {
    "muc_meeting_id";
    "muc_domain_mapper";
    "muc_max_occupants";
    "token_verification";
    "token_affiliation";
    "polls";
    "muc_domain_mapper";
  }
  muc_room_cache_size = 1000
  muc_room_locking = false
  muc_room_default_public_jids = true
  muc_max_occupants = "5"
  muc_access_whitelist = { "focus@auth.meet.jitsi", "jvb@auth.meet.jitsi" }

Component "focus.meet.jitsi" "client_proxy"
  target_address = "focus@auth.meet.jitsi"

Component "speakerstats.meet.jitsi" "speakerstats_component"
  muc_component = "muc.meet.jitsi"

Component "conferenceduration.meet.jitsi" "conference_duration_component"
  muc_component = "muc.meet.jitsi"

Component "avmoderation.meet.jitsi" "av_moderation_component"
  muc_component = "muc.meet.jitsi"

Component "lobby.meet.jitsi" "muc"
  restrict_room_creation = true
  muc_room_locking = false
  muc_room_default_public_jids = true
  storage = "memory"
  modules_enabled = {
    "muc_rate_limit";
  }

Below is my jicofo config file

jicofo {
    
    // Configuration related to jitsi-videobridge
    bridge {
      
      max-bridge-participants = "5"
    
      brewery-jid = "jvbbrewery@internal-muc.meet.jitsi"
    }
    // Configure the codecs and RTP extensions to be used in the offer sent to clients.
    codec {
      video {
        
        vp8 {
          enabled = "true"
        }
        
        vp9 {
          enabled = "true"
        }
        
        h264 {
          enabled = "false"
        }
        
      }
    }

    conference {
      
      enable-auto-owner = false
      
    }

    octo {
      // Whether or not to use Octo. Note that when enabled, its use will be determined by
      // $jicofo.bridge.selection-strategy. There's a corresponding flag in the JVB and these
      // two MUST be in sync (otherwise bridges will crash because they won't know how to
      // deal with octo channels).
      enabled = false

      id = "1"
    }

    sctp {
      enabled = false
    }

    xmpp {
      client {
        enabled = true
        hostname = "shard-1-prosody.jitsi.svc"
        port = "5222"
        domain = "auth.meet.jitsi"
        username = "focus"
        password = "<hidden>"
        conference-muc-jid = "muc.meet.jitsi"
        client-proxy = "focus.meet.jitsi"
        disable-certificate-verification = true
      }
    }
}

Prosody boots up fine

Jicofo boots with exception

Jicofo 2023-03-03 08:02:53.130 SEVERE: [15] [xmpp_connection=client] XmppProviderImpl.doConnect#228: Failed to connect/login: The following addresses failed: 'RFC 6120 A/AAAA Endpoint + [shard-1-prosody.jitsi.svc:5222] (shard-1-prosody.jitsi.svc/1<IP>:5222)' failed because: java.net.ConnectException: Connection refused (Connection refused)
org.jivesoftware.smack.SmackException$EndpointConnectionException: The following addresses failed: 'RFC 6120 A/AAAA Endpoint + [shard-1-prosody.jitsi.svc:5222] (shard-1-prosody.jitsi.svc/<IP>:5222)' failed because: java.net.ConnectException: Connection refused (Connection refused)
	at org.jivesoftware.smack.SmackException$EndpointConnectionException.from(SmackException.java:334)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.connectUsingConfiguration(XMPPTCPConnection.java:663)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.connectInternal(XMPPTCPConnection.java:846)
	at org.jivesoftware.smack.AbstractXMPPConnection.connect(AbstractXMPPConnection.java:526)
	at org.jitsi.impl.protocol.xmpp.XmppProviderImpl.doConnect(XmppProviderImpl.java:208)
	at org.jitsi.retry.RetryStrategy$TaskRunner.run(RetryStrategy.java:167)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:829)

JVB boots with exception

JVB 2023-03-03 10:47:27.061 INFO: [11] org.ice4j.ice.harvest.MappingCandidateHarvesters.initialize: Initialized mapping harvesters (delay=992ms).  stunDiscoveryFailed=false
JVB 2023-03-03 10:47:27.392 WARNING: [1] org.glassfish.jersey.server.wadl.WadlFeature.configure: JAXBContext implementation could not be found. WADL feature is disabled.
JVB 2023-03-03 10:47:27.512 WARNING: [1] org.glassfish.jersey.internal.inject.Providers.checkProviderRuntime: A provider org.jitsi.rest.Health registered in SERVER runtime does not implement any provider interfaces applicable in the SERVER runtime. Due to constraint configuration problems the provider org.jitsi.rest.Health will be ignored. 
JVB 2023-03-03 10:47:27.513 WARNING: [1] org.glassfish.jersey.internal.inject.Providers.checkProviderRuntime: A provider org.jitsi.rest.Version registered in SERVER runtime does not implement any provider interfaces applicable in the SERVER runtime. Due to constraint configuration problems the provider org.jitsi.rest.Version will be ignored. 
JVB 2023-03-03 10:47:27.848 WARNING: [18] [hostname=shard-1-prosody.jitsi.svc id=shard0] MucClient.lambda$getConnectAndLoginCallable$9#631: Error connecting:
org.jivesoftware.smack.SmackException$EndpointConnectionException: The following addresses failed: 'RFC 6120 A/AAAA Endpoint + [shard-1-prosody.jitsi.svc:5222] (shard-1-prosody.jitsi.svc/<IP>:5222)' failed because: java.net.ConnectException: Connection refused (Connection refused)
	at org.jivesoftware.smack.SmackException$EndpointConnectionException.from(SmackException.java:334)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.connectUsingConfiguration(XMPPTCPConnection.java:664)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.connectInternal(XMPPTCPConnection.java:849)
	at org.jivesoftware.smack.AbstractXMPPConnection.connect(AbstractXMPPConnection.java:526)
	at org.jitsi.xmpp.mucclient.MucClient.lambda$getConnectAndLoginCallable$9(MucClient.java:626)
	at org.jitsi.retry.RetryStrategy$TaskRunner.run(RetryStrategy.java:167)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:829)
JVB 2023-03-03 10:47:31.834 WARNING: [24] [hostname=shard-1-prosody.jitsi.svc id=shard0] MucClient.setPresenceExtensions#458: Cannot set presence extension: not connected.

The first user is able to join using a JWT as a moderator

When the second user tries to join with a valid JWT, the Jitsi meet web crashes and has the following error on the web console

<Hce._onConferenceFailed>:  CONFERENCE FAILED: conference.videobridgeNotAvailable

Everyone is kicked out and Jitsi meet web UI keeps on restarting indefinitely

Kindly help me in resolving this.

Jicofo and jvb cannot connect to your prosody server. Something is blocking connection to 5222.

Telnet confirmed prosody is reachable over port 5222 from both Jicofo and JVB

Additional exceptions from jicofo

Jicofo 2023-03-03 08:48:37.087 WARNING: [39] org.jivesoftware.smack.AbstractXMPPConnection.callConnectionClosedOnErrorListener: Connection XMPPTCPConnection[focus@auth.meet.jitsi/focus] (0) closed with error
org.jivesoftware.smack.XMPPException$StreamErrorException: system-shutdown You can read more about the meaning of this stream error at http://xmpp.org/rfcs/rfc6120.html#streams-error-conditions
<stream:error><system-shutdown xmlns='urn:ietf:params:xml:ns:xmpp-streams'/><text>Received SIGTERM</text></stream:error>
	at org.jivesoftware.smack.tcp.XMPPTCPConnection$PacketReader.parsePackets(XMPPTCPConnection.java:981)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection$PacketReader.access$700(XMPPTCPConnection.java:913)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection$PacketReader$1.run(XMPPTCPConnection.java:936)
	at java.base/java.lang.Thread.run(Thread.java:829)
Jicofo 2023-03-03 08:48:37.089 SEVERE: [39] [xmpp_connection=client] XmppProviderImpl$XmppConnectionListener.connectionClosedOnError#380: XMPP connection closed on error: system-shutdown You can read more about the meaning of this stream error at http://xmpp.org/rfcs/rfc6120.html#streams-error-conditions
<stream:error><system-shutdown xmlns='urn:ietf:params:xml:ns:xmpp-streams'/><text>Received SIGTERM</text></stream:error>

Jicofo 2023-03-03 10:43:23.924 INFO: [86] [xmpp_connection=client] XmppProviderImpl$XmppReConnectionListener.reconnectingIn#400: XMPP reconnecting in: 0
Jicofo 2023-03-03 10:43:40.060 WARNING: [86] [xmpp_connection=client] XmppProviderImpl.discoverInfo#346: Failed to discover info
org.jivesoftware.smack.SmackException$NoResponseException: No response received within reply timeout. Timeout was 15000ms (~15s). StanzaCollector has been cancelled. Waited for response using: IQReplyFilter: iqAndIdFilter (AndFilter: (OrFilter: (IQTypeFilter: type=error, IQTypeFilter: type=result), StanzaIdFilter: id=6J1EH-22)), : fromFilter (OrFilter: (FromMatchesFilter (full): meet.jitsi)).
	at org.jivesoftware.smack.SmackException$NoResponseException.newWith(SmackException.java:124)
	at org.jivesoftware.smack.SmackException$NoResponseException.newWith(SmackException.java:104)
	at org.jivesoftware.smack.StanzaCollector.nextResultOrThrow(StanzaCollector.java:281)
	at org.jivesoftware.smack.StanzaCollector.nextResultOrThrow(StanzaCollector.java:228)
	at org.jivesoftware.smackx.disco.ServiceDiscoveryManager.discoverInfo(ServiceDiscoveryManager.java:606)
	at org.jivesoftware.smackx.disco.ServiceDiscoveryManager.discoverInfo(ServiceDiscoveryManager.java:578)
	at org.jitsi.impl.protocol.xmpp.XmppProviderImpl.discoverInfo(XmppProviderImpl.java:339)
	at org.jitsi.jicofo.xmpp.AvModerationHandler.registrationChanged(AvModerationHandler.kt:115)
	at org.jitsi.impl.protocol.xmpp.AbstractXmppProvider.fireRegistrationStateChanged(AbstractXmppProvider.java:88)
	at org.jitsi.impl.protocol.xmpp.XmppProviderImpl.fireRegistrationStateChanged(XmppProviderImpl.java:308)
	at org.jitsi.impl.protocol.xmpp.AbstractXmppProvider.setRegistered(AbstractXmppProvider.java:119)
	at org.jitsi.impl.protocol.xmpp.XmppProviderImpl$XmppConnectionListener.authenticated(XmppProviderImpl.java:362)
	at org.jivesoftware.smack.AbstractXMPPConnection.callConnectionAuthenticatedListener(AbstractXMPPConnection.java:1707)
	at org.jivesoftware.smack.AbstractXMPPConnection.afterSuccessfulLogin(AbstractXMPPConnection.java:792)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.afterSuccessfulLogin(XMPPTCPConnection.java:371)
	at org.jivesoftware.smack.tcp.XMPPTCPConnection.loginInternal(XMPPTCPConnection.java:469)
	at org.jivesoftware.smack.AbstractXMPPConnection.login(AbstractXMPPConnection.java:638)
	at org.jivesoftware.smack.AbstractXMPPConnection.login(AbstractXMPPConnection.java:595)
	at org.jivesoftware.smack.ReconnectionManager$2.run(ReconnectionManager.java:287)
	at java.base/java.lang.Thread.run(Thread.java:829)

Jicofo 2023-03-03 10:43:40.071 SEVERE: [87] org.jivesoftware.smack.roster.Roster$5.processException: Exception reloading roster
org.jivesoftware.smack.SmackException$NoResponseException: No response received within reply timeout. Timeout was 15000ms (~15s). Waited for response using: IQReplyFilter: iqAndIdFilter (AndFilter: (OrFilter: (IQTypeFilter: type=error, IQTypeFilter: type=result), StanzaIdFilter: id=ILRI8-1103)), : fromFilter (OrFilter: (FromMatchesFilter (full): null, FromMatchesFilter (ignoreResourcepart): focus@auth.meet.jitsi, FromMatchesFilter (full): auth.meet.jitsi)).
	at org.jivesoftware.smack.SmackException$NoResponseException.newWith(SmackException.java:124)
	at org.jivesoftware.smack.SmackException$NoResponseException.newWith(SmackException.java:108)
	at org.jivesoftware.smack.AbstractXMPPConnection$7.run(AbstractXMPPConnection.java:2000)
	at org.jivesoftware.smack.ScheduledAction.run(ScheduledAction.java:84)
	at org.jivesoftware.smack.SmackReactor$Reactor.handleScheduledActionsOrPerformSelect(SmackReactor.java:208)
	at org.jivesoftware.smack.SmackReactor$Reactor.reactorLoop(SmackReactor.java:188)
	at org.jivesoftware.smack.SmackReactor$Reactor.run(SmackReactor.java:173)

This is when you restarted prosody and is normal.

Maybe there is an error in prosody when this happens

Prosody is clean, no errors

You have fixed those?

I think I have seen these if you disable the roster module in prosody …

But the discovery info with no reply is strange …

Additional log from prosody when Jitsi crashes upon the second user joining

c2s559caf26fbb0                                              debug	Offering usable mechanisms: ANONYMOUS
c2s559caf26fbb0                                              debug	Sending[c2s_unauthed]: <stream:features>
mod_websocket                                                debug	Websocket received frame: opcode=1, 70 bytes
c2s559caf26fbb0                                              debug	Received[c2s_unauthed]: <auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' xml:lang='en' mechanism='ANONYMOUS'>
c2s559caf26fbb0                                              info	Authenticated as ee2b8139-a5d2-4a27-9e8c-2c28ce9d3229@meet.jitsi
c2s559caf26fbb0                                              debug	Sending[c2s_unbound]: <success xmlns='urn:ietf:params:xml:ns:xmpp-sasl'>
mod_websocket                                                debug	Websocket received frame: opcode=1, 81 bytes
c2s559caf26fbb0                                              debug	Client sent opening <stream:stream> to meet.jitsi
c2s559caf26fbb0                                              debug	Sending[c2s_unbound]: <open xmlns='urn:ietf:params:xml:ns:xmpp-framing' from='meet.jitsi' xml:lang='en' version='1.0' id='f511da34-3bd5-4c6d-8c80-d36404466a8b'>
c2s559caf26fbb0                                              debug	Sent reply <stream:stream> to client
c2s559caf26fbb0                                              debug	Sending[c2s_unbound]: <stream:features>
mod_websocket                                                debug	Websocket received frame: opcode=1, 108 bytes
c2s559caf26fbb0                                              debug	Received[c2s_unbound]: <iq type='set' id='_bind_auth_2' xml:lang='en'>
rostermanager                                                debug	load_roster: asked for: ee2b8139-a5d2-4a27-9e8c-2c28ce9d3229@meet.jitsi
rostermanager                                                debug	load_roster: loading for new user: ee2b8139-a5d2-4a27-9e8c-2c28ce9d3229@meet.jitsi
rostermanager                                                debug	load_roster: cache miss, loading from storage
c2s559caf26fbb0                                              debug	Sending[c2s]: <iq id='_bind_auth_2' type='result'>
c2s559caf26fbb0                                              debug	Resource bound: ee2b8139-a5d2-4a27-9e8c-2c28ce9d3229@meet.jitsi/ckI1R2KpGHQQ
mod_websocket                                                debug	Websocket received frame: opcode=1, 117 bytes
c2s559caf26fbb0                                              debug	Received[c2s]: <iq type='set' id='_session_auth_2' xml:lang='en'>
c2s559caf26fbb0                                              debug	Sending[c2s]: <iq type='result' id='_session_auth_2' to='ee2b8139-a5d2-4a27-9e8c-2c28ce9d3229@meet.jitsi/ckI1R2KpGHQQ'>
mod_websocket                                                debug	Websocket received frame: opcode=1, 45 bytes
c2s559caf26fbb0                                              debug	Received[c2s]: <enable xmlns='urn:xmpp:sm:3' xml:lang='en' resume='true'>
meet.jitsi:smacks                                            debug	Enabling stream management
c2s559caf26fbb0                                              debug	Sending[c2s]: <enabled xmlns='urn:xmpp:sm:3' max='60' id='24637755-4732-4ab9-a825-68d2dac65953' resume='true'>
mod_websocket                                                debug	Websocket received frame: opcode=1, 146 bytes
c2s559caf26fbb0                                              debug	Handled 1 incoming stanzas


What does crashing mean? Any js console errors you see?

JS console error

<Hce._onConferenceFailed>:  CONFERENCE FAILED: conference.videobridgeNotAvailable

You got your answer there, there are no jvbs to be used.