From 3d88b76c0d06265616bb46fea1b4368225f8b19c Mon Sep 17 00:00:00 2001 From: gari Date: Sun, 13 Apr 2025 10:52:03 +0200 Subject: [PATCH] fix: implement correct sdp offer/answer exchange and ice handling --- internal/api/preview.go | 154 +++++++++++++++++++++++++++++++++------- 1 file changed, 129 insertions(+), 25 deletions(-) diff --git a/internal/api/preview.go b/internal/api/preview.go index 3f0220e..b2bdc20 100644 --- a/internal/api/preview.go +++ b/internal/api/preview.go @@ -89,34 +89,103 @@ func (api *API) previewHandler(c echo.Context) error { } defer conn.Close() + // --- WebRTC Signal Handling --- + + // on-negotiation-needed: Triggered when webrtcbin needs to create an offer or answer. + // In our case (server receiving an offer), this fires after the remote description is set. previewPipeline.webrtcbin.Connect("on-negotiation-needed", func(self *gst.Element) { - log.Debug().Msg("🧠 Negotiation needed (ignored)") + log.Info().Msg("🧠 Negotiation needed, creating answer...") + + // Create Answer + promise, err := self.Emit("create-answer") + if err != nil { + log.Error().Err(err).Msg("Failed to emit create-answer") + return + } + if promise == nil { + log.Error().Msg("Emit create-answer returned nil promise") + return + } + + // Handle the promise result (asynchronously) + promise.(*gst.Promise).Interrupt() // Interrupt any previous waits + promise.(*gst.Promise).Wait() // Wait for the answer to be created + reply := promise.(*gst.Promise).GetReply() + if reply == nil { + log.Error().Msg("Promise reply for create-answer was nil") + return + } + + answerValue, ok := reply.GetValue("answer") + if !ok || answerValue == nil { + log.Error().Msg("Failed to get answer from promise reply") + return + } + answer, ok := answerValue.(*gst.WebRTCSessionDescription) + if !ok || answer == nil { + log.Error().Msg("Answer value is not a WebRTCSessionDescription") + return + } + + log.Debug().Str("sdp", answer.GetSDP().String()).Msg("✅ Answer created") + + // Set Local Description + promise, err = self.Emit("set-local-description", answer) + if err != nil { + log.Error().Err(err).Msg("Failed to emit set-local-description") + return + } + if promise == nil { + log.Error().Msg("Emit set-local-description returned nil promise") + return + } + promise.(*gst.Promise).Interrupt() // Interrupt any previous waits + + log.Info().Msg("➡️ Set local description (answer)") + + // Send Answer back to the client + log.Info().Msg("📨 Sending SDP answer back to browser") + response := SignalMessage{ + Type: "answer", + SDP: answer.GetSDP().String(), + } + msg, err := json.Marshal(response) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal answer") + return + } + if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil { + log.Error().Err(err).Msg("Failed to send answer over WebSocket") + } }) - previewPipeline.webrtcbin.Connect("on-ice-candidate", func(self *gst.Element, mlineindex int, candidate string) { + // on-ice-candidate: Triggered when webrtcbin generates a new ICE candidate. + previewPipeline.webrtcbin.Connect("on-ice-candidate", func(self *gst.Element, mlineindex uint, candidate string) { + log.Debug().Uint("mlineindex", mlineindex).Str("candidate", candidate).Msg("🧊 Generated ICE candidate") + // Note: The 'sdpMid' is often derived from the mlineindex or context, + // but the signal itself only provides mlineindex and candidate string directly. + // The JS side seems to handle this structure correctly. ice := SignalMessage{ Type: "ice", ICE: &IceCandidate{ Candidate: candidate, - SDPMLineIndex: uint16(mlineindex), + SDPMLineIndex: uint16(mlineindex), // Assuming JS expects uint16 + // sdpMid might need to be derived if required, but often index is enough }, } - msg, _ := json.Marshal(ice) - conn.WriteMessage(websocket.TextMessage, msg) - }) - - previewPipeline.webrtcbin.Connect("on-sdp-answer", func(self *gst.Element, answer string) { - log.Info().Msg("📨 Sending SDP answer back to browser") - response := SignalMessage{ - Type: "answer", - SDP: answer, + msg, err := json.Marshal(ice) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal ICE candidate") + return + } + if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil { + log.Error().Err(err).Msg("Failed to send ICE candidate over WebSocket") } - msg, _ := json.Marshal(response) - conn.WriteMessage(websocket.TextMessage, msg) }) + // --- WebSocket Message Loop --- for { - _, message, err := conn.ReadMessage() + messageType, message, err := conn.ReadMessage() if err != nil { log.Error().Err(err).Msg("WebSocket read error") break @@ -124,22 +193,57 @@ func (api *API) previewHandler(c echo.Context) error { var signal SignalMessage if err := json.Unmarshal(message, &signal); err != nil { log.Warn().Err(err).Msg("Failed to unmarshal signal message") + log.Warn().Err(err).Str("message", string(message)).Msg("Failed to unmarshal signal message") continue } - if signal.Type == "offer" { + switch signal.Type { + case "offer": log.Info().Msg("📥 Received SDP offer from browser") - // Pass the SDP offer to webrtcbin - previewPipeline.webrtcbin.Emit("set-remote-description", "offer", signal.SDP) + log.Debug().Str("sdp", signal.SDP).Msg("Offer SDP") - // Create an answer - previewPipeline.webrtcbin.Emit("create-answer") - log.Debug().Msg("➡️ Sent SDP offer into webrtcbin and requested answer") - } + // Create WebRTCSessionDescription for the offer + offerDesc, err := gst.NewWebRTCSessionDescription(gst.WebRTCSDPTypeOffer, gst.NewSDPMessageFromString(signal.SDP)) + if err != nil { + log.Error().Err(err).Msg("Failed to create offer description from SDP") + continue + } - if signal.Type == "ice" && signal.ICE != nil { - log.Debug().Str("candidate", signal.ICE.Candidate).Msg("➕ Adding ICE candidate") - previewPipeline.webrtcbin.Emit("add-ice-candidate", signal.ICE.SDPMLineIndex, signal.ICE.Candidate) + // Set Remote Description + // This will trigger on-negotiation-needed if successful and state allows + promise, err := previewPipeline.webrtcbin.Emit("set-remote-description", offerDesc) + if err != nil { + log.Error().Err(err).Msg("Failed to emit set-remote-description") + continue + } + if promise == nil { + log.Error().Msg("Emit set-remote-description returned nil promise") + continue + } + promise.(*gst.Promise).Interrupt() // Interrupt previous waits if any + + log.Info().Msg("➡️ Set remote description (offer)") + // Answer creation is now handled in on-negotiation-needed + + case "ice": + if signal.ICE == nil { + log.Warn().Msg("Received ICE signal with nil ICE field") + continue + } + log.Debug().Str("candidate", signal.ICE.Candidate).Uint16("mlineindex", signal.ICE.SDPMLineIndex).Str("sdpMid", signal.ICE.SDPMid).Msg("➕ Received ICE candidate from browser") + + // Add ICE Candidate + // Note: The signal takes mlineindex (uint) and candidate (string). + // sdpMid is usually associated but not directly passed here. + _, err := previewPipeline.webrtcbin.Emit("add-ice-candidate", uint(signal.ICE.SDPMLineIndex), signal.ICE.Candidate) + if err != nil { + log.Error().Err(err).Msg("Failed to emit add-ice-candidate") + } else { + log.Debug().Msg("Added ICE candidate") + } + + default: + log.Warn().Str("type", signal.Type).Msg("Received unknown signal type") } }