diff --git a/examples/camera-app/linux/include/webrtc-abstract.h b/examples/camera-app/linux/include/webrtc-abstract.h index d5aaebb65670eb..e7725dc22964f5 100644 --- a/examples/camera-app/linux/include/webrtc-abstract.h +++ b/examples/camera-app/linux/include/webrtc-abstract.h @@ -42,8 +42,15 @@ enum class MediaType : uint8_t Video, }; +struct ICECandidateInfo +{ + std::string candidate; + std::string mid; + int mlineIndex; +}; + using OnLocalDescriptionCallback = std::function; -using OnICECandidateCallback = std::function; +using OnICECandidateCallback = std::function; using OnConnectionStateCallback = std::function; using OnTrackCallback = std::function track)>; diff --git a/examples/camera-app/linux/include/webrtc-transport.h b/examples/camera-app/linux/include/webrtc-transport.h index b82df48d6943c9..388d7d9fe78bf1 100644 --- a/examples/camera-app/linux/include/webrtc-transport.h +++ b/examples/camera-app/linux/include/webrtc-transport.h @@ -105,9 +105,9 @@ class WebrtcTransport : public Transport void SetSdpAnswer(std::string localSdp) { mLocalSdp = localSdp; } - std::vector GetCandidates() { return mLocalCandidates; } + const std::vector & GetCandidates() { return mLocalCandidates; } - void SetCandidates(std::vector candidates) { mLocalCandidates = candidates; } + void SetCandidates(std::vector candidates) { mLocalCandidates = candidates; } void AddRemoteCandidate(const std::string & candidate, const std::string & mid); @@ -119,7 +119,7 @@ class WebrtcTransport : public Transport // WebRTC Callbacks void OnLocalDescription(const std::string & sdp, SDPType type); - void OnICECandidate(const std::string & candidate); + void OnICECandidate(const ICECandidateInfo & candidateInfo); void OnConnectionStateChanged(bool connected); void OnTrack(std::shared_ptr track); @@ -138,7 +138,7 @@ class WebrtcTransport : public Transport std::string mLocalSdp; SDPType mLocalSdpType; - std::vector mLocalCandidates; + std::vector mLocalCandidates; RequestArgs mRequestArgs; OnTransportLocalDescriptionCallback mOnLocalDescription = nullptr; diff --git a/examples/camera-app/linux/src/clusters/webrtc-provider/webrtc-provider-manager.cpp b/examples/camera-app/linux/src/clusters/webrtc-provider/webrtc-provider-manager.cpp index c481900a804f64..257e2ca53066bd 100644 --- a/examples/camera-app/linux/src/clusters/webrtc-provider/webrtc-provider-manager.cpp +++ b/examples/camera-app/linux/src/clusters/webrtc-provider/webrtc-provider-manager.cpp @@ -975,7 +975,9 @@ CHIP_ERROR WebRTCProviderManager::SendICECandidatesCommand(Messaging::ExchangeMa ChipLogError(Camera, "WebTransport not found for the sessionId: %u", sessionId); return CHIP_ERROR_INTERNAL; } - std::vector localCandidates = transport->GetCandidates(); + + const std::vector & localCandidates = transport->GetCandidates(); + // Build the command WebRTCTransportRequestor::Commands::ICECandidates::Type command; @@ -986,9 +988,31 @@ CHIP_ERROR WebRTCProviderManager::SendICECandidatesCommand(Messaging::ExchangeMa } std::vector iceCandidateStructList; - for (const auto & candidate : localCandidates) + for (const auto & candidateInfo : localCandidates) { - ICECandidateStruct iceCandidate = { CharSpan::fromCharString(candidate.c_str()) }; + ICECandidateStruct iceCandidate; + iceCandidate.candidate = CharSpan(candidateInfo.candidate.data(), candidateInfo.candidate.size()); + + // Set SDPMid if available + if (!candidateInfo.mid.empty()) + { + iceCandidate.SDPMid.SetNonNull(CharSpan(candidateInfo.mid.data(), candidateInfo.mid.size())); + } + else + { + iceCandidate.SDPMid.SetNull(); + } + + // Set SDPMLineIndex if valid + if (candidateInfo.mlineIndex >= 0) + { + iceCandidate.SDPMLineIndex.SetNonNull(static_cast(candidateInfo.mlineIndex)); + } + else + { + iceCandidate.SDPMLineIndex.SetNull(); + } + iceCandidateStructList.push_back(iceCandidate); } diff --git a/examples/camera-app/linux/src/webrtc-libdatachannel.cpp b/examples/camera-app/linux/src/webrtc-libdatachannel.cpp index 6a37b04dc8884c..695aa55c1a5abc 100644 --- a/examples/camera-app/linux/src/webrtc-libdatachannel.cpp +++ b/examples/camera-app/linux/src/webrtc-libdatachannel.cpp @@ -291,10 +291,38 @@ class LibDataChannelPeerConnection : public WebRTCPeerConnection void SetCallbacks(OnLocalDescriptionCallback onLocalDescription, OnICECandidateCallback onICECandidate, OnConnectionStateCallback onConnectionState, OnTrackCallback onTrack) override { - mPeerConnection->onLocalDescription( - [onLocalDescription](rtc::Description desc) { onLocalDescription(std::string(desc), RtcTypeToSDPType(desc.type())); }); + mPeerConnection->onLocalDescription([onLocalDescription, onICECandidate](rtc::Description desc) { + // First, notify about the local description + onLocalDescription(std::string(desc), RtcTypeToSDPType(desc.type())); - mPeerConnection->onLocalCandidate([onICECandidate](rtc::Candidate candidate) { onICECandidate(std::string(candidate)); }); + // Extract any candidates embedded in the SDP description + std::vector candidates = desc.candidates(); + ChipLogProgress(Camera, "Extracted %lu candidates from SDP description", candidates.size()); + + for (const auto & candidate : candidates) + { + ICECandidateInfo candidateInfo; + candidateInfo.candidate = std::string(candidate); + candidateInfo.mid = candidate.mid(); + candidateInfo.mlineIndex = -1; // libdatachannel doesn't provide mlineIndex + + ChipLogProgress(Camera, "[From SDP] Candidate: %s, mid: %s", candidateInfo.candidate.c_str(), + candidateInfo.mid.c_str()); + + onICECandidate(candidateInfo); + } + }); + + mPeerConnection->onLocalCandidate([onICECandidate](rtc::Candidate candidate) { + ICECandidateInfo candidateInfo; + candidateInfo.candidate = std::string(candidate); + candidateInfo.mid = candidate.mid(); + + // Note: libdatachannel doesn't directly provide mlineIndex, so we use -1 to indicate it is not present. + candidateInfo.mlineIndex = -1; + + onICECandidate(candidateInfo); + }); mPeerConnection->onStateChange([onConnectionState](rtc::PeerConnection::State state) { ChipLogProgress(Camera, "[PeerConnection State: %s]", GetPeerConnectionStateStr(state)); diff --git a/examples/camera-app/linux/src/webrtc-transport.cpp b/examples/camera-app/linux/src/webrtc-transport.cpp index 584dc854c5de84..bc60db7c49e29a 100644 --- a/examples/camera-app/linux/src/webrtc-transport.cpp +++ b/examples/camera-app/linux/src/webrtc-transport.cpp @@ -146,7 +146,7 @@ void WebrtcTransport::Start() mPeerConnection = CreateWebRTCPeerConnection(); mPeerConnection->SetCallbacks([this](const std::string & sdp, SDPType type) { this->OnLocalDescription(sdp, type); }, - [this](const std::string & candidate) { this->OnICECandidate(candidate); }, + [this](const ICECandidateInfo & candidateInfo) { this->OnICECandidate(candidateInfo); }, [this](bool connected) { this->OnConnectionStateChanged(connected); }, [this](std::shared_ptr track) { this->OnTrack(track); }); } @@ -208,12 +208,13 @@ bool WebrtcTransport::ClosePeerConnection() return true; } -void WebrtcTransport::OnICECandidate(const std::string & candidate) +void WebrtcTransport::OnICECandidate(const ICECandidateInfo & candidateInfo) { ChipLogProgress(Camera, "ICE Candidate received for sessionID: %u", mRequestArgs.sessionId); - mLocalCandidates.push_back(candidate); + mLocalCandidates.push_back(candidateInfo); ChipLogProgress(Camera, "Local Candidate:"); - ChipLogProgress(Camera, "%s", candidate.c_str()); + ChipLogProgress(Camera, "%s", candidateInfo.candidate.c_str()); + ChipLogProgress(Camera, " mid: %s, mlineIndex: %d", candidateInfo.mid.c_str(), candidateInfo.mlineIndex); } void WebrtcTransport::OnConnectionStateChanged(bool connected) diff --git a/examples/camera-controller/webrtc-manager/WebRTCManager.cpp b/examples/camera-controller/webrtc-manager/WebRTCManager.cpp index 79f78fa214fc28..bacec6683575bf 100644 --- a/examples/camera-controller/webrtc-manager/WebRTCManager.cpp +++ b/examples/camera-controller/webrtc-manager/WebRTCManager.cpp @@ -277,13 +277,38 @@ CHIP_ERROR WebRTCManager::Connnect(Controller::DeviceCommissioner & commissioner mLocalDescription = std::string(desc); ChipLogProgress(Camera, "Local Description:"); ChipLogProgress(Camera, "%s", mLocalDescription.c_str()); + + // Extract any candidates embedded in the SDP description + std::vector candidates = desc.candidates(); + ChipLogProgress(Camera, "Extracted %lu candidates from SDP description", candidates.size()); + + for (const auto & candidate : candidates) + { + ICECandidateInfo candidateInfo; + candidateInfo.candidate = std::string(candidate); + candidateInfo.mid = candidate.mid(); + candidateInfo.mlineIndex = -1; // libdatachannel doesn't provide mlineIndex + + ChipLogProgress(Camera, "[From SDP] Candidate: %s, mid: %s", candidateInfo.candidate.c_str(), + candidateInfo.mid.c_str()); + + mLocalCandidates.push_back(candidateInfo); + } }); mPeerConnection->onLocalCandidate([this](rtc::Candidate candidate) { - std::string candidateStr = std::string(candidate); - mLocalCandidates.push_back(candidateStr); + ICECandidateInfo candidateInfo; + candidateInfo.candidate = std::string(candidate); + candidateInfo.mid = candidate.mid(); + + // Note: libdatachannel doesn't directly provide mlineIndex, so we use -1 to indicate it is not present. + candidateInfo.mlineIndex = -1; + ChipLogProgress(Camera, "Local Candidate:"); - ChipLogProgress(Camera, "%s", candidateStr.c_str()); + ChipLogProgress(Camera, "%s", candidateInfo.candidate.c_str()); + ChipLogProgress(Camera, " mid: %s, mlineIndex: %d", candidateInfo.mid.c_str(), candidateInfo.mlineIndex); + + mLocalCandidates.push_back(candidateInfo); }); mPeerConnection->onStateChange([this](rtc::PeerConnection::State state) { diff --git a/examples/camera-controller/webrtc-manager/WebRTCManager.h b/examples/camera-controller/webrtc-manager/WebRTCManager.h index b630a434b0c729..df74357409d0ea 100644 --- a/examples/camera-controller/webrtc-manager/WebRTCManager.h +++ b/examples/camera-controller/webrtc-manager/WebRTCManager.h @@ -26,6 +26,13 @@ #include #include +struct ICECandidateInfo +{ + std::string candidate; + std::string mid; + int mlineIndex; +}; + class WebRTCManager { public: @@ -87,8 +94,8 @@ class WebRTCManager uint16_t mPendingSessionId = 0; std::string mLocalDescription; - // Local vector to store the ICE Candidate strings coming from the WebRTC stack - std::vector mLocalCandidates; + // Local vector to store the ICE Candidate info coming from the WebRTC stack + std::vector mLocalCandidates; std::shared_ptr mTrack; std::shared_ptr mAudioTrack; diff --git a/examples/camera-controller/webrtc-manager/WebRTCProviderClient.cpp b/examples/camera-controller/webrtc-manager/WebRTCProviderClient.cpp index 691390e09316eb..2320775c596334 100644 --- a/examples/camera-controller/webrtc-manager/WebRTCProviderClient.cpp +++ b/examples/camera-controller/webrtc-manager/WebRTCProviderClient.cpp @@ -162,7 +162,7 @@ CHIP_ERROR WebRTCProviderClient::ProvideAnswer(uint16_t webRTCSessionId, const s return CHIP_NO_ERROR; } -CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId, const std::vector & iceCandidates) +CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId, const std::vector & iceCandidates) { ChipLogProgress(Camera, "Sending ProvideICECandidates to node " ChipLogFormatX64, ChipLogValueX64(mPeerId.GetNodeId())); @@ -175,14 +175,38 @@ CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId, // Store the command type mCommandType = CommandType::kProvideICECandidates; - // Store ICE Candidates. + // Store ICE Candidates for lifetime management mClientICECandidates = iceCandidates; + mICECandidateStructList.clear(); - for (const auto & candidate : mClientICECandidates) + for (const auto & candidateInfo : mClientICECandidates) { - ICECandidateStruct iceCandidate = { CharSpan::fromCharString(candidate.c_str()) }; + ICECandidateStruct iceCandidate; + iceCandidate.candidate = CharSpan(candidateInfo.candidate.data(), candidateInfo.candidate.size()); + + // Set SDPMid if available + if (!candidateInfo.mid.empty()) + { + iceCandidate.SDPMid.SetNonNull(CharSpan(candidateInfo.mid.data(), candidateInfo.mid.size())); + } + else + { + iceCandidate.SDPMid.SetNull(); + } + + // Set SDPMLineIndex if valid + if (candidateInfo.mlineIndex >= 0) + { + iceCandidate.SDPMLineIndex.SetNonNull(static_cast(candidateInfo.mlineIndex)); + } + else + { + iceCandidate.SDPMLineIndex.SetNull(); + } + mICECandidateStructList.push_back(iceCandidate); } + // Stash data in class members so the CommandSender can safely reference them async mProvideICECandidatesData.webRTCSessionID = webRTCSessionId; mProvideICECandidatesData.ICECandidates = diff --git a/examples/camera-controller/webrtc-manager/WebRTCProviderClient.h b/examples/camera-controller/webrtc-manager/WebRTCProviderClient.h index 15b2b19ebdbb29..23668ef19c0cd4 100644 --- a/examples/camera-controller/webrtc-manager/WebRTCProviderClient.h +++ b/examples/camera-controller/webrtc-manager/WebRTCProviderClient.h @@ -23,6 +23,9 @@ #include #include +// Forward declaration +struct ICECandidateInfo; + /** * @brief This class handles sending CHIP commands for WebRTCTransportProvider cluster, including * sending a ProvideOffer command to a remote camera device. @@ -111,17 +114,17 @@ class WebRTCProviderClient : public chip::app::CommandSender::Callback * @brief Sends a ProvideICECandidates command to the remote device. * * This method populates the ProvideICECandidates command parameters, packages the provided ICE - * candidate strings, and queues them for sending to the target device. This is typically used - * to inform the remote side about potential network endpoints it can use to establish or - * enhance a WebRTC session. + * candidate information including SDPMid and SDPMLineIndex, and queues them for sending to the + * target device. This is typically used to inform the remote side about potential network + * endpoints it can use to establish or enhance a WebRTC session. * * @param webRTCSessionId The unique identifier for the WebRTC session to which these * ICE candidates apply. - * @param ICECandidates A list of ICE candidate structs. + * @param iceCandidates A list of ICE candidate structs. * * @return CHIP_NO_ERROR on success, or an appropriate CHIP_ERROR on failure. */ - CHIP_ERROR ProvideICECandidates(uint16_t webRTCSessionId, const std::vector & iceCandidates); + CHIP_ERROR ProvideICECandidates(uint16_t webRTCSessionId, const std::vector & iceCandidates); /** * @brief Notify WebRTCProviderClient that the Offer command has been received. @@ -231,7 +234,7 @@ class WebRTCProviderClient : public chip::app::CommandSender::Callback std::string mSdpString; // Store the ICECandidates here to use to send asynchronously. - std::vector mClientICECandidates; + std::vector mClientICECandidates; std::vector mICECandidateStructList; chip::Callback::Callback mOnConnectedCallback;