[WebRTC] Update ICECandidates/ProvideICECandidates Commands to populate SDPMid and SDPMLineIndex

examples/camera-app/linux/include/webrtc-abstract.h

@@ -42,8 +42,15 @@ enum class MediaType : uint8_t
     Video,
 };
 
+struct ICECandidateInfo
+{
+    std::string candidate;
+    std::string mid;
+    int mlineIndex;
+};
+
 using OnLocalDescriptionCallback = std::function<void(const std::string & sdp, SDPType type)>;
-using OnICECandidateCallback     = std::function<void(const std::string & candidate)>;
+using OnICECandidateCallback     = std::function<void(const ICECandidateInfo & candidateInfo)>;
 using OnConnectionStateCallback  = std::function<void(bool connected)>;
 using OnTrackCallback            = std::function<void(std::shared_ptr<WebRTCTrack> track)>;
 

examples/camera-app/linux/include/webrtc-transport.h

@@ -105,9 +105,9 @@ class WebrtcTransport : public Transport
 
     void SetSdpAnswer(std::string localSdp) { mLocalSdp = localSdp; }
 
-    std::vector<std::string> GetCandidates() { return mLocalCandidates; }
+    const std::vector<ICECandidateInfo> & GetCandidates() { return mLocalCandidates; }
 
-    void SetCandidates(std::vector<std::string> candidates) { mLocalCandidates = candidates; }
+    void SetCandidates(std::vector<ICECandidateInfo> candidates) { mLocalCandidates = candidates; }
 
     void AddRemoteCandidate(const std::string & candidate, const std::string & mid);
 
@@ -119,7 +119,7 @@ class WebrtcTransport : public Transport
 
     // WebRTC Callbacks
     void OnLocalDescription(const std::string & sdp, SDPType type);
-    void OnICECandidate(const std::string & candidate);
+    void OnICECandidate(const ICECandidateInfo & candidateInfo);
     void OnConnectionStateChanged(bool connected);
     void OnTrack(std::shared_ptr<WebRTCTrack> track);
 
@@ -138,7 +138,7 @@ class WebrtcTransport : public Transport
 
     std::string mLocalSdp;
     SDPType mLocalSdpType;
-    std::vector<std::string> mLocalCandidates;
+    std::vector<ICECandidateInfo> mLocalCandidates;
 
     RequestArgs mRequestArgs;
     OnTransportLocalDescriptionCallback mOnLocalDescription = nullptr;

examples/camera-app/linux/src/clusters/webrtc-provider/webrtc-provider-manager.cpp

@@ -975,7 +975,9 @@ CHIP_ERROR WebRTCProviderManager::SendICECandidatesCommand(Messaging::ExchangeMa
         ChipLogError(Camera, "WebTransport not found for the sessionId: %u", sessionId);
         return CHIP_ERROR_INTERNAL;
     }
-    std::vector<std::string> localCandidates = transport->GetCandidates();
+
+    const std::vector<ICECandidateInfo> & localCandidates = transport->GetCandidates();
+
     // Build the command
     WebRTCTransportRequestor::Commands::ICECandidates::Type command;
 
@@ -986,9 +988,31 @@ CHIP_ERROR WebRTCProviderManager::SendICECandidatesCommand(Messaging::ExchangeMa
     }
 
     std::vector<ICECandidateStruct> iceCandidateStructList;
-    for (const auto & candidate : localCandidates)
+    for (const auto & candidateInfo : localCandidates)
     {
-        ICECandidateStruct iceCandidate = { CharSpan::fromCharString(candidate.c_str()) };
+        ICECandidateStruct iceCandidate;
+        iceCandidate.candidate = CharSpan(candidateInfo.candidate.data(), candidateInfo.candidate.size());
+
+        // Set SDPMid if available
+        if (!candidateInfo.mid.empty())
+        {
+            iceCandidate.SDPMid.SetNonNull(CharSpan(candidateInfo.mid.data(), candidateInfo.mid.size()));
+        }
+        else
+        {
+            iceCandidate.SDPMid.SetNull();
+        }
+
+        // Set SDPMLineIndex if valid
+        if (candidateInfo.mlineIndex >= 0)
+        {
+            iceCandidate.SDPMLineIndex.SetNonNull(static_cast<uint16_t>(candidateInfo.mlineIndex));
+        }
+        else
+        {
+            iceCandidate.SDPMLineIndex.SetNull();
+        }
+
         iceCandidateStructList.push_back(iceCandidate);
     }
 

examples/camera-app/linux/src/webrtc-libdatachannel.cpp

@@ -291,10 +291,38 @@ class LibDataChannelPeerConnection : public WebRTCPeerConnection
     void SetCallbacks(OnLocalDescriptionCallback onLocalDescription, OnICECandidateCallback onICECandidate,
                       OnConnectionStateCallback onConnectionState, OnTrackCallback onTrack) override
     {
-        mPeerConnection->onLocalDescription(
-            [onLocalDescription](rtc::Description desc) { onLocalDescription(std::string(desc), RtcTypeToSDPType(desc.type())); });
+        mPeerConnection->onLocalDescription([onLocalDescription, onICECandidate](rtc::Description desc) {
+            // First, notify about the local description
+            onLocalDescription(std::string(desc), RtcTypeToSDPType(desc.type()));
 
-        mPeerConnection->onLocalCandidate([onICECandidate](rtc::Candidate candidate) { onICECandidate(std::string(candidate)); });
+            // Extract any candidates embedded in the SDP description
+            std::vector<rtc::Candidate> candidates = desc.candidates();
+            ChipLogProgress(Camera, "Extracted %lu candidates from SDP description", candidates.size());
+
+            for (const auto & candidate : candidates)
+            {
+                ICECandidateInfo candidateInfo;
+                candidateInfo.candidate  = std::string(candidate);
+                candidateInfo.mid        = candidate.mid();
+                candidateInfo.mlineIndex = -1; // libdatachannel doesn't provide mlineIndex
+
+                ChipLogProgress(Camera, "[From SDP] Candidate: %s, mid: %s", candidateInfo.candidate.c_str(),
+                                candidateInfo.mid.c_str());
+
+                onICECandidate(candidateInfo);
+            }
+        });
+
+        mPeerConnection->onLocalCandidate([onICECandidate](rtc::Candidate candidate) {
+            ICECandidateInfo candidateInfo;
+            candidateInfo.candidate = std::string(candidate);
+            candidateInfo.mid       = candidate.mid();
+
+            // Note: libdatachannel doesn't directly provide mlineIndex, so we use -1 to indicate it is not present.
+            candidateInfo.mlineIndex = -1;
+
+            onICECandidate(candidateInfo);
+        });
 
         mPeerConnection->onStateChange([onConnectionState](rtc::PeerConnection::State state) {
             ChipLogProgress(Camera, "[PeerConnection State: %s]", GetPeerConnectionStateStr(state));

examples/camera-app/linux/src/webrtc-transport.cpp

@@ -146,7 +146,7 @@ void WebrtcTransport::Start()
     mPeerConnection = CreateWebRTCPeerConnection();
 
     mPeerConnection->SetCallbacks([this](const std::string & sdp, SDPType type) { this->OnLocalDescription(sdp, type); },
-                                  [this](const std::string & candidate) { this->OnICECandidate(candidate); },
+                                  [this](const ICECandidateInfo & candidateInfo) { this->OnICECandidate(candidateInfo); },
                                   [this](bool connected) { this->OnConnectionStateChanged(connected); },
                                   [this](std::shared_ptr<WebRTCTrack> track) { this->OnTrack(track); });
 }
@@ -208,12 +208,13 @@ bool WebrtcTransport::ClosePeerConnection()
     return true;
 }
 
-void WebrtcTransport::OnICECandidate(const std::string & candidate)
+void WebrtcTransport::OnICECandidate(const ICECandidateInfo & candidateInfo)
 {
     ChipLogProgress(Camera, "ICE Candidate received for sessionID: %u", mRequestArgs.sessionId);
-    mLocalCandidates.push_back(candidate);
+    mLocalCandidates.push_back(candidateInfo);
     ChipLogProgress(Camera, "Local Candidate:");
-    ChipLogProgress(Camera, "%s", candidate.c_str());
+    ChipLogProgress(Camera, "%s", candidateInfo.candidate.c_str());
+    ChipLogProgress(Camera, "  mid: %s, mlineIndex: %d", candidateInfo.mid.c_str(), candidateInfo.mlineIndex);
 }
 
 void WebrtcTransport::OnConnectionStateChanged(bool connected)

examples/camera-controller/webrtc-manager/WebRTCManager.cpp

@@ -277,13 +277,38 @@ CHIP_ERROR WebRTCManager::Connnect(Controller::DeviceCommissioner & commissioner
         mLocalDescription = std::string(desc);
         ChipLogProgress(Camera, "Local Description:");
         ChipLogProgress(Camera, "%s", mLocalDescription.c_str());
+
+        // Extract any candidates embedded in the SDP description
+        std::vector<rtc::Candidate> candidates = desc.candidates();
+        ChipLogProgress(Camera, "Extracted %lu candidates from SDP description", candidates.size());
+
+        for (const auto & candidate : candidates)
+        {
+            ICECandidateInfo candidateInfo;
+            candidateInfo.candidate  = std::string(candidate);
+            candidateInfo.mid        = candidate.mid();
+            candidateInfo.mlineIndex = -1; // libdatachannel doesn't provide mlineIndex
+
+            ChipLogProgress(Camera, "[From SDP] Candidate: %s, mid: %s", candidateInfo.candidate.c_str(),
+                            candidateInfo.mid.c_str());
+
+            mLocalCandidates.push_back(candidateInfo);
+        }
     });
 
     mPeerConnection->onLocalCandidate([this](rtc::Candidate candidate) {
-        std::string candidateStr = std::string(candidate);
-        mLocalCandidates.push_back(candidateStr);
+        ICECandidateInfo candidateInfo;
+        candidateInfo.candidate = std::string(candidate);
+        candidateInfo.mid       = candidate.mid();
+
+        // Note: libdatachannel doesn't directly provide mlineIndex, so we use -1 to indicate it is not present.
+        candidateInfo.mlineIndex = -1;
+
         ChipLogProgress(Camera, "Local Candidate:");
-        ChipLogProgress(Camera, "%s", candidateStr.c_str());
+        ChipLogProgress(Camera, "%s", candidateInfo.candidate.c_str());
+        ChipLogProgress(Camera, "  mid: %s, mlineIndex: %d", candidateInfo.mid.c_str(), candidateInfo.mlineIndex);
+
+        mLocalCandidates.push_back(candidateInfo);
     });
 
     mPeerConnection->onStateChange([this](rtc::PeerConnection::State state) {

examples/camera-controller/webrtc-manager/WebRTCManager.h

@@ -26,6 +26,13 @@
 #include <webrtc-manager/WebRTCProviderClient.h>
 #include <webrtc-manager/WebRTCRequestorDelegate.h>
 
+struct ICECandidateInfo
+{
+    std::string candidate;
+    std::string mid;
+    int mlineIndex;
+};
+
 class WebRTCManager
 {
 public:
@@ -87,8 +94,8 @@ class WebRTCManager
     uint16_t mPendingSessionId = 0;
     std::string mLocalDescription;
 
-    // Local vector to store the ICE Candidate strings coming from the WebRTC stack
-    std::vector<std::string> mLocalCandidates;
+    // Local vector to store the ICE Candidate info coming from the WebRTC stack
+    std::vector<ICECandidateInfo> mLocalCandidates;
 
     std::shared_ptr<rtc::Track> mTrack;
     std::shared_ptr<rtc::Track> mAudioTrack;

examples/camera-controller/webrtc-manager/WebRTCProviderClient.cpp

@@ -162,7 +162,7 @@ CHIP_ERROR WebRTCProviderClient::ProvideAnswer(uint16_t webRTCSessionId, const s
     return CHIP_NO_ERROR;
 }
 
-CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId, const std::vector<std::string> & iceCandidates)
+CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId, const std::vector<ICECandidateInfo> & iceCandidates)
 {
     ChipLogProgress(Camera, "Sending ProvideICECandidates to node " ChipLogFormatX64, ChipLogValueX64(mPeerId.GetNodeId()));
 
@@ -175,14 +175,38 @@ CHIP_ERROR WebRTCProviderClient::ProvideICECandidates(uint16_t webRTCSessionId,
     // Store the command type
     mCommandType = CommandType::kProvideICECandidates;
 
-    // Store ICE Candidates.
+    // Store ICE Candidates for lifetime management
     mClientICECandidates = iceCandidates;
+    mICECandidateStructList.clear();
 
-    for (const auto & candidate : mClientICECandidates)
+    for (const auto & candidateInfo : mClientICECandidates)
     {
-        ICECandidateStruct iceCandidate = { CharSpan::fromCharString(candidate.c_str()) };
+        ICECandidateStruct iceCandidate;
+        iceCandidate.candidate = CharSpan(candidateInfo.candidate.data(), candidateInfo.candidate.size());
+
+        // Set SDPMid if available
+        if (!candidateInfo.mid.empty())
+        {
+            iceCandidate.SDPMid.SetNonNull(CharSpan(candidateInfo.mid.data(), candidateInfo.mid.size()));
+        }
+        else
+        {
+            iceCandidate.SDPMid.SetNull();
+        }
+
+        // Set SDPMLineIndex if valid
+        if (candidateInfo.mlineIndex >= 0)
+        {
+            iceCandidate.SDPMLineIndex.SetNonNull(static_cast<uint16_t>(candidateInfo.mlineIndex));
+        }
+        else
+        {
+            iceCandidate.SDPMLineIndex.SetNull();
+        }
+
         mICECandidateStructList.push_back(iceCandidate);
     }
+
     // Stash data in class members so the CommandSender can safely reference them async
     mProvideICECandidatesData.webRTCSessionID = webRTCSessionId;
     mProvideICECandidatesData.ICECandidates =

examples/camera-controller/webrtc-manager/WebRTCProviderClient.h

@@ -23,6 +23,9 @@
 #include <app/clusters/webrtc-transport-requestor-server/webrtc-transport-requestor-cluster.h>
 #include <controller/CHIPDeviceController.h>
 
+// Forward declaration
+struct ICECandidateInfo;
+
 /**
  * @brief This class handles sending CHIP commands for WebRTCTransportProvider cluster, including
  * sending a ProvideOffer command to a remote camera device.
@@ -111,17 +114,17 @@ class WebRTCProviderClient : public chip::app::CommandSender::Callback
      * @brief Sends a ProvideICECandidates command to the remote device.
      *
      * This method populates the ProvideICECandidates command parameters, packages the provided ICE
-     * candidate strings, and queues them for sending to the target device. This is typically used
-     * to inform the remote side about potential network endpoints it can use to establish or
-     * enhance a WebRTC session.
+     * candidate information including SDPMid and SDPMLineIndex, and queues them for sending to the
+     * target device. This is typically used to inform the remote side about potential network
+     * endpoints it can use to establish or enhance a WebRTC session.
      *
      * @param webRTCSessionId   The unique identifier for the WebRTC session to which these
      *                          ICE candidates apply.
-     * @param ICECandidates     A list of ICE candidate structs.
+     * @param iceCandidates     A list of ICE candidate structs.
      *
      * @return CHIP_NO_ERROR on success, or an appropriate CHIP_ERROR on failure.
      */
-    CHIP_ERROR ProvideICECandidates(uint16_t webRTCSessionId, const std::vector<std::string> & iceCandidates);
+    CHIP_ERROR ProvideICECandidates(uint16_t webRTCSessionId, const std::vector<ICECandidateInfo> & iceCandidates);
 
     /**
      * @brief Notify WebRTCProviderClient that the Offer command has been received.
@@ -231,7 +234,7 @@ class WebRTCProviderClient : public chip::app::CommandSender::Callback
     std::string mSdpString;
 
     // Store the ICECandidates here to use to send asynchronously.
-    std::vector<std::string> mClientICECandidates;
+    std::vector<ICECandidateInfo> mClientICECandidates;
     std::vector<ICECandidateStruct> mICECandidateStructList;
 
     chip::Callback::Callback<chip::OnDeviceConnected> mOnConnectedCallback;