[React] Implement multi-participant handling & state

client-react/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to **Pipecat Client React** will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- **Multi-participant support**: Full support for handling multiple participants in sessions
+  - New `usePipecatClientParticipant()` hook to access individual participant data
+  - New `usePipecatClientParticipantIds()` hook to get all participant IDs with filtering options
+  - Support for both regular and screen share media tracks per participant
+
+### Changed
+
+- `PipecatClientVideo` component now requires `participantId` prop when `participant` is set to `"remote"`
+- Enhanced `PipecatClientAudio` component now renders audio for all participants automatically
+- Enhanced `PipecatClientVideo` component now supports remote participants
+- `usePipecatClientMediaTrack()` hook signature updated to support remote participants
+
 ## [1.0.1]
 
 - Fixed state synchronization between different instances of `usePipecatClientCamControl()`, `usePipecatClientMicControl()` and `usePipecatClientTransportState()` ([#125](https://github.com/pipecat-ai/pipecat-client-web/pull/125))

client-react/src/PipecatClientAudio.tsx

@@ -5,38 +5,112 @@
  */
 
 import { RTVIEvent } from "@pipecat-ai/client-js";
-import { useCallback, useEffect, useRef } from "react";
+import React, { useCallback, useEffect, useRef } from "react";
 
 import { usePipecatClientMediaTrack } from "./usePipecatClientMediaTrack";
+import { usePipecatClientParticipantIds } from "./usePipecatClientParticipantIds";
 import { useRTVIClientEvent } from "./useRTVIClientEvent";
 
-export const PipecatClientAudio = () => {
-  const botAudioRef = useRef<HTMLAudioElement>(null);
-  const botAudioTrack = usePipecatClientMediaTrack("audio", "bot");
+interface AudioElementProps
+  extends React.AudioHTMLAttributes<HTMLAudioElement> {
+  participantId: string;
+  track: MediaStreamTrack | null;
+}
+
+const AudioElement = ({
+  participantId,
+  track,
+  ...props
+}: AudioElementProps) => {
+  const audioRef = useRef<HTMLAudioElement>(null);
 
   useEffect(() => {
-    if (!botAudioRef.current || !botAudioTrack) return;
-    if (botAudioRef.current.srcObject) {
+    if (!audioRef.current || !track) return;
+    if (audioRef.current.srcObject) {
       const oldTrack = (
-        botAudioRef.current.srcObject as MediaStream
+        audioRef.current.srcObject as MediaStream
       ).getAudioTracks()[0];
-      if (oldTrack.id === botAudioTrack.id) return;
+      if (oldTrack.id === track.id) return;
     }
-    botAudioRef.current.srcObject = new MediaStream([botAudioTrack]);
-  }, [botAudioTrack]);
+    audioRef.current.srcObject = new MediaStream([track]);
+  }, [track]);
 
   useRTVIClientEvent(
     RTVIEvent.SpeakerUpdated,
     useCallback((speaker: MediaDeviceInfo) => {
-      if (!botAudioRef.current) return;
-      if (typeof botAudioRef.current.setSinkId !== "function") return;
-      botAudioRef.current.setSinkId(speaker.deviceId);
+      if (!audioRef.current) return;
+      if (typeof audioRef.current.setSinkId !== "function") return;
+      audioRef.current.setSinkId(speaker.deviceId);
     }, [])
   );
 
+  return (
+    <audio
+      ref={audioRef}
+      autoPlay
+      data-participant-id={participantId}
+      {...props}
+    />
+  );
+};
+
+/**
+ * Component for individual participant audio
+ */
+const ParticipantAudio = ({ participantId }: { participantId: string }) => {
+  // Determine participant type and get appropriate tracks
+  const isLocal = participantId === "local";
+  const isBot = participantId === "bot";
+  const isRemote = !isLocal && !isBot;
+
+  const audioTrack = usePipecatClientMediaTrack(
+    "audio",
+    isLocal ? "local" : isBot ? "bot" : "remote",
+    isRemote ? participantId : undefined
+  );
+
+  const screenAudioTrack = usePipecatClientMediaTrack(
+    "screenAudio",
+    isLocal ? "local" : "remote",
+    isRemote ? participantId : undefined
+  );
+
+  return (
+    <>
+      <AudioElement
+        data-bot={isBot}
+        data-local={isLocal}
+        data-remote={isRemote}
+        data-track-type="audio"
+        participantId={participantId}
+        track={audioTrack}
+      />
+      {screenAudioTrack && (
+        <AudioElement
+          data-bot={isBot}
+          data-local={isLocal}
+          data-remote={isRemote}
+          data-track-type="screenAudio"
+          participantId={`${participantId}-screen`}
+          track={screenAudioTrack}
+        />
+      )}
+    </>
+  );
+};
+
+/**
+ * Component that renders all participant audio.
+ */
+export const PipecatClientAudio = () => {
+  const { participantIds } = usePipecatClientParticipantIds(false, true);
+
   return (
     <>
-      <audio ref={botAudioRef} autoPlay />
+      {/* All participant audio */}
+      {participantIds.map((participantId) => (
+        <ParticipantAudio key={participantId} participantId={participantId} />
+      ))}
     </>
   );
 };

client-react/src/PipecatClientParticipantManager.tsx

@@ -0,0 +1,203 @@
+/**
+ * Copyright (c) 2025, Daily.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+import { Participant, RTVIEvent, Tracks } from "@pipecat-ai/client-js";
+import { atom } from "jotai";
+import { atomFamily, useAtomCallback } from "jotai/utils";
+import { memo, useCallback } from "react";
+
+import { useRTVIClientEvent } from "./useRTVIClientEvent";
+
+// Remote participants tracking
+export const remoteParticipantIdsAtom = atom<string[]>([]);
+
+// Bot ID mapping - maps the bot's UUID to our internal "bot" identifier
+const botIdAtom = atom<string | null>(null);
+
+const localParticipantAtom = atom<Participant | null>(null);
+const botParticipantAtom = atom<Participant | null>(null);
+
+export const participantAtom = atomFamily((participantId: string) =>
+  atom<Participant | null>((get) => {
+    // Handle special cases for local and bot participants
+    if (participantId === "local") {
+      return get(localParticipantAtom);
+    }
+
+    if (participantId === "bot") {
+      return get(botParticipantAtom);
+    }
+
+    // For remote participants, return the stored value
+    return get(remoteParticipantAtom(participantId));
+  })
+);
+
+// Keep the original remoteParticipantAtom for internal use
+const remoteParticipantAtom = atomFamily(() => atom<Participant | null>(null));
+
+// Unified track atom family for all participants and track types
+type TrackType = keyof Tracks["local"];
+type TrackKey = `${string}:${TrackType}`;
+
+export const trackAtom = atomFamily((key: TrackKey) => {
+  // Create a unique atom for each participant/track combination
+  // Key format: "participantId:trackType"
+  void key; // Acknowledge the key parameter
+  return atom<MediaStreamTrack | null>(null);
+});
+
+/**
+ * Component that manages participant events globally.
+ */
+export const PipecatClientParticipantManager = memo(() => {
+  const addParticipant = useAtomCallback(
+    useCallback((get, set, participant: Participant) => {
+      if (participant.local) {
+        set(localParticipantAtom, participant);
+        return;
+      }
+
+      if (participant.id === get(botIdAtom)) {
+        set(botParticipantAtom, participant);
+        return;
+      }
+
+      const currentIds = get(remoteParticipantIdsAtom);
+      if (!currentIds.includes(participant.id)) {
+        set(remoteParticipantIdsAtom, [...currentIds, participant.id]);
+      }
+      set(remoteParticipantAtom(participant.id), participant);
+    }, [])
+  );
+
+  const removeParticipant = useAtomCallback(
+    useCallback((get, set, participant: Participant) => {
+      if (participant.local) {
+        set(localParticipantAtom, null);
+        return;
+      }
+
+      if (participant.id === get(botIdAtom)) {
+        set(botParticipantAtom, null);
+        return;
+      }
+
+      const currentIds = get(remoteParticipantIdsAtom);
+      set(
+        remoteParticipantIdsAtom,
+        currentIds.filter((id) => id !== participant.id)
+      );
+
+      // Clean up participant data
+      set(remoteParticipantAtom(participant.id), null);
+
+      // Clean up all track types for this participant
+      const trackTypes: TrackType[] = [
+        "audio",
+        "video",
+        "screenAudio",
+        "screenVideo",
+      ];
+      trackTypes.forEach((trackType) => {
+        const atom = trackAtom(`${participant.id}:${trackType}`);
+        set(atom, null);
+      });
+    }, [])
+  );
+
+  // Set up event listeners for participant events
+  useRTVIClientEvent(
+    RTVIEvent.ParticipantConnected,
+    useCallback(
+      (participant: Participant) => {
+        addParticipant(participant);
+      },
+      [addParticipant]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.ParticipantLeft,
+    useCallback(
+      (participant: Participant) => {
+        removeParticipant(participant);
+      },
+      [removeParticipant]
+    )
+  );
+
+  // Handle bot connection to map bot UUID to our internal "bot" identifier
+  useRTVIClientEvent(
+    RTVIEvent.BotConnected,
+    useAtomCallback(
+      useCallback((_get, set, participant: Participant) => {
+        set(botIdAtom, participant.id);
+      }, [])
+    )
+  );
+
+  // Set up event listeners for media track events
+  const handleTrackStarted = useAtomCallback(
+    useCallback(
+      (get, set, track: MediaStreamTrack, participant?: Participant) => {
+        if (!participant) return;
+
+        const trackType = track.kind as TrackType;
+        // Map participant to our internal ID system
+        let internalId: string;
+        if (participant.local) {
+          internalId = "local";
+        } else {
+          // Check if this is the bot by comparing with stored bot ID
+          const botId = get(botIdAtom);
+          internalId =
+            botId && participant.id === botId ? "bot" : participant.id;
+        }
+        // Update track directly
+        const atom = trackAtom(`${internalId}:${trackType}`);
+        const oldTrack = get(atom);
+        if (oldTrack?.id === track.id) return;
+        set(atom, track);
+      },
+      []
+    )
+  );
+
+  const handleScreenTrackStarted = useAtomCallback(
+    useCallback(
+      (get, set, track: MediaStreamTrack, participant?: Participant) => {
+        if (!participant) return;
+
+        const trackType =
+          track.kind === "audio" ? "screenAudio" : "screenVideo";
+        // Map participant to our internal ID system
+        let internalId: string;
+        if (participant.local) {
+          internalId = "local";
+        } else {
+          // Check if this is the bot by comparing with stored bot ID
+          const botId = get(botIdAtom);
+          internalId =
+            botId && participant.id === botId ? "bot" : participant.id;
+        }
+        // Update track directly
+        const atom = trackAtom(`${internalId}:${trackType}`);
+        const oldTrack = get(atom);
+        if (oldTrack?.id === track.id) return;
+        set(atom, track);
+      },
+      []
+    )
+  );
+
+  useRTVIClientEvent(RTVIEvent.TrackStarted, handleTrackStarted);
+  useRTVIClientEvent(RTVIEvent.ScreenTrackStarted, handleScreenTrackStarted);
+
+  return null;
+});
+
+PipecatClientParticipantManager.displayName = "PipecatClientParticipantManager";

client-react/src/PipecatClientProvider.tsx

@@ -18,6 +18,7 @@ import {
   name as packageName,
   version as packageVersion,
 } from "../package.json";
+import { PipecatClientParticipantManager } from "./PipecatClientParticipantManager";
 import { PipecatClientStateProvider } from "./PipecatClientState";
 import { RTVIEventContext } from "./RTVIEventContext";
 
@@ -116,7 +117,10 @@ export const PipecatClientProvider: React.FC<
     <JotaiProvider store={jotaiStore}>
       <PipecatClientContext.Provider value={{ client }}>
         <RTVIEventContext.Provider value={{ on, off }}>
-          <PipecatClientStateProvider>{children}</PipecatClientStateProvider>
+          <PipecatClientStateProvider>
+            {children}
+            <PipecatClientParticipantManager />
+          </PipecatClientStateProvider>
         </RTVIEventContext.Provider>
       </PipecatClientContext.Provider>
     </JotaiProvider>