diff --git a/client/package.json b/client/package.json index a0a429bcd4..1024f7ea2d 100644 --- a/client/package.json +++ b/client/package.json @@ -57,7 +57,7 @@ "shelljs": "^0.8.5", "svgo": "^3.0.0", "ts-node": "^10.4.0", - "typescript": "^4.8.4", + "typescript": "^5.6.2", "whatwg-fetch": "^3.6.2" }, "lint-staged": { @@ -73,6 +73,7 @@ "eslint-plugin-only-ascii@^0.0.0": "patch:eslint-plugin-only-ascii@npm%3A0.0.0#./.yarn/patches/eslint-plugin-only-ascii-npm-0.0.0-29e3417685.patch" }, "dependencies": { + "@kinde-oss/kinde-auth-pkce-js": "^4.3.0", "@lottiefiles/react-lottie-player": "^3.5.3", "@remixicon/react": "^4.1.1", "@supabase/supabase-js": "^2.45.4", diff --git a/client/packages/lowcoder-comps/package.json b/client/packages/lowcoder-comps/package.json index 6d61b820fc..bc3c907b01 100644 --- a/client/packages/lowcoder-comps/package.json +++ b/client/packages/lowcoder-comps/package.json @@ -17,8 +17,8 @@ "@fullcalendar/resource-timeline": "^6.1.11", "@fullcalendar/timegrid": "^6.1.6", "@fullcalendar/timeline": "^6.1.6", - "agora-rtc-sdk-ng": "^4.20.2", - "agora-rtm-sdk": "^1.5.1", + "agora-rtc-sdk-ng": "^4.24.3", + "agora-rtm-sdk": "^2.2.4", "big.js": "^6.2.1", "echarts-extension-gmap": "^1.6.0", "echarts-gl": "^2.0.9", diff --git a/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingControllerComp.tsx b/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingControllerComp.tsx index 9af1f3f6cb..39ebda863d 100644 --- a/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingControllerComp.tsx +++ b/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingControllerComp.tsx @@ -33,7 +33,6 @@ import { MeetingEventHandlerControl, } from "lowcoder-sdk"; import { default as CloseOutlined } from "@ant-design/icons/CloseOutlined"; -import type { JSONValue } from "../../../../lowcoder/src/util/jsonTypes"; // import { default as Button } from "antd/es/button"; const EventOptions = [closeEvent] as const; @@ -49,12 +48,14 @@ import AgoraRTC, { type IAgoraRTCRemoteUser, type UID, type ILocalVideoTrack, + type IRemoteVideoTrack, } from "agora-rtc-sdk-ng"; -import type { RtmChannel, RtmClient } from "agora-rtm-sdk"; -import { useCallback, useEffect, useState } from "react"; +import AgoraRtmSdk, { RTMEvents } from "agora-rtm-sdk"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ResizeHandle } from "react-resizable"; import { v4 as uuidv4 } from "uuid"; +import { meetingShareElementId } from "./meetingStreamUtils"; const DEFAULT_SIZE = 378; const DEFAULT_PADDING = 16; @@ -77,12 +78,54 @@ AgoraRTC.setLogLevel(4); 4: NONE. Do not output any log. */ -let audioTrack: IMicrophoneAudioTrack; -let videoTrack: ICameraVideoTrack; -let screenShareStream: ILocalVideoTrack; +let audioTrack: IMicrophoneAudioTrack | undefined; +let videoTrack: ICameraVideoTrack | undefined; +let screenShareStream: ILocalVideoTrack | undefined; +let screenShareTrackEndedHandler: (() => void) | null = null; +let screenShareTeardownInFlight = false; let userId: UID | null | undefined; -let rtmChannelResponse: RtmChannel; -let rtmClient: RtmClient; +let rtmClient: InstanceType | undefined; +/** Serializes leave/join across app navigations (singleton RTC client). */ +let channelLeavePromise: Promise = Promise.resolve(); +/** MESSAGE channel name subscribed after login (same as RTC channel / meeting name). */ +let rtmSubscribedChannelName: string | null = null; +/** RTM channel payload: sync local mic/camera to other clients (setEnabled may not fire user-unpublished). */ +const RTM_MEETING_USER_STATE = "meetingUserState" as const; +/** Ask everyone on the channel to re-send meetingUserState (joiner may have missed earlier RTM). */ +const RTM_MEETING_REQUEST_PRESENCE = "meetingRequestPresence" as const; + +const rtmMessageSinkRef: { + current: null | ((event: RTMEvents.MessageEvent) => void); +} = { current: null }; + +function onRtmMessage(event: RTMEvents.MessageEvent) { + rtmMessageSinkRef.current?.(event); +} + +/** Invokes broadcastLocalMeetingUserState with latest mic/video/name snapshot (see React effect). */ +const presenceRequestSinkRef: { current: null | (() => void) } = { + current: null, +}; + +/** Sync sharing control + RTM when the browser stops screen capture (track-ended). */ +const screenShareEndedSinkRef: { current: null | (() => void) } = { + current: null, +}; + +/** Dedupe by stringified id so RTC numeric uid and RTM string uid stay one row. */ +function meetingParticipantsDedupe(arr: any[], prop: string) { + const byKey = new Map(); + for (const obj of arr) { + if (obj == null || obj[prop] === undefined || obj[prop] === null) { + continue; + } + const key = String(obj[prop]); + const prev = byKey.get(key); + byKey.set(key, prev ? { ...prev, ...obj } : { ...obj }); + } + return Array.from(byKey.values()); +} + // const ButtonStyle = styledm(Button)` // position: absolute; // left: 0; @@ -111,32 +154,240 @@ let rtmClient: RtmClient; // color: rgba(0, 0, 0, 0.75); // } // `; +function isClientInChannel(): boolean { + const state = client.connectionState; + return ( + state === "CONNECTED" || + state === "CONNECTING" || + state === "RECONNECTING" || + state === "DISCONNECTING" + ); +} + +function isCameraTrackUsable( + track: ICameraVideoTrack | undefined +): track is ICameraVideoTrack { + return !!track && !(track as { closed?: boolean }).closed; +} + +function isAudioTrackUsable( + track: IMicrophoneAudioTrack | undefined +): track is IMicrophoneAudioTrack { + return !!track && !(track as { closed?: boolean }).closed; +} + const turnOnCamera = async (flag?: boolean) => { - if (videoTrack) { + if (isCameraTrackUsable(videoTrack)) { return videoTrack.setEnabled(flag!); } videoTrack = await AgoraRTC.createCameraVideoTrack(); - videoTrack.play(userId + ""); + if (userId != null && userId !== "") { + videoTrack.play(userId + ""); + } }; const turnOnMicrophone = async (flag?: boolean) => { - if (audioTrack) { - return audioTrack.setEnabled(flag!); + if (isAudioTrackUsable(audioTrack)) { + await audioTrack.setEnabled(flag!); + if (flag) { + try { + await client.publish(audioTrack); + } catch { + /* already published */ + } + } else { + try { + await client.unpublish(audioTrack); + } catch { + /* already unpublished */ + } + } + return; } audioTrack = await AgoraRTC.createMicrophoneAudioTrack(); - if (!flag) { - await client.unpublish(audioTrack); - } else { + if (flag) { await client.publish(audioTrack); } }; +function playTrackWhenElementReady( + track: ILocalVideoTrack, + elementId: string, + attempt = 0 +) { + const el = document.getElementById(elementId); + if (el) { + try { + track.stop(); + } catch { + /* not playing yet */ + } + track.play(elementId); + return; + } + if (attempt < 30) { + requestAnimationFrame(() => + playTrackWhenElementReady(track, elementId, attempt + 1) + ); + } +} + +type ScreenShareAttachState = { + elementId: string; + track: ILocalVideoTrack | IRemoteVideoTrack; +}; + +const screenShareAttachByUid = new Map(); + +function clearScreenShareAttach(targetUid: string) { + screenShareAttachByUid.delete(targetUid); +} + +function isScreenShareAttached( + targetUid: string, + elementId: string, + track: ILocalVideoTrack | IRemoteVideoTrack +): boolean { + const prev = screenShareAttachByUid.get(targetUid); + return ( + prev?.elementId === elementId && + prev.track === track && + track.isPlaying + ); +} + +/** Attach local or remote screen-share video to the tile for `targetUid`. */ +export async function playScreenShareToElement( + targetUid: string, + isLocalPublisher: boolean +): Promise { + const elementId = meetingShareElementId(targetUid); + const el = document.getElementById(elementId); + if (!el) { + return false; + } + + try { + if (isLocalPublisher) { + const track = + screenShareStream ?? + (client.localTracks.find( + (t) => t.trackMediaType === "video" + ) as ILocalVideoTrack | undefined); + if (!track) { + clearScreenShareAttach(targetUid); + return false; + } + if (isScreenShareAttached(targetUid, elementId, track)) { + return true; + } + try { + track.stop(); + } catch { + /* ignore */ + } + track.play(elementId); + screenShareAttachByUid.set(targetUid, { elementId, track }); + return true; + } + + const user = client.remoteUsers.find((u) => String(u.uid) === targetUid); + if (!user?.hasVideo) { + clearScreenShareAttach(targetUid); + return false; + } + const track = + user.videoTrack ?? (await client.subscribe(user, "video")); + if (isScreenShareAttached(targetUid, elementId, track)) { + return true; + } + try { + track.stop(); + } catch { + /* ignore */ + } + track.play(elementId); + screenShareAttachByUid.set(targetUid, { elementId, track }); + return true; + } catch { + clearScreenShareAttach(targetUid); + return false; + } +} + +function unbindScreenShareTrackEnded() { + if (screenShareStream && screenShareTrackEndedHandler) { + try { + screenShareStream.off("track-ended", screenShareTrackEndedHandler); + } catch { + /* ignore */ + } + } + screenShareTrackEndedHandler = null; +} + +function bindScreenShareTrackEnded(track: ILocalVideoTrack) { + unbindScreenShareTrackEnded(); + screenShareTrackEndedHandler = () => { + void handleScreenShareStoppedByBrowser(); + }; + track.on("track-ended", screenShareTrackEndedHandler); +} + +async function teardownScreenShareTrack(): Promise { + unbindScreenShareTrackEnded(); + const track = screenShareStream; + if (!track) { + return; + } + screenShareStream = undefined; + try { + await client.unpublish(track); + } catch { + /* already unpublished when the browser ended capture */ + } + try { + track.close(); + } catch { + /* ignore */ + } + if (userId != null && userId !== "") { + clearScreenShareAttach(String(userId)); + } +} + +async function republishCameraAfterScreenShare(): Promise { + if (!videoTrack) { + return; + } + try { + await client.publish(videoTrack); + if (userId != null && userId !== "") { + videoTrack.play(userId + ""); + } + } catch { + /* ignore */ + } +} + +async function handleScreenShareStoppedByBrowser(): Promise { + if (screenShareTeardownInFlight) { + return; + } + screenShareTeardownInFlight = true; + try { + await teardownScreenShareTrack(); + await republishCameraAfterScreenShare(); + screenShareEndedSinkRef.current?.(); + } finally { + screenShareTeardownInFlight = false; + } +} + const shareScreen = async (sharing: boolean) => { try { if (sharing === false) { - await client.unpublish(screenShareStream); - screenShareStream.close(); - await client.publish(videoTrack); - videoTrack.play(userId + ""); + await teardownScreenShareTrack(); + await republishCameraAfterScreenShare(); } else { screenShareStream = await AgoraRTC.createScreenVideoTrack( { @@ -144,66 +395,201 @@ const shareScreen = async (sharing: boolean) => { }, "disable" ); - await client.unpublish(videoTrack); - screenShareStream.play("share-screen"); + bindScreenShareTrackEnded(screenShareStream); + if (videoTrack) { + await client.unpublish(videoTrack); + } await client.publish(screenShareStream); + if (userId != null && userId !== "") { + playTrackWhenElementReady( + screenShareStream, + meetingShareElementId(userId) + ); + } } } catch (error) { console.error("Failed to create screen share stream:", error); + screenShareStream = undefined; + unbindScreenShareTrackEnded(); } }; -const leaveChannel = async () => { - //stops local sharing video +async function closeLocalMediaTracks(): Promise { + unbindScreenShareTrackEnded(); + if (screenShareStream) { - screenShareStream.close(); + try { + await client.unpublish(screenShareStream); + } catch { + /* ignore */ + } + try { + screenShareStream.close(); + } catch { + /* ignore */ + } + screenShareStream = undefined; } - //stops local video streaming and puts off the camera if (videoTrack) { - await client.unpublish(videoTrack); - await turnOnCamera(false); + try { + await client.unpublish(videoTrack); + } catch { + /* ignore */ + } + try { + videoTrack.stop(); + videoTrack.close(); + } catch { + /* ignore */ + } + videoTrack = undefined; } - //mutes and stops locla audio stream if (audioTrack) { - await turnOnMicrophone(false); + try { + await client.unpublish(audioTrack); + } catch { + /* ignore */ + } + try { + audioTrack.stop(); + audioTrack.close(); + } catch { + /* ignore */ + } + audioTrack = undefined; } - await client.leave(); - await rtmChannelResponse.leave(); + + screenShareAttachByUid.clear(); +} + +async function teardownRtmClient(): Promise { + if (rtmClient && rtmSubscribedChannelName) { + try { + await rtmClient.unsubscribe(rtmSubscribedChannelName); + } catch { + /* ignore */ + } + rtmSubscribedChannelName = null; + } + if (rtmClient) { + try { + rtmClient.removeEventListener("message", onRtmMessage); + } catch { + /* ignore */ + } + try { + await rtmClient.logout(); + } catch { + /* ignore */ + } + rtmClient = undefined; + } +} + +async function leaveChannelInternal(): Promise { + await closeLocalMediaTracks(); + if (isClientInChannel()) { + try { + await client.leave(); + } catch { + /* ignore */ + } + } + await teardownRtmClient(); +} + +const leaveChannel = async () => { + channelLeavePromise = channelLeavePromise + .catch(() => {}) + .then(() => leaveChannelInternal()); + return channelLeavePromise; }; +async function ensureChannelLeft(): Promise { + await channelLeavePromise.catch(() => {}); + if (isClientInChannel()) { + await leaveChannel(); + await channelLeavePromise.catch(() => {}); + } +}; + +/** Joins RTC + RTM; publishes camera only if creation succeeds (e.g. permission granted). */ const publishVideo = async ( appId: string, channel: string, rtmToken: string, rtcToken: string -) => { - await turnOnCamera(true); +): Promise => { + await ensureChannelLeft(); + videoTrack = undefined; + audioTrack = undefined; await client.join(appId, channel, rtcToken, userId); - await client.publish(videoTrack); + let videoPublished = false; + try { + await turnOnCamera(true); + if (isCameraTrackUsable(videoTrack)) { + await client.publish(videoTrack); + videoPublished = true; + } + } catch (error) { + console.warn( + "Meeting: camera unavailable (permission denied or no device); joining without video.", + error + ); + } await rtmInit(appId, userId, rtmToken, channel); + return videoPublished; }; const sendMessageRtm = (message: any) => { - rtmChannelResponse.sendMessage({ text: JSON.stringify(message) }); + if (!rtmClient || !rtmSubscribedChannelName) return; + void rtmClient + .publish(rtmSubscribedChannelName, JSON.stringify(message)) + .catch(() => {}); }; +function broadcastLocalMeetingUserState(payload: { + user: string; + audiostatus: boolean; + streamingVideo: boolean; + streamingSharing?: boolean; + speaking?: boolean; + userName?: string; +}) { + sendMessageRtm({ + type: RTM_MEETING_USER_STATE, + time: Date.now(), + user: payload.user, + audiostatus: payload.audiostatus, + streamingVideo: payload.streamingVideo, + streamingSharing: payload.streamingSharing ?? false, + speaking: payload.speaking ?? false, + userName: payload.userName ?? "", + }); +} + +function sendMeetingPresenceRequest() { + sendMessageRtm({ + type: RTM_MEETING_REQUEST_PRESENCE, + time: Date.now(), + }); +} + const sendPeerMessageRtm = (message: any, toId: string) => { - rtmClient.sendMessageToPeer({ text: JSON.stringify(message) }, toId); + if (!rtmClient) return; + void rtmClient + .publish(String(toId), JSON.stringify(message), { channelType: "USER" }) + .catch(() => {}); }; const rtmInit = async (appId: any, uid: any, token: any, channel: any) => { - const AgoraRTM = (await import("agora-rtm-sdk")).default; - rtmClient = AgoraRTM.createInstance(appId); - let options = { - uid: String(uid), - token: token ? token : null, - }; - await rtmClient.login(options); - - rtmChannelResponse = rtmClient.createChannel(channel); - - await rtmChannelResponse.join(); + rtmClient = new AgoraRtmSdk.RTM(String(appId), String(uid)); + await rtmClient.login({ token: token ? String(token) : undefined }); + const channelName = String(channel); + await rtmClient.subscribe(channelName); + rtmSubscribedChannelName = channelName; + rtmClient.addEventListener("message", onRtmMessage); }; const CanvasContainerID = "__canvas_container__"; @@ -224,20 +610,24 @@ const meetingControllerChildren = { endCall: withDefault(BooleanStateControl, "false"), sharing: withDefault(BooleanStateControl, "false"), appId: withDefault(StringControl, trans("meeting.appid")), - participants: stateComp([]), - usersScreenShared: stateComp([]), + participants: (stateComp as any)([]) as ReturnType, + usersScreenShared: (stateComp as any)([]) as ReturnType, localUser: jsonObjectExposingStateControl(""), localUserID: withDefault( stringStateControl(trans("meeting.localUserID")), uuidv4() + "" ), + localUserName: withDefault( + stringStateControl(trans("meeting.localUserName")), + "" + ), meetingName: withDefault( stringStateControl(trans("meeting.meetingName")), uuidv4() + "" ), rtmToken: stringStateControl(trans("meeting.rtmToken")), rtcToken: stringStateControl(trans("meeting.rtcToken")), - messages: stateComp([]), + messages: (stateComp as any)([]) as ReturnType, }; let MeetingControllerComp = () => ( @@ -268,7 +658,12 @@ if (typeof ContainerCompBuilder === "function") { }, [dispatch, isTopBom] ); - const [userIds, setUserIds] = useState([]); + const [localParticipants, setLocalParticipants] = useState( + () => { + const p = props.participants as any; + return Array.isArray(p) ? [...p] : []; + } + ); const [updateVolume, setUpdateVolume] = useState({ update: false, userid: null, @@ -277,110 +672,213 @@ if (typeof ContainerCompBuilder === "function") { const [localUserSpeaking, setLocalUserSpeaking] = useState(false); const [localUserVideo, setLocalUserVideo] = useState(); - const [userJoined, setUserJoined] = useState(); - const [userLeft, setUserLeft] = useState(); + + const sharingUserIdsRef = useRef([]); + + const latestMeetingBroadcastRef = useRef({ + meetingActive: false, + audiostatus: false, + streamingVideo: false, + streamingSharing: false, + speaking: false, + userName: "", + }); useEffect(() => { - if (userJoined) { - // console.log("userJoined ", userJoined); - - let prevUsers: any[] = props.participants as []; - // console.log("prevUsers ", prevUsers); - let userData = { - user: userJoined.uid, - audiostatus: userJoined.hasAudio, - streamingVideo: true, - }; - // console.log("userData ", userData); - setUserIds((userIds: any) => [...userIds, userData]); - // console.log("userIds ", userIds); - /* console.log( - "removeDuplicates ", - removeDuplicates(getData([...prevUsers, userData]).data, "user") - ); */ - dispatch( - changeChildAction( - "participants", - removeDuplicates( - getData([...prevUsers, userData]).data, - "user" - ), - false - ) + latestMeetingBroadcastRef.current = { + meetingActive: !!props.meetingActive.value, + audiostatus: !!props.audioControl.value, + streamingVideo: !!props.videoControl.value, + streamingSharing: !!props.sharing.value, + speaking: !!localUserSpeaking, + userName: String(props.localUserName.value ?? ""), + }; + }, [ + props.meetingActive.value, + props.audioControl.value, + props.videoControl.value, + props.sharing.value, + props.localUserName.value, + localUserSpeaking, + ]); + + useEffect(() => { + screenShareEndedSinkRef.current = () => { + if (!props.sharing.value) { + return; + } + props.sharing.onChange(false); + setLocalParticipants((prevUsers) => + prevUsers.map((userInfo: any) => { + if ( + userId != null && + userId !== "" && + String(userInfo.user) === String(userId) + ) { + return { ...userInfo, streamingSharing: false }; + } + return userInfo; + }) ); + const localObject = { + user: userId + "", + audiostatus: props.audioControl.value, + streamingVideo: props.videoControl.value, + streamingSharing: false, + speaking: localUserSpeaking, + userName: props.localUserName.value, + }; + props.localUser.onChange(localObject); + if ( + props.meetingActive.value && + userId != null && + userId !== "" + ) { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: props.audioControl.value, + streamingVideo: props.videoControl.value, + streamingSharing: false, + speaking: !!localUserSpeaking, + userName: props.localUserName.value, + }); + } + }; + return () => { + screenShareEndedSinkRef.current = null; + }; + }, [ + props.sharing, + props.audioControl.value, + props.videoControl.value, + props.meetingActive.value, + props.localUserName.value, + localUserSpeaking, + ]); + + useEffect(() => { + presenceRequestSinkRef.current = () => { + const snap = latestMeetingBroadcastRef.current; + if ( + !snap.meetingActive || + userId == null || + userId === "" + ) { + return; + } + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: snap.audiostatus, + streamingVideo: snap.streamingVideo, + streamingSharing: snap.streamingSharing, + speaking: snap.speaking, + userName: snap.userName, + }); + }; + return () => { + presenceRequestSinkRef.current = null; + }; + }, []); + + useEffect(() => { + if ( + !props.meetingActive.value || + userId == null || + userId === "" + ) { + return; } - }, [userJoined]); + if (!rtmClient || !rtmSubscribedChannelName) { + return; + } + sendMeetingPresenceRequest(); + }, [props.meetingActive.value]); - function removeDuplicates(arr: any, prop: any) { - const uniqueObjects = []; - const seenValues = new Set(); + useEffect(() => { + const exposed = + userId != null && userId !== "" + ? localParticipants.filter( + (u: any) => String(u.user) !== String(userId) + ) + : localParticipants; + dispatch( + changeChildAction( + "participants", + getData(exposed).data, + false + ) + ); + }, [localParticipants, dispatch]); - for (const obj of arr) { - const objValue = obj[prop]; + const sharingUserIdsKey = useMemo( + () => + localParticipants + .filter((p: any) => p.streamingSharing) + .map((p: any) => String(p.user)) + .sort() + .join(","), + [localParticipants] + ); - if (!seenValues.has(objValue)) { - seenValues.add(objValue); - uniqueObjects.push(obj); - } - } + useEffect(() => { + sharingUserIdsRef.current = sharingUserIdsKey + ? sharingUserIdsKey.split(",") + : []; + }, [sharingUserIdsKey]); - return uniqueObjects; - } + // Re-attach only when who is sharing changes — not on every speaking/RTM tick. useEffect(() => { - if (userLeft) { - let newUsers = userIds.filter( - (item: any) => item.user !== userLeft.uid - ); - let hostExists = newUsers.filter((f: any) => f.host === true); - if (hostExists.length == 0 && newUsers.length > 0) { - newUsers[0].host = true; - } - setUserIds(newUsers); - dispatch( - changeChildAction( - "participants", - removeDuplicates(getData(newUsers).data, "user"), - false - ) - ); + const ids = sharingUserIdsRef.current; + if (ids.length === 0) { + return; } - }, [userLeft]); + + const syncSharingTiles = () => { + for (const uid of ids) { + void playScreenShareToElement( + uid, + userId != null && + userId !== "" && + String(userId) === uid + ); + } + }; + + syncSharingTiles(); + const timers = [300, 1000, 2000].map((ms) => + window.setTimeout(syncSharingTiles, ms) + ); + return () => timers.forEach((id) => window.clearTimeout(id)); + }, [sharingUserIdsKey]); // console.log("sharing", props.sharing); useEffect(() => { - if (updateVolume.userid) { - let prevUsers: [] = props.participants as []; - - const updatedItems = prevUsers.map((userInfo: any) => { + if (!updateVolume.userid) return; + setLocalParticipants((prevUsers) => + prevUsers.map((userInfo: any) => { if ( - userInfo.user === updateVolume.userid && + String(userInfo.user) === String(updateVolume.userid) && userInfo.speaking != updateVolume.update ) { return { ...userInfo, speaking: updateVolume.update }; } return userInfo; - }); - dispatch( - changeChildAction( - "participants", - getData(updatedItems).data, - false - ) - ); - } + }) + ); }, [updateVolume]); useEffect(() => { - let prevUsers: [] = props.participants as []; - const updatedItems = prevUsers.map((userInfo: any) => { - if (userInfo.user === localUserVideo?.uid) { - return { ...userInfo, streamingSharing: props.sharing.value }; - } - return userInfo; - }); - dispatch( - changeChildAction("participants", getData(updatedItems).data, false) + setLocalParticipants((prevUsers) => + prevUsers.map((userInfo: any) => { + if ( + localUserVideo?.uid != null && + String(userInfo.user) === String(localUserVideo.uid) + ) { + return { ...userInfo, streamingSharing: props.sharing.value }; + } + return userInfo; + }) ); let localObject = { @@ -389,22 +887,38 @@ if (typeof ContainerCompBuilder === "function") { streamingVideo: props.videoControl.value, streamingSharing: props.sharing.value, speaking: localUserSpeaking, + userName: props.localUserName.value, }; props.localUser.onChange(localObject); - }, [props.sharing.value]); + + if (props.meetingActive.value && userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: props.audioControl.value, + streamingVideo: props.videoControl.value, + streamingSharing: props.sharing.value, + speaking: localUserSpeaking, + userName: props.localUserName.value, + }); + } + }, [props.sharing.value, props.localUserName.value]); // console.log("participants ", props.participants); useEffect(() => { - let prevUsers: [] = props.participants as []; - const updatedItems = prevUsers.map((userInfo: any) => { - if (userInfo.user === localUserVideo?.uid) { - return { ...userInfo, streamingVideo: localUserVideo?.hasVideo }; - } - return userInfo; - }); - dispatch( - changeChildAction("participants", getData(updatedItems).data, false) + setLocalParticipants((prevUsers) => + prevUsers.map((userInfo: any) => { + if ( + localUserVideo?.uid != null && + String(userInfo.user) === String(localUserVideo.uid) + ) { + return { + ...userInfo, + streamingVideo: localUserVideo?.hasVideo, + }; + } + return userInfo; + }) ); }, [localUserVideo?.hasVideo]); @@ -422,100 +936,276 @@ if (typeof ContainerCompBuilder === "function") { user: userId + "", audiostatus: props.audioControl.value, streamingVideo: props.videoControl.value, + streamingSharing: props.sharing.value, speaking: localUserSpeaking, + userName: props.localUserName.value, }; props.localUser.onChange(localObject); } - }, [localUserSpeaking]); + }, [localUserSpeaking, props.localUserName.value]); useEffect(() => { - if (rtmChannelResponse) { - rtmClient.on("MessageFromPeer", function (message, peerId) { - setRtmMessages((prevMessages: any[]) => { - // Check if the messages array exceeds the maximum limit - if (prevMessages.length >= 500) { - prevMessages.pop(); // Remove the oldest message - } - return [ - ...prevMessages, - { peermessage: JSON.parse(message.text + ""), from: peerId }, - ]; - }); - }); + if (!props.meetingActive.value || userId == null || userId === "") { + return; + } + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: props.audioControl.value, + streamingVideo: props.videoControl.value, + streamingSharing: props.sharing.value, + speaking: !!props.localUser.value?.speaking, + userName: props.localUserName.value, + }); + }, [props.localUserName.value]); - rtmChannelResponse.on( - "ChannelMessage", - function (message, memberId) { + useEffect(() => { + rtmMessageSinkRef.current = (event: RTMEvents.MessageEvent) => { + try { + const raw = + typeof event.message === "string" + ? event.message + : new TextDecoder().decode(event.message); + const parsed = raw ? JSON.parse(raw) : null; + if (event.channelType === "USER") { setRtmMessages((prevMessages: any[]) => { - // Check if the messages array exceeds the maximum limit - if (prevMessages.length >= 500) { - prevMessages.pop(); // Remove the oldest message - } + const next = [...prevMessages]; + if (next.length >= 500) next.shift(); return [ - ...prevMessages, - { - channelmessage: JSON.parse(message.text + ""), - from: memberId, - }, + ...next, + { peermessage: parsed, from: event.publisher }, ]; }); - - dispatch( - changeChildAction( - "messages", - getData(rtmMessages).data, - false - ) - ); - } - ); - } - }, [rtmChannelResponse]); - useEffect(() => { - if (client) { - //Enable Agora to send audio bytes - client.enableAudioVolumeIndicator(); - //user activity listeners - client.on("user-joined", (user: IAgoraRTCRemoteUser) => { - setUserJoined(user); - }); - client.on("user-left", (user: IAgoraRTCRemoteUser, reason: any) => { - setUserLeft(user); - }); - - //listen to user speaking, - client.on("volume-indicator", (volumeInfos: any) => { - if (volumeInfos.length === 0) return; - volumeInfos.map((volumeInfo: any) => { - //when the volume is above 30, user is probably speaking - const speaking = volumeInfo.level >= 30; + } else if ( + event.channelType === "MESSAGE" && + rtmSubscribedChannelName && + event.channelName === rtmSubscribedChannelName + ) { + if ( + parsed && + typeof parsed === "object" && + parsed.type === RTM_MEETING_REQUEST_PRESENCE + ) { + presenceRequestSinkRef.current?.(); + } if ( - volumeInfo.uid === userId && - props.localUser.value.speaking != speaking + parsed && + typeof parsed === "object" && + parsed.type === RTM_MEETING_USER_STATE && + parsed.user != null ) { - setLocalUserSpeaking(speaking); - } else { - setUpdateVolume({ update: speaking, userid: volumeInfo.uid }); + const skipParticipantsUpdate = + userId != null && + userId !== "" && + String(parsed.user) === String(userId); + if (!skipParticipantsUpdate) { + setLocalParticipants((prev) => { + const uid = parsed.user; + let matched = false; + const next = prev.map((u: any) => { + if (String(u.user) === String(uid)) { + matched = true; + return { + ...u, + audiostatus: !!parsed.audiostatus, + streamingVideo: + parsed.streamingVideo !== undefined + ? parsed.streamingVideo + : u.streamingVideo, + speaking: + parsed.speaking !== undefined + ? parsed.speaking + : u.speaking, + userName: + parsed.userName !== undefined + ? parsed.userName + : u.userName, + streamingSharing: + parsed.streamingSharing !== undefined + ? parsed.streamingSharing + : u.streamingSharing, + }; + } + return u; + }); + const merged = matched + ? next + : [ + ...next, + { + user: uid, + audiostatus: !!parsed.audiostatus, + streamingVideo: + parsed.streamingVideo !== undefined + ? parsed.streamingVideo + : true, + streamingSharing: + parsed.streamingSharing !== undefined + ? parsed.streamingSharing + : false, + speaking: + parsed.speaking !== undefined + ? parsed.speaking + : false, + userName: + parsed.userName !== undefined + ? parsed.userName + : "", + }, + ]; + return meetingParticipantsDedupe( + getData(merged).data, + "user" + ); + }); + } } + if ( + parsed == null || + typeof parsed !== "object" || + parsed.type !== RTM_MEETING_REQUEST_PRESENCE + ) { + setRtmMessages((prevMessages: any[]) => { + const next = [...prevMessages]; + if (next.length >= 500) next.shift(); + return [ + ...next, + { + channelmessage: parsed, + from: event.publisher, + }, + ]; + }); + } + } + } catch { + /* ignore malformed payloads */ + } + }; + return () => { + rtmMessageSinkRef.current = null; + }; + }, []); + useEffect(() => { + if (!client) { + return; + } + + client.enableAudioVolumeIndicator(); + + const onUserJoined = (user: IAgoraRTCRemoteUser) => { + if ( + userId != null && + userId !== "" && + String(user.uid) === String(userId) + ) { + return; + } + const userData = { + user: user.uid, + audiostatus: user.hasAudio, + streamingVideo: true, + streamingSharing: false, + userName: "", + }; + setLocalParticipants((prev) => + meetingParticipantsDedupe( + getData([...prev, userData]).data, + "user" + ) + ); + const snap = latestMeetingBroadcastRef.current; + if (snap.meetingActive && userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: snap.audiostatus, + streamingVideo: snap.streamingVideo, + streamingSharing: snap.streamingSharing, + speaking: snap.speaking, + userName: snap.userName, }); - }); + } + }; - client.on( - "user-published", - async ( - user: IAgoraRTCRemoteUser, - mediaType: "video" | "audio" - ) => { - setLocalUserVideo(user); + const onUserLeft = (user: IAgoraRTCRemoteUser) => { + setLocalParticipants((prev) => { + let newUsers = prev.filter( + (item: any) => String(item.user) !== String(user.uid) + ); + const hostExists = newUsers.some((f: any) => f.host === true); + if (!hostExists && newUsers.length > 0) { + newUsers = [ + { ...newUsers[0], host: true }, + ...newUsers.slice(1), + ]; } - ); - client.on( - "user-unpublished", - (user: IAgoraRTCRemoteUser, mediaType: "video" | "audio") => { - setLocalUserVideo(user); + return meetingParticipantsDedupe(getData(newUsers).data, "user"); + }); + }; + + const onVolumeIndicator = (volumeInfos: any) => { + if (volumeInfos.length === 0) return; + volumeInfos.map((volumeInfo: any) => { + const speaking = volumeInfo.level >= 30; + if ( + volumeInfo.uid === userId && + props.localUser.value.speaking != speaking + ) { + setLocalUserSpeaking(speaking); + } else { + setUpdateVolume({ update: speaking, userid: volumeInfo.uid }); } + }); + }; + + const onUserPublished = async ( + user: IAgoraRTCRemoteUser, + mediaType: "video" | "audio" + ) => { + setTimeout(() => { + setLocalUserVideo(user); + }, 1000); + if (mediaType !== "video") { + return; + } + const uid = String(user.uid); + if (!sharingUserIdsRef.current.includes(uid)) { + return; + } + const attachShare = () => { + void playScreenShareToElement( + uid, + userId != null && userId !== "" && String(userId) === uid + ); + }; + attachShare(); + [300, 1000, 2000].forEach((ms) => + window.setTimeout(attachShare, ms) ); - } + }; + + const onUserUnpublished = ( + user: IAgoraRTCRemoteUser, + mediaType: "video" | "audio" + ) => { + setLocalUserVideo(user); + if (mediaType === "video") { + clearScreenShareAttach(String(user.uid)); + } + }; + + client.on("user-joined", onUserJoined); + client.on("user-left", onUserLeft); + client.on("volume-indicator", onVolumeIndicator); + client.on("user-published", onUserPublished); + client.on("user-unpublished", onUserUnpublished); + + return () => { + client.off("user-joined", onUserJoined); + client.off("user-left", onUserLeft); + client.off("volume-indicator", onVolumeIndicator); + client.off("user-published", onUserPublished); + client.off("user-unpublished", onUserUnpublished); + }; }, [client]); return ( @@ -602,6 +1292,9 @@ if (typeof ContainerCompBuilder === "function") { {children.localUserID.propertyView({ label: trans("meeting.localUserID"), })} + {children.localUserName.propertyView({ + label: trans("meeting.localUserName"), + })} {children.rtmToken.propertyView({ label: trans("meeting.rtmToken"), })} @@ -682,6 +1375,16 @@ if (typeof ContainerCompBuilder === "function") { let sharing = !comp.children.sharing.getView().value; await shareScreen(sharing); comp.children.sharing.change(sharing); + if (userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: comp.children.audioControl.getView().value, + streamingVideo: comp.children.videoControl.getView().value, + streamingSharing: sharing, + speaking: comp.children.localUser.getView().value?.speaking ?? false, + userName: comp.children.localUserName.getView().value, + }); + } }, }, { @@ -697,10 +1400,22 @@ if (typeof ContainerCompBuilder === "function") { user: userId + "", audiostatus: value, streamingVideo: comp.children.videoControl.getView().value, + streamingSharing: comp.children.sharing.getView().value, speaking: false, + userName: comp.children.localUserName.getView().value, }); await turnOnMicrophone(value); comp.children.audioControl.change(value); + if (userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: value, + streamingVideo: comp.children.videoControl.getView().value, + streamingSharing: comp.children.sharing.getView().value, + speaking: false, + userName: comp.children.localUserName.getView().value, + }); + } }, }, { @@ -714,21 +1429,40 @@ if (typeof ContainerCompBuilder === "function") { if (!comp.children.meetingActive.getView().value) return; //toggle videoControl let value = !comp.children.videoControl.getView().value; - if (videoTrack) { + if (isCameraTrackUsable(videoTrack)) { videoTrack.setEnabled(value); - } else { - await turnOnCamera(value); + } else if (value) { + try { + await turnOnCamera(true); + if (isCameraTrackUsable(videoTrack)) { + await client.publish(videoTrack); + } + } catch { + value = false; + } } //change my local user data let localData = { user: userId + "", streamingVideo: value, audiostatus: comp.children.audioControl.getView().value, + streamingSharing: comp.children.sharing.getView().value, speaking: comp.children.localUser.getView().value.speaking, + userName: comp.children.localUserName.getView().value, }; comp.children.localUser.change(localData); comp.children.videoControl.change(value); + if (userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: comp.children.audioControl.getView().value, + streamingVideo: value, + streamingSharing: comp.children.sharing.getView().value, + speaking: comp.children.localUser.getView().value.speaking, + userName: comp.children.localUserName.getView().value, + }); + } }, }, { @@ -738,44 +1472,26 @@ if (typeof ContainerCompBuilder === "function") { params: [], }, execute: async (comp: any, values: any) => { + if (comp.children.meetingActive.getView().value) { + if (isClientInChannel()) { + return; + } + comp.children.meetingActive.change(false); + } + const resolvedUserName = String( + comp.children.localUserName.getView().value ?? "" + ).trim(); /* console.log("startMeeting ", { // user: userId + "", audiostatus: false, speaking: false, streamingVideo: true, }); */ - if (comp.children.meetingActive.getView().value) return; userId = comp.children.localUserID.getView().value === "" ? uuidv4() : comp.children.localUserID.getView().value; - comp.children.localUser.change({ - user: userId + "", - audiostatus: false, - speaking: false, - streamingVideo: true, - }); - /* console.log("startMeeting localUser ", { - user: userId + "", - audiostatus: false, - speaking: false, - streamingVideo: true, - }); */ - - comp.children.localUser.children.value.dispatch( - changeChildAction( - "localUser", - { - user: userId + "", - audiostatus: false, - speaking: false, - streamingVideo: true, - }, - false - ) - ); - comp.children.videoControl.change(true); - await publishVideo( + const videoPublished = await publishVideo( comp.children.appId.getView(), comp.children.meetingName.getView().value === "" ? uuidv4() @@ -783,7 +1499,35 @@ if (typeof ContainerCompBuilder === "function") { comp.children.rtmToken.getView().value, comp.children.rtcToken.getView().value ); + comp.children.videoControl.change(videoPublished); + comp.children.localUser.change({ + user: userId + "", + audiostatus: false, + speaking: false, + streamingVideo: videoPublished, + streamingSharing: false, + userName: resolvedUserName, + }); + console.log("publishVideo ", { + appId: comp.children.appId.getView(), + meetingName: comp.children.meetingName.getView().value === "" + ? uuidv4() + : comp.children.meetingName.getView().value, + rtmToken: comp.children.rtmToken.getView().value, + rtcToken: comp.children.rtcToken.getView().value, + videoPublished, + }); comp.children.meetingActive.change(true); + if (userId != null && userId !== "") { + broadcastLocalMeetingUserState({ + user: String(userId), + audiostatus: false, + streamingVideo: videoPublished, + streamingSharing: false, + speaking: false, + userName: resolvedUserName, + }); + } }, }, { @@ -833,8 +1577,13 @@ if (typeof ContainerCompBuilder === "function") { }, execute: async (comp: any, values: any) => { let userName: any = values[0]; + const nameStr = + userName != null && String(userName).trim() !== "" + ? String(userName).trim() + : ""; + comp.children.localUserName.change(nameStr); let userLocal = comp.children.localUser.getView().value; - comp.children.localUser.change({ ...userLocal, userName: userName }); + comp.children.localUser.change({ ...userLocal, userName: nameStr }); }, }, { @@ -871,12 +1620,14 @@ if (typeof ContainerCompBuilder === "function") { let value = !comp.children.endCall.getView().value; comp.children.endCall.change(value); comp.children.meetingActive.change(false); + comp.children.sharing.change(false); await leaveChannel(); comp.children.localUser.change({ user: userId + "", streamingVideo: false, + userName: comp.children.localUserName.getView().value, }); }, }, @@ -889,6 +1640,7 @@ if (typeof ContainerCompBuilder === "function") { new NameConfig("meetingActive", trans("meeting.meetingActive")), new NameConfig("meetingName", trans("meeting.meetingName")), new NameConfig("localUserID", trans("meeting.localUserID")), + new NameConfig("localUserName", trans("meeting.localUserName")), new NameConfig("messages", trans("meeting.messages")), new NameConfig("rtmToken", trans("meeting.rtmToken")), new NameConfig("rtcToken", trans("meeting.rtcToken")), diff --git a/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingStreamUtils.ts b/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingStreamUtils.ts new file mode 100644 index 0000000000..1cef753fbb --- /dev/null +++ b/client/packages/lowcoder-comps/src/comps/agoraMeetingComp/meetingStreamUtils.ts @@ -0,0 +1,62 @@ +/** Stable DOM id for a participant's screen-share