Merge pull request #109 from musehq/dev

v2.3.0

Author: alex-shortt

README.md

@@ -190,7 +190,7 @@ type PlayerState = {
 
 #### Network Layer
 
-_Provides multiplayer out-of-the-box. Muse provides signalling servers for everyone, but not STUN/TURN servers, which [you need for best performance](https://www.twilio.com/docs/stun-turn/faq#faq-what-is-nat)._
+_Provides multiplayer out-of-the-box. Muse provides signalling servers and [STUN/TURN](https://www.twilio.com/docs/stun-turn/faq#faq-what-is-nat) servers for everyone :)._
 
 ```tsx
 type NetworkProps = {
@@ -200,6 +200,7 @@ type NetworkProps = {
   host?: string; // signalling host url, uses Muse's servers by default
   sessionId?: string; // if you know the session id you want to use, enter it here
   worldName?: string; // the worldname to hash your signal peers by, by default set to the path name
+  voice?: boolean; // whether to enable spatial voice chat, false by default
 };
 ```
 

examples/worlds/Multiplayer/index.tsx

@@ -5,7 +5,7 @@ export default function Multiplayer() {
   return (
     <StandardReality
       playerProps={{ pos: [5, 1, 0], rot: Math.PI }}
-      networkProps={{ autoconnect: true }}
+      networkProps={{ autoconnect: true, voice: true }}
     >
       <Background color={0xffffff} />
       <fog attach="fog" args={[0xffffff, 10, 90]} />

package.json

@@ -1,6 +1,6 @@
 {
   "name": "spacesvr",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "private": true,
   "description": "A standardized reality for future of the 3D Web",
   "keywords": [

src/layers/Environment/logic/environment.ts

@@ -7,6 +7,7 @@ import {
   useState,
 } from "react";
 import { Device, DeviceState, useDevice } from "./device";
+import { AudioContext } from "three";
 
 export type MenuItem = { text: string; action: () => void };
 
@@ -33,6 +34,12 @@ export const useEnvironmentState = (name: string): EnvironmentState => {
 
   const setPaused = (p: boolean) => {
     setPausedValue(p);
+
+    // hook into paused click event to make sure global context is running
+    // https://github.com/mrdoob/three.js/blob/342946c8392639028da439b6dc0597e58209c696/src/audio/AudioContext.js#L9
+    const context = AudioContext.getContext();
+    if (context.state !== "running") context.resume();
+
     // call all pause events
     events.map((ev: PauseEvent) => ev.apply(null, [p]));
   };

src/layers/Environment/ui/PauseMenu/index.tsx

@@ -48,7 +48,7 @@ export default function PauseMenu(props: PauseMenuProps) {
   const PAUSE_ITEMS: PauseItem[] = [
     ...pauseMenuItems,
     {
-      text: "v2.2.1",
+      text: "v2.3.0",
       link: "https://www.npmjs.com/package/spacesvr",
     },
     ...menuItems,

src/layers/Network/ideas/NetworkedEntities.tsx renamed to src/layers/Network/ideas/NetworkedEntities/index.tsx

@@ -1,72 +1,62 @@
-import { useMemo, useRef, useState } from "react";
+import { useMemo, useRef } from "react";
 import {
   CylinderBufferGeometry,
   InstancedMesh,
   MeshNormalMaterial,
-  Object3D,
 } from "three";
-import { useNetwork } from "../logic/network";
-import { useLimitedFrame } from "../../../logic/limiter";
+import { useNetwork } from "../../logic/network";
+import { useLimitedFrame } from "../../../../logic/limiter";
 import { SnapshotInterpolation } from "@geckos.io/snapshot-interpolation";
 import {
   Snapshot,
   Entity as EntityState,
   Quat,
 } from "@geckos.io/snapshot-interpolation/lib/types";
+import { useObj } from "./logic/resources";
+import { useEntities } from "./logic/entity";
 
 export default function NetworkedEntities() {
-  const { connections, connected, useChannel } = useNetwork();
+  const { connected, useChannel } = useNetwork();
 
   const mesh = useRef<InstancedMesh>(null);
   const geo = useMemo(() => new CylinderBufferGeometry(0.3, 0.3, 1, 32), []);
   const mat = useMemo(() => new MeshNormalMaterial(), []);
-  const obj = useMemo(() => {
-    const o = new Object3D();
-    o.matrixAutoUpdate = false;
-    return o;
-  }, []);
-
-  // check for a change in player list, re-render if there is a change
-  const [entityIds, setEntityIds] = useState<string[]>([]);
-  useLimitedFrame(6, () => {
-    if (!connected) return;
-    const ids = Array.from(connections.keys());
-    const sameIds = ids.sort().join(",") === entityIds.sort().join(",");
-    if (!sameIds) setEntityIds(ids);
-  });
+  const obj = useObj();
+
+  const entities = useEntities();
 
+  // set up channel to send/receive data
   const NETWORK_FPS = 12;
-  type Entity = { pos: number[]; rot: number[] };
+  type EntityTransform = { pos: number[]; rot: number[] };
   const SI = useMemo(() => new SnapshotInterpolation(NETWORK_FPS), []);
-  const entityChannel = useChannel<Entity, { [key in string]: Entity }>(
-    "player",
-    "stream",
-    (m, s) => {
-      if (!m.conn) return;
-      s[m.conn.peer] = m.data;
-
-      const state: EntityState[] = Object.keys(s).map((key) => ({
-        id: key,
-        x: s[key].pos[0],
-        y: s[key].pos[1],
-        z: s[key].pos[2],
-        q: {
-          x: s[key].rot[0],
-          y: s[key].rot[1],
-          z: s[key].rot[2],
-          w: s[key].rot[3],
-        },
-      }));
-
-      const snapshot: Snapshot = {
-        id: Math.random().toString(),
-        time: new Date().getTime(),
-        state,
-      };
-
-      SI.vault.add(snapshot);
-    }
-  );
+  const entityChannel = useChannel<
+    EntityTransform,
+    { [id: string]: EntityTransform }
+  >("player", "stream", (m, s) => {
+    if (!m.conn) return;
+    s[m.conn.peer] = m.data;
+
+    const state: EntityState[] = Object.keys(s).map((key) => ({
+      id: key,
+      x: s[key].pos[0],
+      y: s[key].pos[1],
+      z: s[key].pos[2],
+      q: {
+        x: s[key].rot[0],
+        y: s[key].rot[1],
+        z: s[key].rot[2],
+        w: s[key].rot[3],
+      },
+    }));
+
+    const snapshot: Snapshot = {
+      id: Math.random().toString(),
+      time: new Date().getTime(),
+      state,
+    };
+
+    SI.vault.add(snapshot);
+  });
 
   // send own player data
   useLimitedFrame(NETWORK_FPS, ({ camera }) => {
@@ -98,6 +88,14 @@ export default function NetworkedEntities() {
       obj.quaternion.w = quat.w;
       obj.updateMatrix();
       mesh.current.setMatrixAt(i, obj.matrix);
+
+      const audio = entities[i]?.posAudio;
+      if (audio) {
+        obj.matrix.decompose(audio.position, audio.quaternion, audio.scale);
+        audio.updateMatrix();
+        audio.rotateY(Math.PI); // for some reason it's flipped
+      }
+
       i++;
     }
 
@@ -110,9 +108,15 @@ export default function NetworkedEntities() {
 
   return (
     <group>
+      {entities.map(
+        (entity) =>
+          entity.posAudio && (
+            <primitive key={entity.posAudio.uuid} object={entity.posAudio} />
+          )
+      )}
       <instancedMesh
         ref={mesh}
-        args={[geo, mat, entityIds.length]}
+        args={[geo, mat, entities.length]}
         matrixAutoUpdate={false}
       />
     </group>

src/layers/Network/ideas/NetworkedEntities/logic/entity.ts

@@ -0,0 +1,103 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import { useLimitedFrame } from "../../../../../logic/limiter";
+import { useNetwork } from "../../../logic/network";
+import { PositionalAudio } from "three";
+import { useListener } from "./resources";
+import { useEnvironment } from "../../../../Environment";
+import { PositionalAudioHelper } from "three/examples/jsm/helpers/PositionalAudioHelper";
+
+type Entity = {
+  id: string;
+  posAudio: PositionalAudio | undefined;
+};
+
+export const useEntities = (): Entity[] => {
+  const { connections, connected, voiceStreams } = useNetwork();
+  const { paused } = useEnvironment();
+
+  const listener = useListener();
+  const [ct, setCt] = useState(0);
+  const rerender = () => setCt(Math.random());
+  const [firstPaused, setFirstPaused] = useState(true);
+  useEffect(() => setFirstPaused(paused && firstPaused), [paused, firstPaused]);
+
+  const entities = useMemo<Entity[]>(() => [], []);
+
+  const sameIds = (ids1: string[], ids2: string[]) =>
+    ids1.sort().join(",") === ids2.sort().join(",");
+
+  // check for a change in player list, re-render if there is a change
+  const connectionIds = useRef<string[]>([]);
+  const voiceIds = useRef<string[]>([]);
+  useLimitedFrame(6, () => {
+    if (!connected) return;
+
+    // check for changes in connections
+    if (!sameIds(connectionIds.current, Array.from(connections.keys()))) {
+      connectionIds.current = Array.from(connections.keys());
+
+      // remove entities that are no longer connected
+      entities.map((e) => {
+        if (!connectionIds.current.includes(e.id)) {
+          entities.splice(entities.indexOf(e), 1);
+        }
+      });
+
+      // add in new entities
+      for (const id of connectionIds.current) {
+        if (!entities.some((e) => e.id === id)) {
+          entities.push({ id, posAudio: undefined });
+        }
+      }
+
+      rerender();
+    }
+
+    // dont run until first time unpaused to make sure audio context is running from first press
+    if (
+      !firstPaused &&
+      !sameIds(voiceIds.current, Array.from(voiceStreams.keys()))
+    ) {
+      voiceIds.current = Array.from(voiceStreams.keys());
+
+      // remove voice streams that are no longer connected
+      entities.map((e) => {
+        if (!voiceIds.current.includes(e.id)) {
+          e.posAudio?.remove();
+          e.posAudio = undefined;
+        }
+      });
+
+      // add in new voice streams
+      for (const id of voiceIds.current) {
+        const entity = entities.find((e) => e.id === id);
+        if (!entity) continue;
+
+        const stream = voiceStreams.get(id)!;
+        if (!stream) continue;
+
+        const audioElem = document.createElement("audio");
+        audioElem.srcObject = stream;
+        audioElem.muted = true;
+        audioElem.autoplay = true;
+        audioElem.loop = true;
+        //@ts-ignore
+        audioElem.playsInline = true;
+
+        const posAudio = new PositionalAudio(listener);
+        posAudio.userData.peerId = id;
+        posAudio.setMediaStreamSource(stream);
+        posAudio.setRefDistance(2);
+        posAudio.setDirectionalCone(200, 290, 0.2);
+        posAudio.setVolume(0.6);
+
+        // posAudio.add(new PositionalAudioHelper(posAudio, 1));
+        entity.posAudio = posAudio;
+      }
+
+      rerender();
+    }
+  });
+
+  return entities;
+};

src/layers/Network/ideas/NetworkedEntities/logic/resources.ts

@@ -0,0 +1,20 @@
+import { useThree } from "@react-three/fiber";
+import { useMemo } from "react";
+import { AudioListener, Object3D } from "three";
+
+export const useListener = (): AudioListener => {
+  const cam = useThree((st) => st.camera);
+  return useMemo(() => {
+    const listen = new AudioListener();
+    cam.add(listen);
+    return listen;
+  }, [cam]);
+};
+
+export const useObj = (): Object3D => {
+  return useMemo(() => {
+    const o = new Object3D();
+    o.matrixAutoUpdate = false;
+    return o;
+  }, []);
+};

src/layers/Network/logic/connection.ts

@@ -1,21 +1,26 @@
-import { useMemo, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { DataConnection, Peer } from "peerjs";
 import { isLocalNetwork } from "./local";
 import { LocalSignaller } from "./signallers/LocalSignaller";
 import { MuseSignaller } from "./signallers/MuseSignaller";
 import { useWaving } from "./wave";
 import { Signaller, SignallerConfig } from "./signallers";
 import { Channels, useChannels } from "./channels";
+import { useVoice } from "./voice";
+import { getMuseIceServers } from "./ice";
 
 export type ConnectionState = {
   connected: boolean;
   connect: (config?: ConnectionConfig) => Promise<void>;
   connections: Map<string, DataConnection>;
+  voiceStreams: Map<string, MediaStream>;
   disconnect: () => void;
+  setVoice: (v: boolean) => void;
 } & Pick<Channels, "useChannel">;
 
 export type ConnectionConfig = {
   iceServers?: RTCIceServer[];
+  voice?: boolean;
 } & SignallerConfig;
 
 export const useConnection = (
@@ -56,8 +61,14 @@ export const useConnection = (
 
     const finalConfig = { ...externalConfig, ...config };
 
+    if (!finalConfig.iceServers) {
+      const servers = await getMuseIceServers(finalConfig.host);
+      if (servers) finalConfig.iceServers = servers;
+    }
+
     const peerConfig: any = {};
     if (finalConfig.iceServers) peerConfig.iceServers = finalConfig.iceServers;
+
     const p = new Peer({ config: peerConfig });
 
     p.on("connection", registerConnection); // incoming
@@ -111,11 +122,17 @@ export const useConnection = (
 
   useWaving(1, signaller, disconnect);
 
+  const [voice, setVoice] = useState(!!externalConfig.voice);
+  useEffect(() => setVoice(!!externalConfig.voice), [externalConfig.voice]);
+  const voiceStreams = useVoice(voice, peer, connections);
+
   return {
     connected,
     connect,
     disconnect,
     connections,
+    voiceStreams,
     useChannel: channels.useChannel,
+    setVoice,
   };
 };