// ---------------------------------------------------------------------------- // // Photon Voice for Unity - Copyright (C) 2018 Exit Games GmbH // // // Component that represents a client voice connection to Photon Servers. // // developer@photonengine.com // ---------------------------------------------------------------------------- using System; using System.Collections.Generic; using ExitGames.Client.Photon; using Photon.Realtime; using UnityEngine; using UnityEngine.Serialization; namespace Photon.Voice.Unity { /// Component that represents a Voice client and manages a simple Unity integration: a single Recorder and multiple remote speakers. [AddComponentMenu("Photon Voice/Unity Voice Client")] [HelpURL("https://doc.photonengine.com/en-us/voice/v2/getting-started/voice-intro")] public class UnityVoiceClient : VoiceConnection { public override bool AlwaysUsePrimaryRecorder => true; /// /// Whether or not to use the Voice AppId and all the other AppSettings from Fusion's RealtimeAppSettings ScriptableObject singleton in the Voice client/app. /// [field: SerializeField] public bool UseVoiceAppSettings = false; protected virtual void Start() { if (this.PrimaryRecorder != null) { AddRecorder(this.PrimaryRecorder); } } public override bool ConnectUsingSettings(AppSettings overwriteSettings = null) { if (overwriteSettings != null) { return base.ConnectUsingSettings(overwriteSettings); } if (this.UseVoiceAppSettings) { return base.ConnectUsingSettings(PhotonAppSettings.Instance.AppSettings); } else { return base.ConnectUsingSettings(); } } protected override Speaker InstantiateSpeakerForRemoteVoice(int playerId, byte voiceId, object userData) { // Create a new Speaker on each OnRemoteVoiceInfo() call return this.InstantiateSpeakerPrefab(this.gameObject, true); } } [DisallowMultipleComponent] /// Component that represents a Voice client. public class VoiceConnection : ConnectionHandler { /// Recommended Photon Transport channel for audio. Chosen not to interfere with video and default channel. public const int ChannelAudio = 1; /// Recommended Photon Transport channel for video. Chosen not to interfere with audio and default channel. public const int ChannelVideo = 2; #region Private Fields // VoiceComponentImpl instance instead if VoiceComponent inheritance private VoiceComponentImpl voiceComponentImpl = new VoiceComponentImpl(); /// Key to save the "Best Region Summary" in the Player Preferences. private const string PlayerPrefsKey = "VoiceCloudBestRegion"; private LoadBalancingTransport client; private SupportLogger supportLoggerComponent; [SerializeField] private bool runInBackground = true; /// /// time [ms] between statistics calculations /// [SerializeField] private int statsResetInterval = 1000; private int nextStatsTickCount = Environment.TickCount; private float statsReferenceTime; private int referenceFramesLost; private int referenceFramesReceived; [SerializeField] private GameObject speakerPrefab; private List cachedRemoteVoices = new List(); [SerializeField] [FormerlySerializedAs("PrimaryRecorder")] private Recorder primaryRecorder; /// /// If true, will be used by this VoiceConnection instnance directly. /// [SerializeField] [Tooltip("Use primary recorder directly by Voice Client")] private bool usePrimaryRecorder; [SerializeField] [Tooltip("Use the protocol compatible with Photon Voice C++ API")] private bool cppCompatibilityMode; // to allow VoiceConnection ignore usePrimaryRecorder and do not show it in Editor public virtual bool AlwaysUsePrimaryRecorder => false; private List linkedSpeakers = new List(); private List recorders = new List(); #endregion private void Init() { this.client = new LoadBalancingTransport2(this.Logger, ConnectionProtocol.Udp, cppCompatibilityMode); this.client.VoiceClient.OnRemoteVoiceInfoAction += this.OnRemoteVoiceInfo; this.client.StateChanged += this.OnVoiceStateChanged; this.client.OpResponseReceived += this.OnOperationResponseReceived; base.Client = this.client; this.StartFallbackSendAckThread(); } #region Public Fields /// Settings to be used by this Voice Client public AppSettings Settings; #if UNITY_EDITOR [HideInInspector] public bool ShowSettings = true; #endif /// Fires when a speaker has been linked to a remote audio stream public event Action SpeakerLinked; /// Fires when a remote voice stream is added public event Action RemoteVoiceAdded; #if UNITY_PS4 || UNITY_PS5 /// PlayStation user ID of the local user /// Pass the userID of the local PlayStation user who should receive any incoming audio. This value is used by Photon Voice when sending output to the headphones on the PlayStation. /// If you don't provide a user ID, then Photon Voice uses the user ID of the user at index 0 in the list of local users /// and in case that there are multiple local users, the audio output might be sent to the headphones of a different user than intended. public int PlayStationUserID = 0; // set from your game's code #endif #endregion #region Properties public Voice.ILogger Logger => voiceComponentImpl.Logger; // to set logging level from code public VoiceLogger VoiceLogger => voiceComponentImpl.VoiceLogger; public new LoadBalancingTransport Client { get { return this.client; } } /// Returns underlying Photon Voice client. public VoiceClient VoiceClient { get { return this.Client.VoiceClient; } } /// Returns Photon Voice client state. public ClientState ClientState { get { return this.Client.State; } } /// Number of frames received per second. public float FramesReceivedPerSecond { get; private set; } /// Number of frames lost per second. public float FramesLostPerSecond { get; private set; } /// Percentage of lost frames. public float FramesLostPercent { get; private set; } /// Prefab that contains Speaker component to be instantiated when receiving a new remote audio source info public GameObject SpeakerPrefab { get => this.speakerPrefab; set => this.speakerPrefab = value; } #if UNITY_EDITOR public List CachedRemoteVoices { get { return this.cachedRemoteVoices; } } #endif /// /// Primary Recorder to be used by VoiceConnection implementations directly or via integration objects. /// public Recorder PrimaryRecorder { get => this.primaryRecorder; set => this.primaryRecorder = value; } /// /// Use directly. /// public bool UsePrimaryRecorder => this.usePrimaryRecorder; /// Used to store and access the "Best Region Summary" in the Player Preferences. public string BestRegionSummaryInPreferences { get { return PlayerPrefs.GetString(PlayerPrefsKey, null); } set { if (string.IsNullOrEmpty(value)) { PlayerPrefs.DeleteKey(PlayerPrefsKey); } else { PlayerPrefs.SetString(PlayerPrefsKey, value); } } } #endregion #region Public Methods /// /// Connect to Photon server using /// /// Overwrites before connecting /// If true voice connection command was sent from client public virtual bool ConnectUsingSettings(AppSettings overwriteSettings = null) { if (this.Client.LoadBalancingPeer.PeerState != PeerStateValue.Disconnected) { this.Logger.Log(LogLevel.Warning, "ConnectUsingSettings() failed. Can only connect while in state 'Disconnected'. Current state: {0}", this.Client.LoadBalancingPeer.PeerState); return false; } if (overwriteSettings != null) { this.Settings = overwriteSettings; } if (this.Settings == null) { this.Logger.Log(LogLevel.Error, "Settings are null"); return false; } if (string.IsNullOrEmpty(this.Settings.AppIdVoice) && string.IsNullOrEmpty(this.Settings.Server)) { this.Logger.Log(LogLevel.Error, "Provide an AppId or a Server address in Settings to be able to connect"); return false; } if (this.Settings.IsMasterServerAddress && string.IsNullOrEmpty(this.Client.UserId)) { this.Client.UserId = Guid.NewGuid().ToString(); // this is a workaround to use when connecting to self-hosted Photon Server v4, which does not return a UserId to the client if generated randomly server side } if (string.IsNullOrEmpty(this.Settings.BestRegionSummaryFromStorage)) { this.Settings.BestRegionSummaryFromStorage = this.BestRegionSummaryInPreferences; } return this.client.ConnectUsingSettings(this.Settings); } /// /// Tries to link local Speaker with remote voice stream using UserData. /// Useful if Speaker created after stream is started. /// /// Speaker ot try linking. /// UserData object used to bind local Speaker with remote voice stream. /// public bool AddSpeaker(Speaker speaker, object userData) { for (int i = 0; i < this.cachedRemoteVoices.Count; i++) { RemoteVoiceLink rvl = this.cachedRemoteVoices[i]; if (userData.Equals(rvl.VoiceInfo.UserData)) { this.Logger.Log(LogLevel.Debug, "Speaker linking for remoteVoice {0}.", rvl); this.LinkSpeaker(speaker, rvl); return speaker.IsLinked; } } return false; } #endregion #region Private Methods protected override void Awake() { base.Awake(); voiceComponentImpl.Awake(this); Init(); if (this.ApplyDontDestroyOnLoad) { // also apply to the relevant VoiceLogger DontDestroyOnLoad(voiceComponentImpl.VoiceLogger.gameObject); } this.supportLoggerComponent = this.GetComponent(); if (this.supportLoggerComponent != null) { this.supportLoggerComponent.Client = this.Client; this.supportLoggerComponent.LogTrafficStats = true; } if (this.runInBackground) { Application.runInBackground = this.runInBackground; } } protected virtual void Update() { this.VoiceClient.Service(); } protected virtual void FixedUpdate() { while (this.Client.LoadBalancingPeer.DispatchIncomingCommands()) ; } private void LateUpdate() { while (this.Client.LoadBalancingPeer.SendOutgoingCommands()) ; if (this.statsResetInterval > 0) { int currentMsSinceStart = Environment.TickCount; // avoiding Environment.TickCount, which could be negative on long-running platforms if (currentMsSinceStart - this.nextStatsTickCount > 0) { this.CalcStatistics(); this.nextStatsTickCount = currentMsSinceStart + this.statsResetInterval; } } } protected virtual void OnDestroy() { this.client.StateChanged -= this.OnVoiceStateChanged; this.client.OpResponseReceived -= this.OnOperationResponseReceived; this.client.Disconnect(); if (this.client.LoadBalancingPeer != null) { this.client.LoadBalancingPeer.Disconnect(); this.client.LoadBalancingPeer.StopThread(); } this.client.Dispose(); } protected virtual Speaker InstantiateSpeakerForRemoteVoice(int playerId, byte voiceId, object userData) { throw new Exception("FindSpeakerByUserData: VoiceConnection does not provide userData linkage"); } /// /// Instantiates , optionally attaches it to the provided parent. /// /// /// VoiceConnection manages the instantiated object (destroys on OnRemoteVoiceRemoveAction). /// /// The object to attach Steaker to. /// Automatically destroy instantiated prefab when remote voice is removed (the caller does not manages the instance). /// Instantiated Speaker or null. public Speaker InstantiateSpeakerPrefab(GameObject parent, bool destroyOnRemove) { if (this.SpeakerPrefab == null) { this.Logger.Log(LogLevel.Error, "SpeakerPrefab is not set."); return null; } var go = Instantiate(this.SpeakerPrefab); Speaker[] speakers = go.GetComponentsInChildren(true); if (speakers.Length > 0) { if (speakers.Length > 1) { this.Logger.Log(LogLevel.Warning, "Multiple Speaker components found attached to the GameObject (VoiceConnection.SpeakerPrefab) or its children. Using the first one we found."); } if (destroyOnRemove) { speakers[0].OnRemoteVoiceRemoveAction += (s) => { this.Logger.Log(LogLevel.Info, "OnRemoteVoiceRemoveAction: destroying VoiceConnection.SpeakerPrefab instance [{0}]", go.name); Destroy(go); }; } if (parent != null) { if(parent.transform.Find("Headset")) go.transform.SetParent(parent.transform.Find("Headset"), false); else go.transform.SetParent(parent.transform, false); } this.Logger.Log(LogLevel.Info, "Instance of VoiceConnection.SpeakerPrefab instantiated."); return speakers[0]; } else { this.Logger.Log(LogLevel.Error, "SpeakerPrefab does not have a component of type Speaker in its hierarchy."); Destroy(go); return null; } } private void OnRemoteVoiceInfo(int channelId, int playerId, byte voiceId, VoiceInfo voiceInfo, ref RemoteVoiceOptions options) { if (voiceInfo.Codec != Codec.AudioOpus) { this.Logger.Log(LogLevel.Info, "OnRemoteVoiceInfo: Skipped as codec is not Opus, [p#{0} v#{1} c#{2} i:{{{3}}}]", playerId, voiceId, channelId, voiceInfo); return; } RemoteVoiceLink remoteVoice = new RemoteVoiceLink(voiceInfo, playerId, voiceId, channelId, ref options); if (Application.platform == RuntimePlatform.WebGLPlayer) { #if !UNITY_2021_2_OR_NEWER // opus lib requires Emscripten 2.0.19 this.Logger.Log(LogLevel.Error, "Remote voice Opus decoder requies Unity 2021.2 or newer for WebGL"); options.Decoder = null; // null Opus decoder set by RemoteVoiceLink #endif } this.Logger.Log(LogLevel.Info, "OnRemoteVoiceInfo: {0}", remoteVoice); this.cachedRemoteVoices.Add(remoteVoice); if (RemoteVoiceAdded != null) { RemoteVoiceAdded(remoteVoice); } remoteVoice.RemoteVoiceRemoved += () => { this.Logger.Log(LogLevel.Info, "OnRemoteVoiceInfo: RemoteVoiceRemoved {0}", remoteVoice); this.cachedRemoteVoices.Remove(remoteVoice); }; var speaker = this.InstantiateSpeakerForRemoteVoice(playerId, voiceId, voiceInfo.UserData); if (speaker == null) { this.Logger.Log(LogLevel.Debug, "OnRemoteVoiceInfo: Remote GameObject not found or does not have a Speaker {0}", remoteVoice); } else { speaker.Name = string.Format("Remote p#{0} v#{1}", playerId, voiceId); this.LinkSpeaker(speaker, remoteVoice); } } protected virtual void OnVoiceStateChanged(ClientState fromState, ClientState toState) { this.Logger.Log(LogLevel.Info, "OnVoiceStateChanged from {0} to {1}", fromState, toState); if (fromState == ClientState.Joined) { for (int i = 0; i < this.recorders.Count; i++) { Recorder rec = this.recorders[i]; if (rec.RecordWhenJoined) { rec.RecordingEnabled = false; } } this.cachedRemoteVoices.Clear(); } switch (toState) { case ClientState.ConnectedToMasterServer: { if (this.Client.RegionHandler != null) { if (this.Settings != null) { this.Settings.BestRegionSummaryFromStorage = this.Client.RegionHandler.SummaryToCache; } this.BestRegionSummaryInPreferences = this.Client.RegionHandler.SummaryToCache; } break; } case ClientState.Joined: { for (int i = 0; i < this.recorders.Count; i++) { Recorder rec = this.recorders[i]; if (rec.RecordWhenJoined) { rec.RecordingEnabled = true; } } break; } } } protected void CalcStatistics() { float now = Time.time; int recv = this.VoiceClient.FramesReceived - this.referenceFramesReceived; int lost = this.VoiceClient.FramesLost - this.referenceFramesLost; float t = now - this.statsReferenceTime; if (t > 0f) { if (recv + lost > 0) { this.FramesReceivedPerSecond = recv / t; this.FramesLostPerSecond = lost / t; this.FramesLostPercent = 100f * lost / (recv + lost); } else { this.FramesReceivedPerSecond = 0f; this.FramesLostPerSecond = 0f; this.FramesLostPercent = 0f; } } this.referenceFramesReceived = this.VoiceClient.FramesReceived; this.referenceFramesLost = this.VoiceClient.FramesLost; this.statsReferenceTime = now; } private void LinkSpeaker(Speaker speaker, RemoteVoiceLink remoteVoice) { #if UNITY_PS4 || UNITY_PS5 speaker.PlayStationUserID = this.PlayStationUserID; #endif if (speaker.Link(remoteVoice)) { this.Logger.Log(LogLevel.Info, "Speaker linked with remote voice {0}", remoteVoice); this.linkedSpeakers.Add(speaker); remoteVoice.RemoteVoiceRemoved += () => { this.linkedSpeakers.Remove(speaker); }; if (SpeakerLinked != null) { SpeakerLinked(speaker); } } } public bool AddRecorder(Recorder rec) { if (!this.recorders.Contains(rec)) { if (rec.Init(this)) { this.recorders.Add(rec); return true; } else { this.Logger.Log(LogLevel.Warning, "AddRecorder: failed to init recorder {0}.", rec); } } else { this.Logger.Log(LogLevel.Error, "AddRecorder: recorder {0} already added.", rec); } return false; } public void RemoveRecorder(Recorder rec) { if (rec != null) { rec.Deinit(this); this.recorders.Remove(rec); } } protected virtual void OnOperationResponseReceived(OperationResponse operationResponse) { if (operationResponse.ReturnCode != ErrorCode.Ok && (operationResponse.OperationCode != OperationCode.JoinRandomGame || operationResponse.ReturnCode == ErrorCode.NoRandomMatchFound)) { this.Logger.Log(LogLevel.Error, "Operation {0} response error code {1} message {2}", operationResponse.OperationCode, operationResponse.ReturnCode, operationResponse.DebugMessage); } } #endregion } }