001/**
002 * Copyright 2015, Digium, Inc.
003 * All rights reserved.
004 *
005 * This source code is licensed under The MIT License found in the
006 * LICENSE file in the root directory of this source tree.
007 *
008 * For all details and documentation:  https://www.respoke.io
009 */
010
011package com.digium.respokesdk;
012
013import android.content.Context;
014import android.media.AudioManager;
015import android.opengl.GLSurfaceView;
016import android.os.Handler;
017import android.os.Looper;
018import android.util.Log;
019
020import org.json.JSONArray;
021import org.json.JSONException;
022import org.json.JSONObject;
023import org.webrtc.DataChannel;
024import org.webrtc.IceCandidate;
025import org.webrtc.MediaConstraints;
026import org.webrtc.MediaStream;
027import org.webrtc.MediaStreamTrack;
028import org.webrtc.PeerConnection;
029import org.webrtc.PeerConnectionFactory;
030import org.webrtc.SdpObserver;
031import org.webrtc.SessionDescription;
032import org.webrtc.VideoCapturer;
033import org.webrtc.VideoRenderer;
034import org.webrtc.VideoRendererGui;
035import org.webrtc.VideoSource;
036import org.webrtc.VideoTrack;
037
038import java.lang.ref.WeakReference;
039import java.util.ArrayList;
040import java.util.Date;
041import java.util.concurrent.Semaphore;
042import java.util.regex.Matcher;
043import java.util.regex.Pattern;
044
045
046/**
047 *  WebRTC Call including getUserMedia, path and codec negotation, and call state.
048 */
049public class RespokeCall {
050
051    private final static String TAG = "RespokeCall";
052    private WeakReference<Listener> listenerReference;
053    private RespokeSignalingChannel signalingChannel;
054    private ArrayList<PeerConnection.IceServer> iceServers;
055    private static PeerConnectionFactory peerConnectionFactory;
056    private PeerConnection peerConnection;
057    private VideoSource videoSource;
058    private ArrayList<IceCandidate> queuedRemoteCandidates;
059    private ArrayList<IceCandidate> queuedLocalCandidates;
060    private ArrayList<IceCandidate> collectedLocalCandidates;
061    private Semaphore queuedRemoteCandidatesSemaphore;
062    private Semaphore localCandidatesSemaphore;
063    private org.webrtc.VideoRenderer.Callbacks localRender;
064    private org.webrtc.VideoRenderer.Callbacks remoteRender;
065    private boolean caller;
066    private JSONObject incomingSDP;
067
068    // Addressing and call identification fields
069    private String sessionID;
070    private String toConnection;
071    private String toEndpointId;
072    // Options are web, conference, did, or sip
073    private String toType;
074
075    // Used for direct connections
076    public RespokeEndpoint endpoint;
077    public boolean audioOnly;
078    public Date timestamp;
079    private final PCObserver pcObserver = new PCObserver();
080    private final SDPObserver sdpObserver = new SDPObserver();
081    private boolean videoSourceStopped;
082    private MediaStream localStream;
083    private boolean directConnectionOnly;
084    private RespokeDirectConnection directConnection;
085    private boolean isHangingUp;
086
087
088    /**
089     *  An interface to notify the receiver of events occurring with the call
090     */
091    public interface Listener {
092
093
094        /**
095         *  Receive a notification that an error has occurred while on a call
096         *
097         *  @param errorMessage A human-readable description of the error.
098         *  @param sender       The RespokeCall that experienced the error
099         */
100        public void onError(String errorMessage, RespokeCall sender);
101
102
103        /**
104         *  When on a call, receive notification the call has been hung up
105         *
106         *  @param sender The RespokeCall that has hung up
107         */
108        public void onHangup(RespokeCall sender);
109
110
111        /**
112         *  When on a call, receive remote media when it becomes available. This is what you will need to provide if you want
113         *  to show the user the other party's video during a call.
114         *
115         *  @param sender The RespokeCall that has connected
116         */
117        public void onConnected(RespokeCall sender);
118
119
120        /**
121         *  This event is fired when the local end of the directConnection is available. It still will not be
122         *  ready to send and receive messages until the 'open' event fires.
123         *
124         *  @param directConnection The direct connection object
125         *  @param endpoint         The remote endpoint
126         */
127        public void directConnectionAvailable(RespokeDirectConnection directConnection, RespokeEndpoint endpoint);
128    }
129
130
131    /**
132     *  Determines if the specified SDP data contains definitions for a video stream
133     *
134     *  @param sdp  The SDP data to examine
135     *
136     *  @return True if a video stream definition is present, false otherwise
137     */
138    public static boolean sdpHasVideo(JSONObject sdp) {
139        boolean hasVideo = false;
140
141        if (null != sdp) {
142            try {
143                String sdpString = sdp.getString("sdp");
144                hasVideo = sdpString.contains("m=video");
145            } catch (JSONException e) {
146                // Bad SDP?
147                Log.d(TAG, "ERROR: Incoming call appears to have an invalid SDP");
148            }
149        }
150
151        return hasVideo;
152    }
153    
154
155    /**
156     *  Constructor primarily used for starting conference calls
157     *
158     *  @param channel         The signaling channel to use for the call
159     *  @param remoteEndpoint  The remote recipient of the call
160     *  @param remoteType      The type of remote recipient (i.e. "conference", "web", etc)
161     */
162    public RespokeCall(RespokeSignalingChannel channel, String remoteEndpoint, String remoteType) {
163        commonConstructor(channel);
164        toEndpointId = remoteEndpoint;
165        toType = remoteType;
166    }
167
168
169    /**
170     *  Constructor used for outbound calls 
171     *
172     *  @param channel               The signaling channel to use for the call
173     *  @param newEndpoint           The remote recipient of the call
174     *  @param directConnectionOnly  Specify true if this call is only for establishing a direct data connection (i.e. no audio/video)
175     */
176    public RespokeCall(RespokeSignalingChannel channel, RespokeEndpoint newEndpoint, boolean directConnectionOnly) {
177        commonConstructor(channel);
178
179        endpoint = newEndpoint;
180        toEndpointId = newEndpoint.getEndpointID();
181        toType = "web";
182
183        this.directConnectionOnly = directConnectionOnly;
184    }
185
186
187    /**
188     *  Constructor used for inbound calls 
189     *
190     *  @param channel               The signaling channel to use for the call
191     *  @param sdp                   The SDP data from the call offer
192     *  @param newSessionID          The session ID to use for the call signaling
193     *  @param newConnectionID       The ID of the remote connection initiating the call
194     *  @param endpointID            The ID of the remote endpoint
195     *  @param fromType              The type of remote recipient (i.e. "conference", "web", etc)
196     *  @param newEndpoint           The remote recipient of the call
197     *  @param directConnectionOnly  Specify true if this call is only for establishing a direct data connection (i.e. no audio/video)
198     *  @param newTimestamp          The timestamp when the call was initiated remotely
199     */
200    public RespokeCall(RespokeSignalingChannel channel, JSONObject sdp, String newSessionID, String newConnectionID, String endpointID, String fromType, RespokeEndpoint newEndpoint, boolean directConnectionOnly, Date newTimestamp) {
201        commonConstructor(channel);
202
203        incomingSDP = sdp;
204        sessionID = newSessionID;
205        endpoint = newEndpoint;
206        toEndpointId = endpointID;
207        toType = fromType;
208
209        if (fromType == null) {
210            toType = "web";
211        }
212
213        toConnection = newConnectionID;
214        this.directConnectionOnly = directConnectionOnly;
215        timestamp = newTimestamp;
216        audioOnly = !RespokeCall.sdpHasVideo(sdp);
217
218        if ((directConnectionOnly) && (endpoint != null)) {
219            actuallyAddDirectConnection();
220        }
221    }
222
223
224    /**
225     *  Common constructor logic
226     *
227     *  @param channel  The signaling channel to use for the call
228     */
229    private void commonConstructor(RespokeSignalingChannel channel) {
230        signalingChannel = channel;
231        iceServers = new ArrayList<PeerConnection.IceServer>();
232        queuedLocalCandidates = new ArrayList<IceCandidate>();
233        queuedRemoteCandidates = new ArrayList<IceCandidate>();
234        collectedLocalCandidates = new ArrayList<IceCandidate>();
235        sessionID = Respoke.makeGUID();
236        timestamp = new Date();
237        queuedRemoteCandidatesSemaphore = new Semaphore(1); // remote candidates queue mutex
238        localCandidatesSemaphore = new Semaphore(1); // local candidates queue mutex
239
240        if (null != signalingChannel) {
241            RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener();
242            if (null != signalingChannelListener) {
243                signalingChannelListener.callCreated(this);
244            }
245        }
246
247        //TODO resign active handler?
248    }
249
250
251    /**
252     *  Set a receiver for the Listener interface
253     *
254     *  @param listener  The new receiver for events from the Listener interface for this call instance
255     */
256    public void setListener(Listener listener) {
257        listenerReference = new WeakReference<Listener>(listener);
258    }
259
260
261    /**
262     *  Get the session ID of this call
263     *
264     *  @return The session ID
265     */
266    public String getSessionID() {
267        return sessionID;
268    }
269
270
271    /**
272     *  Start the outgoing call process. This method is used internally by the SDK and should never be called directly from your client application
273     *
274     *  @param context      An application context with which to access shared resources
275     *  @param glView       The GLSurfaceView on which to render video if applicable
276     *  @param isAudioOnly  Specify true if this call should be audio only
277     */
278    public void startCall(final Context context, GLSurfaceView glView, boolean isAudioOnly) {
279        caller = true;
280        audioOnly = isAudioOnly;
281
282        if (directConnectionOnly) {
283            if (null == directConnection) {
284                actuallyAddDirectConnection();
285            }
286
287            directConnectionDidAccept(context);
288        } else {
289            attachVideoRenderer(glView);
290
291            getTurnServerCredentials(new Respoke.TaskCompletionListener() {
292                @Override
293                public void onSuccess() {
294                    Log.d(TAG, "Got TURN credentials");
295                    initializePeerConnection(context);
296                    addLocalStreams(context);
297                    createOffer();
298                }
299
300                @Override
301                public void onError(String errorMessage) {
302                    postErrorToListener(errorMessage);
303                }
304            });
305        }
306    }
307
308
309    /**
310     *  Attach the call's video renderers to the specified GLSurfaceView
311     *
312     *  @param glView  The GLSurfaceView on which to render video
313     */
314    public void attachVideoRenderer(GLSurfaceView glView) {
315        if (null != glView) {
316            VideoRendererGui.setView(glView, new Runnable() {
317                @Override
318                public void run() {
319                    Log.d(TAG, "VideoRendererGui GL Context ready");
320                }
321            });
322
323            remoteRender = VideoRendererGui.create(0, 0, 100, 100,
324                    VideoRendererGui.ScalingType.SCALE_ASPECT_FILL, false);
325            localRender = VideoRendererGui.create(70, 5, 25, 25,
326                    VideoRendererGui.ScalingType.SCALE_ASPECT_FILL, false);
327        }
328    }
329
330
331    /**
332     *  Answer the call and start the process of obtaining media. This method is called automatically on the caller's
333     *  side. This method must be called on the callee's side to indicate that the endpoint does wish to accept the
334     *  call. 
335     *
336     *  @param context      An application context with which to access shared resources
337     *  @param newListener  A listener to receive notifications of call-related events
338     */
339    public void answer(final Context context, Listener newListener) {
340        if (!caller) {
341            listenerReference = new WeakReference<Listener>(newListener);
342
343            getTurnServerCredentials(new Respoke.TaskCompletionListener() {
344                @Override
345                public void onSuccess() {
346                    initializePeerConnection(context);
347                    addLocalStreams(context);
348                    processRemoteSDP();
349                }
350
351                @Override
352                public void onError(String errorMessage) {
353                    postErrorToListener(errorMessage);
354                }
355            });
356        }
357    }
358
359
360    /**
361     *  Tear down the call and release resources
362     *
363     *  @param shouldSendHangupSignal Send a hangup signal to the remote party if signal is not false and we have not received a hangup signal from the remote party.
364     */
365    public void hangup(boolean shouldSendHangupSignal) {
366        if (!isHangingUp) {
367            isHangingUp = true;
368
369            if (shouldSendHangupSignal) {
370
371                try {
372                    JSONObject data = new JSONObject("{'signalType':'bye','version':'1.0'}");
373                    data.put("target", directConnectionOnly ? "directConnection" : "call");
374                    data.put("sessionId", sessionID);
375                    data.put("signalId", Respoke.makeGUID());
376
377                    // Keep a second reference to the listener since the disconnect method will clear it before the success handler is fired
378                    final WeakReference<Listener> hangupListener = listenerReference;
379
380                    if (null != signalingChannel) {
381                        signalingChannel.sendSignal(data, toEndpointId, toConnection, toType, true, new Respoke.TaskCompletionListener() {
382                            @Override
383                            public void onSuccess() {
384                                if (null != hangupListener) {
385                                    new Handler(Looper.getMainLooper()).post(new Runnable() {
386                                        public void run() {
387                                            Listener listener = hangupListener.get();
388                                            if (null != listener) {
389                                                listener.onHangup(RespokeCall.this);
390                                            }
391                                        }
392                                    });
393                                }
394                            }
395
396                            @Override
397                            public void onError(String errorMessage) {
398                                postErrorToListener(errorMessage);
399                            }
400                        });
401                    }
402                } catch (JSONException e) {
403                    postErrorToListener("Error encoding signal to json");
404                }
405            }
406
407            disconnect();
408        }
409    }
410
411
412    /**
413     *  Mute or unmute the local video
414     *
415     *  @param mute If true, mute the video. If false, unmute the video
416     */
417    public void muteVideo(boolean mute) {
418        if (!audioOnly && (null != localStream) && (isActive())) {
419            for (MediaStreamTrack eachTrack : localStream.videoTracks) {
420                eachTrack.setEnabled(!mute);
421            }
422        }
423    }
424
425
426    /**
427     *  Indicates if the local video stream is muted
428     *
429     *  @return returns true if the local video stream is currently muted
430     */
431    public boolean videoIsMuted() {
432        boolean isMuted = true;
433
434        if (!audioOnly && (null != localStream)) {
435            for (MediaStreamTrack eachTrack : localStream.videoTracks) {
436                if (eachTrack.enabled()) {
437                    isMuted = false;
438                }
439            }
440        }
441
442        return isMuted;
443    }
444
445
446    /**
447     *  Mute or unmute the local audio
448     *
449     *  @param mute If true, mute the audio. If false, unmute the audio
450     */
451    public void muteAudio(boolean mute) {
452        if ((null != localStream) && isActive()) {
453            for (MediaStreamTrack eachTrack : localStream.audioTracks) {
454                eachTrack.setEnabled(!mute);
455            }
456        }
457    }
458
459
460    /**
461     *  Indicates if the local audio stream is muted
462     *
463     *  @return returns true if the local audio stream is currently muted
464     */
465    public boolean audioIsMuted() {
466        boolean isMuted = true;
467
468        if (null != localStream) {
469            for (MediaStreamTrack eachTrack : localStream.audioTracks) {
470                if (eachTrack.enabled()) {
471                    isMuted = false;
472                }
473            }
474        }
475
476        return isMuted;
477    }
478
479
480    /**
481     *  Notify the call that the UI controls associated with rendering video are no longer available, such as during activity lifecycle changes
482     */
483    public void pause() {
484        if (videoSource != null) {
485            videoSource.stop();
486            videoSourceStopped = true;
487        }
488    }
489
490
491    /**
492     *  Notify the call that the UI controls associated with rendering video are available again
493     */
494    public void resume() {
495        if (videoSource != null && videoSourceStopped) {
496            videoSource.restart();
497        }
498    }
499
500
501    /**
502     *  Process a hangup message received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application.
503     */
504    public void hangupReceived() {
505        if (!isHangingUp) {
506            isHangingUp = true;
507
508            if (null != listenerReference) {
509                // Disconnect will clear the listenerReference, so grab a reference to the
510                // listener while it's still alive since the listener will be notified in a
511                // different (UI) thread
512                final Listener listener = listenerReference.get();
513
514                new Handler(Looper.getMainLooper()).post(new Runnable() {
515                    public void run() {
516                        if (null != listener) {
517                            listener.onHangup(RespokeCall.this);
518                        }
519                    }
520                });
521            }
522
523            disconnect();
524        }
525    }
526
527
528    /**
529     *  Process an answer message received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application.
530     *
531     *  @param remoteSDP        Remote SDP data
532     *  @param remoteConnection Remote connection that answered the call
533     */
534    public void answerReceived(JSONObject remoteSDP, String remoteConnection) {
535        if (isActive()) {
536            incomingSDP = remoteSDP;
537            toConnection = remoteConnection;
538
539            try {
540                JSONObject signalData = new JSONObject("{'signalType':'connected','version':'1.0'}");
541                signalData.put("target", directConnectionOnly ? "directConnection" : "call");
542                signalData.put("connectionId", toConnection);
543                signalData.put("sessionId", sessionID);
544                signalData.put("signalId", Respoke.makeGUID());
545
546                if (null != signalingChannel) {
547                    signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() {
548                        @Override
549                        public void onSuccess() {
550                            if (isActive()) {
551                                processRemoteSDP();
552
553                                if (null != listenerReference) {
554                                    final Listener listener = listenerReference.get();
555                                    if (null != listener) {
556                                        new Handler(Looper.getMainLooper()).post(new Runnable() {
557                                            public void run() {
558                                                if (isActive()) {
559                                                    listener.onConnected(RespokeCall.this);
560                                                }
561                                            }
562                                        });
563                                    }
564                                }
565                            }
566                        }
567
568                        @Override
569                        public void onError(final String errorMessage) {
570                            postErrorToListener(errorMessage);
571                        }
572                    });
573                }
574            } catch (JSONException e) {
575                postErrorToListener("Error encoding answer signal");
576            }
577        }
578    }
579
580
581    /**
582     *  Process a connected messsage received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application.
583     */
584    public void connectedReceived() {
585        if (null != listenerReference) {
586            final Listener listener = listenerReference.get();
587            if (null != listener) {
588                new Handler(Looper.getMainLooper()).post(new Runnable() {
589                    public void run() {
590                        if (isActive()) {
591                            listener.onConnected(RespokeCall.this);
592                        }
593                    }
594                });
595            }
596        }
597    }
598
599
600    /**
601     *  Process ICE candidates suggested by the remote endpoint. This is used internally to the SDK and should not be called directly by your client application.
602     *
603     *  @param candidates Array of candidates to evaluate
604     */
605    public void iceCandidatesReceived(JSONArray candidates) {
606        if (isActive()) {
607            for (int ii = 0; ii < candidates.length(); ii++) {
608                try {
609                    JSONObject eachCandidate = (JSONObject) candidates.get(ii);
610                    String mid = eachCandidate.getString("sdpMid");
611                    int sdpLineIndex = eachCandidate.getInt("sdpMLineIndex");
612                    String sdp = eachCandidate.getString("candidate");
613
614                    IceCandidate rtcCandidate = new IceCandidate(mid, sdpLineIndex, sdp);
615
616                    try {
617                        // Start critical block
618                        queuedRemoteCandidatesSemaphore.acquire();
619
620                        if (null != queuedRemoteCandidates) {
621                            queuedRemoteCandidates.add(rtcCandidate);
622                        } else {
623                            peerConnection.addIceCandidate(rtcCandidate);
624                        }
625
626                        // End critical block
627                        queuedRemoteCandidatesSemaphore.release();
628                    } catch (InterruptedException e) {
629                        Log.d(TAG, "Error with remote candidates semaphore");
630                    }
631
632                } catch (JSONException e) {
633                    Log.d(TAG, "Error processing remote ice candidate data");
634                }
635            }
636        }
637    }
638
639
640    /**
641     *  Indicates if the local client initiated the call
642     *
643     *  @return True if the local client initiated the call
644     */
645    public boolean isCaller() {
646        return caller;
647    }
648
649
650    /**
651     *  Retrieve the WebRTC peer connection handling the call
652     *
653     *  @return The WebRTC PeerConnection instance
654     */
655    public PeerConnection getPeerConnection() {
656        return peerConnection;
657    }
658
659
660    /**
661     *  Indicate whether a call is being setup or is in progress.
662     *
663     *  @return True if the call is active
664     */
665    public boolean isActive() {
666        return (!isHangingUp && (null != signalingChannel));
667    }
668
669
670    //** Private methods
671
672
673    private void disconnect() {
674        localStream = null;
675        localRender = null;
676        remoteRender = null;
677
678        if (peerConnection != null) {
679            peerConnection.dispose();
680            peerConnection = null;
681        }
682
683        if (videoSource != null) {
684            videoSource.dispose();
685            videoSource = null;
686        }
687
688        if (null != directConnection) {
689            directConnection.setListener(null);
690            directConnection = null;
691        }
692
693        if (null != signalingChannel) {
694            RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener();
695            if (null != signalingChannelListener) {
696                signalingChannelListener.callTerminated(this);
697            }
698        }
699
700        listenerReference = null;
701        endpoint = null;
702        toEndpointId = null;
703        toType = null;
704        signalingChannel = null;
705    }
706
707
708    private void processRemoteSDP() {
709        try {
710            String type = incomingSDP.getString("type");
711            String sdpString = incomingSDP.getString("sdp");
712
713            SessionDescription sdp = new SessionDescription(SessionDescription.Type.fromCanonicalForm(type), preferISAC(sdpString));
714            peerConnection.setRemoteDescription(this.sdpObserver, sdp);
715        } catch (JSONException e) {
716            postErrorToListener("Error processing remote SDP.");
717        }
718    }
719
720
721    private void getTurnServerCredentials(final Respoke.TaskCompletionListener completionListener) {
722        if (isActive()) {
723            // get TURN server credentials
724            signalingChannel.sendRESTMessage("get", "/v1/turn", null, new RespokeSignalingChannel.RESTListener() {
725                @Override
726                public void onSuccess(Object response) {
727                    if (isActive()) {
728                        JSONObject jsonResponse = (JSONObject) response;
729                        String username = "";
730                        String password = "";
731
732                        try {
733                            username = jsonResponse.getString("username");
734                            password = jsonResponse.getString("password");
735                        } catch (JSONException e) {
736                            // No auth info? Must be accessible without TURN
737                        }
738
739                        try {
740                            JSONArray uris = (JSONArray) jsonResponse.get("uris");
741
742                            for (int ii = 0; ii < uris.length(); ii++) {
743                                String eachUri = uris.getString(ii);
744
745                                PeerConnection.IceServer server = new PeerConnection.IceServer(eachUri, username, password);
746                                iceServers.add(server);
747                            }
748
749                            if (iceServers.size() > 0) {
750                                completionListener.onSuccess();
751                            } else {
752                                completionListener.onError("No ICE servers were found");
753                            }
754                        } catch (JSONException e) {
755                            completionListener.onError("Unexpected response from server");
756                        }
757                    }
758                }
759
760                @Override
761                public void onError(String errorMessage) {
762                    completionListener.onError(errorMessage);
763                }
764            });
765        }
766    }
767
768
769    private void initializePeerConnection(Context context) {
770
771        if (peerConnectionFactory == null) {
772            // peerConnectionFactory should only be alloc'd and setup once per program lifecycle.
773
774            PeerConnectionFactory.initializeFieldTrials(null);
775
776            if (!PeerConnectionFactory.initializeAndroidGlobals(context, true, true, true, VideoRendererGui.getEGLContext())) {
777                Log.d(TAG, "Failed to initializeAndroidGlobals");
778            }
779
780            peerConnectionFactory = new PeerConnectionFactory();
781        }
782
783        if ((null == remoteRender) && (null == localRender)) {
784            // If the client application did not provide UI elements on which to render video, force this to be an audio call
785            audioOnly = true;
786        }
787
788        MediaConstraints sdpMediaConstraints = new MediaConstraints();
789        sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", directConnectionOnly ? "false" : "true"));
790        sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", (directConnectionOnly || audioOnly) ? "false" : "true"));
791        sdpMediaConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true"));
792        sdpMediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"));
793
794        peerConnection = peerConnectionFactory.createPeerConnection(iceServers, sdpMediaConstraints, pcObserver);
795    }
796
797
798    private void addLocalStreams(Context context) {
799        AudioManager audioManager = ((AudioManager) context.getSystemService(Context.AUDIO_SERVICE));
800        // TODO(fischman): figure out how to do this Right(tm) and remove the suppression.
801        @SuppressWarnings("deprecation")
802        boolean isWiredHeadsetOn = audioManager.isWiredHeadsetOn();
803        audioManager.setMode(isWiredHeadsetOn ? AudioManager.MODE_IN_CALL : AudioManager.MODE_IN_COMMUNICATION);
804        audioManager.setSpeakerphoneOn(!isWiredHeadsetOn);
805
806        localStream = peerConnectionFactory.createLocalMediaStream("ARDAMS");
807
808        if (!audioOnly) {
809            VideoCapturer capturer = getVideoCapturer();
810            MediaConstraints videoConstraints = new MediaConstraints();
811            videoSource = peerConnectionFactory.createVideoSource(capturer, videoConstraints);
812            VideoTrack videoTrack = peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource);
813            videoTrack.addRenderer(new VideoRenderer(localRender));
814            localStream.addTrack(videoTrack);
815        }
816
817        localStream.addTrack(peerConnectionFactory.createAudioTrack("ARDAMSa0", peerConnectionFactory.createAudioSource(new MediaConstraints())));
818
819        peerConnection.addStream(localStream);
820    }
821
822
823    // Cycle through likely device names for the camera and return the first
824    // capturer that works, or crash if none do.
825    private VideoCapturer getVideoCapturer() {
826        String[] cameraFacing = { "front", "back" };
827        int[] cameraIndex = { 0, 1 };
828        int[] cameraOrientation = { 0, 90, 180, 270 };
829        for (String facing : cameraFacing) {
830            for (int index : cameraIndex) {
831                for (int orientation : cameraOrientation) {
832                    String name = "Camera " + index + ", Facing " + facing +
833                            ", Orientation " + orientation;
834                    VideoCapturer capturer = VideoCapturer.create(name);
835                    if (capturer != null) {
836                        //logAndToast("Using camera: " + name);
837                        Log.d(TAG, "Using camera: " + name);
838                        return capturer;
839                    }
840                }
841            }
842        }
843        throw new RuntimeException("Failed to open capturer");
844    }
845
846
847    private void createOffer() {
848        MediaConstraints sdpMediaConstraints = new MediaConstraints();
849        sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair(
850                "OfferToReceiveAudio", directConnectionOnly ? "false" : "true"));
851        sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair(
852                "OfferToReceiveVideo", (directConnectionOnly || audioOnly) ? "false" : "true"));
853
854        peerConnection.createOffer(sdpObserver, sdpMediaConstraints);
855    }
856
857
858    private void updateVideoViewLayout() {
859        //TODO
860    }
861
862
863    private void actuallyAddDirectConnection() {
864        if ((null != directConnection) && (directConnection.isActive())) {
865            // There is already an active direct connection, so ignore this
866        } else {
867            directConnection = new RespokeDirectConnection(this);
868            endpoint.setDirectConnection(directConnection);
869
870            new Handler(Looper.getMainLooper()).post(new Runnable() {
871                public void run() {
872                    if (isActive() && (null != listenerReference)) {
873                        Listener listener = listenerReference.get();
874                        if (null != listener) {
875                            listener.directConnectionAvailable(directConnection, endpoint);
876                        }
877                    }
878                }
879            });
880
881            if ((null != directConnection) && !caller && (null != signalingChannel)) {
882                RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener();
883                if (null != signalingChannelListener) {
884                    // Inform the client that a remote endpoint is attempting to open a direct connection
885                    signalingChannelListener.directConnectionAvailable(directConnection, endpoint);
886                }
887            }
888        }
889    }
890
891    public void directConnectionDidAccept(final Context context) {
892        getTurnServerCredentials(new Respoke.TaskCompletionListener() {
893            @Override
894            public void onSuccess() {
895                initializePeerConnection(context);
896
897                if (caller) {
898                    directConnection.createDataChannel();
899                    createOffer();
900                } else {
901                    processRemoteSDP();
902                }
903            }
904
905            @Override
906            public void onError(String errorMessage) {
907                postErrorToListener(errorMessage);
908            }
909        });
910    }
911
912
913    public void directConnectionDidOpen(RespokeDirectConnection sender) {
914
915    }
916
917
918    public void directConnectionDidClose(RespokeDirectConnection sender) {
919        if (sender == directConnection) {
920            directConnection = null;
921
922            if (null != endpoint) {
923                endpoint.setDirectConnection(null);
924            }
925        }
926    }
927
928
929    // Implementation detail: observe ICE & stream changes and react accordingly.
930    private class PCObserver implements PeerConnection.Observer {
931        @Override public void onIceCandidate(final IceCandidate candidate){
932            new Handler(Looper.getMainLooper()).post(new Runnable() {
933                public void run() {
934                    if (isActive()) {
935                        Log.d(TAG, "onIceCandidate");
936                        handleLocalCandidate(candidate);
937                    }
938                }
939            });
940        }
941
942        @Override public void onSignalingChange(
943            PeerConnection.SignalingState newState) {
944        }
945
946        @Override public void onIceConnectionChange(
947                PeerConnection.IceConnectionState newState) {
948            if (isActive()) {
949                if (newState == PeerConnection.IceConnectionState.CONNECTED) {
950                    Log.d(TAG, "ICE Connection connected");
951                } else if (newState == PeerConnection.IceConnectionState.FAILED) {
952                    Log.d(TAG, "ICE Connection FAILED");
953
954                    if (null != listenerReference) {
955                        // Disconnect will clear the listenerReference, so grab a reference to the
956                        // listener while it's still alive since the listener will be notified in a
957                        // different (UI) thread
958                        final Listener listener = listenerReference.get();
959
960                        if (null != listener) {
961                            new Handler(Looper.getMainLooper()).post(new Runnable() {
962                                public void run() {
963                                    if (isActive()) {
964                                        listener.onError("ICE Connection failed!", RespokeCall.this);
965                                        listener.onHangup(RespokeCall.this);
966                                    }
967                                }
968                            });
969                        }
970                    }
971
972                    disconnect();
973                } else {
974                    Log.d(TAG, "ICE Connection state: " + newState.toString());
975                }
976            }
977        }
978
979        @Override public void onIceGatheringChange(
980                PeerConnection.IceGatheringState newState) {
981            if (isActive()) {
982                Log.d(TAG, "ICE Gathering state: " + newState.toString());
983                if (newState == PeerConnection.IceGatheringState.COMPLETE) {
984                    sendFinalCandidates();
985                }
986            }
987        }
988
989        @Override public void onAddStream(final MediaStream stream){
990            new Handler(Looper.getMainLooper()).post(new Runnable() {
991                public void run() {
992                    if (isActive()) {
993                        if (stream.audioTracks.size() <= 1 && stream.videoTracks.size() <= 1) {
994                            if (stream.videoTracks.size() == 1) {
995                                stream.videoTracks.get(0).addRenderer(
996                                        new VideoRenderer(remoteRender));
997                            }
998                        } else {
999                            postErrorToListener("An invalid stream was added");
1000                        }
1001                    }
1002                }
1003            });
1004        }
1005
1006        @Override public void onRemoveStream(final MediaStream stream){
1007            new Handler(Looper.getMainLooper()).post(new Runnable() {
1008                public void run() {
1009                    if (isActive()) {
1010                        stream.videoTracks.get(0).dispose();
1011                    }
1012                }
1013            });
1014        }
1015
1016        @Override public void onDataChannel(final DataChannel dc) {
1017            if (isActive()) {
1018                if (null != directConnection) {
1019                    directConnection.peerConnectionDidOpenDataChannel(dc);
1020                } else {
1021                    Log.d(TAG, "Direct connection opened, but no object to handle it!");
1022                }
1023            }
1024        }
1025
1026        @Override public void onRenegotiationNeeded() {
1027            // No need to do anything; AppRTC follows a pre-agreed-upon
1028            // signaling/negotiation protocol.
1029        }
1030    }
1031
1032    private void handleLocalCandidate(IceCandidate candidate) {
1033        try {
1034            // Start critical block
1035            localCandidatesSemaphore.acquire();
1036
1037            // Collect candidates that are generated in addition to sending them immediately.
1038            // This allows us to send a 'finalCandidates' signal when the iceGatheringState has
1039            // changed to COMPLETED. 'finalCandidates' are used by the backend to smooth inter-op
1040            // between clients that generate trickle ice, and clients that do not support trickle ice.
1041            collectedLocalCandidates.add(candidate);
1042
1043            if (null != queuedLocalCandidates) {
1044                queuedLocalCandidates.add(candidate);
1045            } else {
1046                sendLocalCandidate(candidate);
1047            }
1048
1049            // End critical block
1050            localCandidatesSemaphore.release();
1051        } catch (InterruptedException e) {
1052            Log.d(TAG, "Error with local candidates semaphore");
1053        }
1054    }
1055
1056
1057    // Implementation detail: handle offer creation/signaling and answer setting,
1058    // as well as adding remote ICE candidates once the answer SDP is set.
1059    private class SDPObserver implements SdpObserver {
1060
1061        @Override public void onCreateSuccess(final SessionDescription origSdp) {
1062            //abortUnless(localSdp == null, "multiple SDP create?!?");
1063            final SessionDescription sdp = new SessionDescription(
1064                    origSdp.type, preferISAC(origSdp.description));
1065
1066            new Handler(Looper.getMainLooper()).post(new Runnable() {
1067                public void run() {
1068                    if (isActive()) {
1069                        Log.d(TAG, "onSuccess(Create SDP)");
1070                        peerConnection.setLocalDescription(sdpObserver, sdp);
1071
1072                        try {
1073                            JSONObject data = new JSONObject("{'version':'1.0'}");
1074                            data.put("target", directConnectionOnly ? "directConnection" : "call");
1075                            String type = sdp.type.toString().toLowerCase();
1076                            data.put("signalType", type);
1077                            data.put("sessionId", sessionID);
1078                            data.put("signalId", Respoke.makeGUID());
1079
1080                            JSONObject sdpJSON = new JSONObject();
1081                            sdpJSON.put("sdp", sdp.description);
1082                            sdpJSON.put("type", type);
1083
1084                            data.put("sessionDescription", sdpJSON);
1085
1086                            if (null != signalingChannel) {
1087                                signalingChannel.sendSignal(data, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() {
1088                                    @Override
1089                                    public void onSuccess() {
1090                                        drainLocalCandidates();
1091                                    }
1092
1093                                    @Override
1094                                    public void onError(String errorMessage) {
1095                                        postErrorToListener(errorMessage);
1096                                    }
1097                                });
1098                            }
1099                        } catch (JSONException e) {
1100                            postErrorToListener("Error encoding sdp");
1101                        }
1102                    }
1103                }
1104            });
1105        }
1106
1107
1108        @Override public void onSetSuccess() {
1109            new Handler(Looper.getMainLooper()).post(new Runnable() {
1110                public void run() {
1111                    if (isActive()) {
1112                        Log.d(TAG, "onSuccess(Set SDP)");
1113                        if (caller) {
1114                            if (peerConnection.getRemoteDescription() != null) {
1115                                // We've set our local offer and received & set the remote
1116                                // answer, so drain candidates.
1117                                drainRemoteCandidates();
1118                            }
1119                        } else {
1120                            if (peerConnection.getLocalDescription() == null) {
1121                                // We just set the remote offer, time to create our answer.
1122                                MediaConstraints sdpMediaConstraints = new MediaConstraints();
1123                                sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair(
1124                                        "OfferToReceiveAudio", "true"));
1125                                sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair(
1126                                        "OfferToReceiveVideo", audioOnly ? "false" : "true"));
1127
1128                                peerConnection.createAnswer(SDPObserver.this, sdpMediaConstraints);
1129                            } else {
1130                                drainRemoteCandidates();
1131                            }
1132                        }
1133                    }
1134                }
1135            });
1136        }
1137
1138        @Override public void onCreateFailure(final String error) {
1139            postErrorToListener("createSDP error: " + error);
1140        }
1141
1142        @Override public void onSetFailure(final String error) {
1143            postErrorToListener("setSDP error: " + error);
1144        }
1145
1146        private void drainRemoteCandidates() {
1147            try {
1148                // Start critical block
1149                queuedRemoteCandidatesSemaphore.acquire();
1150
1151                for (IceCandidate candidate : queuedRemoteCandidates) {
1152                    peerConnection.addIceCandidate(candidate);
1153                }
1154                queuedRemoteCandidates = null;
1155
1156                // End critical block
1157                queuedRemoteCandidatesSemaphore.release();
1158            } catch (InterruptedException e) {
1159                Log.d(TAG, "Error with remote candidates semaphore");
1160            }
1161        }
1162
1163        private void drainLocalCandidates() {
1164            try {
1165                // Start critical block
1166                localCandidatesSemaphore.acquire();
1167
1168                for (IceCandidate candidate : queuedLocalCandidates) {
1169                    sendLocalCandidate(candidate);
1170                }
1171                queuedLocalCandidates = null;
1172
1173                // End critical block
1174                localCandidatesSemaphore.release();
1175            } catch (InterruptedException e) {
1176                Log.d(TAG, "Error with local candidates semaphore");
1177            }
1178        }
1179    }
1180
1181    private JSONObject getCandidateDict(IceCandidate candidate) {
1182        JSONObject result = new JSONObject();
1183
1184        try {
1185            result.put("sdpMLineIndex", candidate.sdpMLineIndex);
1186            result.put("sdpMid", candidate.sdpMid);
1187            result.put("candidate", candidate.sdp);
1188        } catch (JSONException e) {
1189            postErrorToListener("Unable to encode local candidate");
1190        }
1191
1192        return result;
1193    }
1194
1195    private JSONArray getLocalCandidateJSONArray() {
1196        JSONArray result = new JSONArray();
1197
1198        try {
1199            // Begin critical block
1200            localCandidatesSemaphore.acquire();
1201
1202            for (IceCandidate candidate: collectedLocalCandidates) {
1203                result.put(getCandidateDict(candidate));
1204            }
1205
1206            // End critical block
1207            localCandidatesSemaphore.release();
1208        } catch (InterruptedException e) {
1209            e.printStackTrace();
1210        }
1211
1212        return result;
1213    }
1214
1215    private void sendFinalCandidates() {
1216        Log.d(TAG, "Sending final candidates");
1217        JSONObject signalData;
1218        try {
1219            signalData = new JSONObject("{ 'signalType': 'iceCandidates', 'version': '1.0' }");
1220            signalData.put("target", directConnectionOnly ? "directConnection" : "call");
1221            signalData.put("sessionId", sessionID);
1222            signalData.put("signalId", Respoke.makeGUID());
1223            signalData.put("iceCandidates", new JSONArray());
1224            signalData.put("finalCandidates", getLocalCandidateJSONArray());
1225
1226            if (null != signalingChannel) {
1227                signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() {
1228                    @Override
1229                    public void onSuccess() {
1230                        // Do nothing
1231                    }
1232
1233                    @Override
1234                    public void onError(String errorMessage) {
1235                        postErrorToListener(errorMessage);
1236                    }
1237                });
1238            }
1239        } catch (JSONException e) {
1240            postErrorToListener("Error encoding signal to send final candidates");
1241        }
1242    }
1243
1244    private void sendLocalCandidate(IceCandidate candidate) {
1245        JSONArray candidateArray = new JSONArray();
1246        try {
1247            candidateArray.put(getCandidateDict(candidate));
1248
1249            JSONObject signalData = new JSONObject("{'signalType':'iceCandidates','version':'1.0'}");
1250            signalData.put("target", directConnectionOnly ? "directConnection" : "call");
1251            signalData.put("sessionId", sessionID);
1252            signalData.put("signalId", Respoke.makeGUID());
1253            signalData.put("iceCandidates", candidateArray);
1254
1255            if (null != signalingChannel) {
1256                signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() {
1257                    @Override
1258                    public void onSuccess() {
1259                        // Do nothing
1260                    }
1261
1262                    @Override
1263                    public void onError(String errorMessage) {
1264                        postErrorToListener(errorMessage);
1265                    }
1266                });
1267            }
1268        } catch (JSONException e) {
1269            postErrorToListener("Error encoding signal to send local candidate");
1270        }
1271    }
1272
1273
1274    // Mangle SDP to prefer ISAC/16000 over any other audio codec.
1275    private static String preferISAC(String sdpDescription) {
1276        String[] lines = sdpDescription.split("\r\n");
1277        int mLineIndex = -1;
1278        String isac16kRtpMap = null;
1279        Pattern isac16kPattern =
1280                Pattern.compile("^a=rtpmap:(\\d+) ISAC/16000[\r]?$");
1281        for (int i = 0;
1282             (i < lines.length) && (mLineIndex == -1 || isac16kRtpMap == null);
1283             ++i) {
1284            if (lines[i].startsWith("m=audio ")) {
1285                mLineIndex = i;
1286                continue;
1287            }
1288            Matcher isac16kMatcher = isac16kPattern.matcher(lines[i]);
1289            if (isac16kMatcher.matches()) {
1290                isac16kRtpMap = isac16kMatcher.group(1);
1291                //continue;
1292            }
1293        }
1294        if (mLineIndex == -1) {
1295            //Log.d(TAG, "No m=audio line, so can't prefer iSAC");
1296            return sdpDescription;
1297        }
1298        if (isac16kRtpMap == null) {
1299            //Log.d(TAG, "No ISAC/16000 line, so can't prefer iSAC");
1300            return sdpDescription;
1301        }
1302        String[] origMLineParts = lines[mLineIndex].split(" ");
1303        StringBuilder newMLine = new StringBuilder();
1304        int origPartIndex = 0;
1305        // Format is: m=<media> <port> <proto> <fmt> ...
1306        newMLine.append(origMLineParts[origPartIndex++]).append(" ");
1307        newMLine.append(origMLineParts[origPartIndex++]).append(" ");
1308        newMLine.append(origMLineParts[origPartIndex++]).append(" ");
1309        newMLine.append(isac16kRtpMap);
1310        for (; origPartIndex < origMLineParts.length; ++origPartIndex) {
1311            if (!origMLineParts[origPartIndex].equals(isac16kRtpMap)) {
1312                newMLine.append(" ").append(origMLineParts[origPartIndex]);
1313            }
1314        }
1315        lines[mLineIndex] = newMLine.toString();
1316        StringBuilder newSdpDescription = new StringBuilder();
1317        for (String line : lines) {
1318            newSdpDescription.append(line).append("\r\n");
1319        }
1320        return newSdpDescription.toString();
1321    }
1322
1323
1324    private void postErrorToListener(final String errorMessage) {
1325        // All listener methods should be called from the UI thread
1326        new Handler(Looper.getMainLooper()).post(new Runnable() {
1327            public void run() {
1328                if ((isActive()) && (null != listenerReference)) {
1329                    Listener listener = listenerReference.get();
1330                    if (null != listener) {
1331                        listener.onError(errorMessage, RespokeCall.this);
1332                    }
1333                }
1334            }
1335        });
1336    }
1337
1338
1339}