001/** 002 * Copyright 2015, Digium, Inc. 003 * All rights reserved. 004 * 005 * This source code is licensed under The MIT License found in the 006 * LICENSE file in the root directory of this source tree. 007 * 008 * For all details and documentation: https://www.respoke.io 009 */ 010 011package com.digium.respokesdk; 012 013import android.content.Context; 014import android.media.AudioManager; 015import android.opengl.GLSurfaceView; 016import android.os.Handler; 017import android.os.Looper; 018import android.util.Log; 019 020import org.json.JSONArray; 021import org.json.JSONException; 022import org.json.JSONObject; 023import org.webrtc.DataChannel; 024import org.webrtc.IceCandidate; 025import org.webrtc.MediaConstraints; 026import org.webrtc.MediaStream; 027import org.webrtc.MediaStreamTrack; 028import org.webrtc.PeerConnection; 029import org.webrtc.PeerConnectionFactory; 030import org.webrtc.SdpObserver; 031import org.webrtc.SessionDescription; 032import org.webrtc.VideoCapturer; 033import org.webrtc.VideoRenderer; 034import org.webrtc.VideoRendererGui; 035import org.webrtc.VideoSource; 036import org.webrtc.VideoTrack; 037 038import java.lang.ref.WeakReference; 039import java.util.ArrayList; 040import java.util.Date; 041import java.util.concurrent.Semaphore; 042import java.util.regex.Matcher; 043import java.util.regex.Pattern; 044 045 046/** 047 * WebRTC Call including getUserMedia, path and codec negotation, and call state. 048 */ 049public class RespokeCall { 050 051 private final static String TAG = "RespokeCall"; 052 private WeakReference<Listener> listenerReference; 053 private RespokeSignalingChannel signalingChannel; 054 private ArrayList<PeerConnection.IceServer> iceServers; 055 private static PeerConnectionFactory peerConnectionFactory; 056 private PeerConnection peerConnection; 057 private VideoSource videoSource; 058 private ArrayList<IceCandidate> queuedRemoteCandidates; 059 private ArrayList<IceCandidate> queuedLocalCandidates; 060 private ArrayList<IceCandidate> collectedLocalCandidates; 061 private Semaphore queuedRemoteCandidatesSemaphore; 062 private Semaphore localCandidatesSemaphore; 063 private org.webrtc.VideoRenderer.Callbacks localRender; 064 private org.webrtc.VideoRenderer.Callbacks remoteRender; 065 private boolean caller; 066 private JSONObject incomingSDP; 067 068 // Addressing and call identification fields 069 private String sessionID; 070 private String toConnection; 071 private String toEndpointId; 072 // Options are web, conference, did, or sip 073 private String toType; 074 075 // Used for direct connections 076 public RespokeEndpoint endpoint; 077 public boolean audioOnly; 078 public Date timestamp; 079 private final PCObserver pcObserver = new PCObserver(); 080 private final SDPObserver sdpObserver = new SDPObserver(); 081 private boolean videoSourceStopped; 082 private MediaStream localStream; 083 private boolean directConnectionOnly; 084 private RespokeDirectConnection directConnection; 085 private boolean isHangingUp; 086 087 088 /** 089 * An interface to notify the receiver of events occurring with the call 090 */ 091 public interface Listener { 092 093 094 /** 095 * Receive a notification that an error has occurred while on a call 096 * 097 * @param errorMessage A human-readable description of the error. 098 * @param sender The RespokeCall that experienced the error 099 */ 100 public void onError(String errorMessage, RespokeCall sender); 101 102 103 /** 104 * When on a call, receive notification the call has been hung up 105 * 106 * @param sender The RespokeCall that has hung up 107 */ 108 public void onHangup(RespokeCall sender); 109 110 111 /** 112 * When on a call, receive remote media when it becomes available. This is what you will need to provide if you want 113 * to show the user the other party's video during a call. 114 * 115 * @param sender The RespokeCall that has connected 116 */ 117 public void onConnected(RespokeCall sender); 118 119 120 /** 121 * This event is fired when the local end of the directConnection is available. It still will not be 122 * ready to send and receive messages until the 'open' event fires. 123 * 124 * @param directConnection The direct connection object 125 * @param endpoint The remote endpoint 126 */ 127 public void directConnectionAvailable(RespokeDirectConnection directConnection, RespokeEndpoint endpoint); 128 } 129 130 131 /** 132 * Determines if the specified SDP data contains definitions for a video stream 133 * 134 * @param sdp The SDP data to examine 135 * 136 * @return True if a video stream definition is present, false otherwise 137 */ 138 public static boolean sdpHasVideo(JSONObject sdp) { 139 boolean hasVideo = false; 140 141 if (null != sdp) { 142 try { 143 String sdpString = sdp.getString("sdp"); 144 hasVideo = sdpString.contains("m=video"); 145 } catch (JSONException e) { 146 // Bad SDP? 147 Log.d(TAG, "ERROR: Incoming call appears to have an invalid SDP"); 148 } 149 } 150 151 return hasVideo; 152 } 153 154 155 /** 156 * Constructor primarily used for starting conference calls 157 * 158 * @param channel The signaling channel to use for the call 159 * @param remoteEndpoint The remote recipient of the call 160 * @param remoteType The type of remote recipient (i.e. "conference", "web", etc) 161 */ 162 public RespokeCall(RespokeSignalingChannel channel, String remoteEndpoint, String remoteType) { 163 commonConstructor(channel); 164 toEndpointId = remoteEndpoint; 165 toType = remoteType; 166 } 167 168 169 /** 170 * Constructor used for outbound calls 171 * 172 * @param channel The signaling channel to use for the call 173 * @param newEndpoint The remote recipient of the call 174 * @param directConnectionOnly Specify true if this call is only for establishing a direct data connection (i.e. no audio/video) 175 */ 176 public RespokeCall(RespokeSignalingChannel channel, RespokeEndpoint newEndpoint, boolean directConnectionOnly) { 177 commonConstructor(channel); 178 179 endpoint = newEndpoint; 180 toEndpointId = newEndpoint.getEndpointID(); 181 toType = "web"; 182 183 this.directConnectionOnly = directConnectionOnly; 184 } 185 186 187 /** 188 * Constructor used for inbound calls 189 * 190 * @param channel The signaling channel to use for the call 191 * @param sdp The SDP data from the call offer 192 * @param newSessionID The session ID to use for the call signaling 193 * @param newConnectionID The ID of the remote connection initiating the call 194 * @param endpointID The ID of the remote endpoint 195 * @param fromType The type of remote recipient (i.e. "conference", "web", etc) 196 * @param newEndpoint The remote recipient of the call 197 * @param directConnectionOnly Specify true if this call is only for establishing a direct data connection (i.e. no audio/video) 198 * @param newTimestamp The timestamp when the call was initiated remotely 199 */ 200 public RespokeCall(RespokeSignalingChannel channel, JSONObject sdp, String newSessionID, String newConnectionID, String endpointID, String fromType, RespokeEndpoint newEndpoint, boolean directConnectionOnly, Date newTimestamp) { 201 commonConstructor(channel); 202 203 incomingSDP = sdp; 204 sessionID = newSessionID; 205 endpoint = newEndpoint; 206 toEndpointId = endpointID; 207 toType = fromType; 208 209 if (fromType == null) { 210 toType = "web"; 211 } 212 213 toConnection = newConnectionID; 214 this.directConnectionOnly = directConnectionOnly; 215 timestamp = newTimestamp; 216 audioOnly = !RespokeCall.sdpHasVideo(sdp); 217 218 if ((directConnectionOnly) && (endpoint != null)) { 219 actuallyAddDirectConnection(); 220 } 221 } 222 223 224 /** 225 * Common constructor logic 226 * 227 * @param channel The signaling channel to use for the call 228 */ 229 private void commonConstructor(RespokeSignalingChannel channel) { 230 signalingChannel = channel; 231 iceServers = new ArrayList<PeerConnection.IceServer>(); 232 queuedLocalCandidates = new ArrayList<IceCandidate>(); 233 queuedRemoteCandidates = new ArrayList<IceCandidate>(); 234 collectedLocalCandidates = new ArrayList<IceCandidate>(); 235 sessionID = Respoke.makeGUID(); 236 timestamp = new Date(); 237 queuedRemoteCandidatesSemaphore = new Semaphore(1); // remote candidates queue mutex 238 localCandidatesSemaphore = new Semaphore(1); // local candidates queue mutex 239 240 if (null != signalingChannel) { 241 RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener(); 242 if (null != signalingChannelListener) { 243 signalingChannelListener.callCreated(this); 244 } 245 } 246 247 //TODO resign active handler? 248 } 249 250 251 /** 252 * Set a receiver for the Listener interface 253 * 254 * @param listener The new receiver for events from the Listener interface for this call instance 255 */ 256 public void setListener(Listener listener) { 257 listenerReference = new WeakReference<Listener>(listener); 258 } 259 260 261 /** 262 * Get the session ID of this call 263 * 264 * @return The session ID 265 */ 266 public String getSessionID() { 267 return sessionID; 268 } 269 270 271 /** 272 * Start the outgoing call process. This method is used internally by the SDK and should never be called directly from your client application 273 * 274 * @param context An application context with which to access shared resources 275 * @param glView The GLSurfaceView on which to render video if applicable 276 * @param isAudioOnly Specify true if this call should be audio only 277 */ 278 public void startCall(final Context context, GLSurfaceView glView, boolean isAudioOnly) { 279 caller = true; 280 audioOnly = isAudioOnly; 281 282 if (directConnectionOnly) { 283 if (null == directConnection) { 284 actuallyAddDirectConnection(); 285 } 286 287 directConnectionDidAccept(context); 288 } else { 289 attachVideoRenderer(glView); 290 291 getTurnServerCredentials(new Respoke.TaskCompletionListener() { 292 @Override 293 public void onSuccess() { 294 Log.d(TAG, "Got TURN credentials"); 295 initializePeerConnection(context); 296 addLocalStreams(context); 297 createOffer(); 298 } 299 300 @Override 301 public void onError(String errorMessage) { 302 postErrorToListener(errorMessage); 303 } 304 }); 305 } 306 } 307 308 309 /** 310 * Attach the call's video renderers to the specified GLSurfaceView 311 * 312 * @param glView The GLSurfaceView on which to render video 313 */ 314 public void attachVideoRenderer(GLSurfaceView glView) { 315 if (null != glView) { 316 VideoRendererGui.setView(glView, new Runnable() { 317 @Override 318 public void run() { 319 Log.d(TAG, "VideoRendererGui GL Context ready"); 320 } 321 }); 322 323 remoteRender = VideoRendererGui.create(0, 0, 100, 100, 324 VideoRendererGui.ScalingType.SCALE_ASPECT_FILL, false); 325 localRender = VideoRendererGui.create(70, 5, 25, 25, 326 VideoRendererGui.ScalingType.SCALE_ASPECT_FILL, false); 327 } 328 } 329 330 331 /** 332 * Answer the call and start the process of obtaining media. This method is called automatically on the caller's 333 * side. This method must be called on the callee's side to indicate that the endpoint does wish to accept the 334 * call. 335 * 336 * @param context An application context with which to access shared resources 337 * @param newListener A listener to receive notifications of call-related events 338 */ 339 public void answer(final Context context, Listener newListener) { 340 if (!caller) { 341 listenerReference = new WeakReference<Listener>(newListener); 342 343 getTurnServerCredentials(new Respoke.TaskCompletionListener() { 344 @Override 345 public void onSuccess() { 346 initializePeerConnection(context); 347 addLocalStreams(context); 348 processRemoteSDP(); 349 } 350 351 @Override 352 public void onError(String errorMessage) { 353 postErrorToListener(errorMessage); 354 } 355 }); 356 } 357 } 358 359 360 /** 361 * Tear down the call and release resources 362 * 363 * @param shouldSendHangupSignal Send a hangup signal to the remote party if signal is not false and we have not received a hangup signal from the remote party. 364 */ 365 public void hangup(boolean shouldSendHangupSignal) { 366 if (!isHangingUp) { 367 isHangingUp = true; 368 369 if (shouldSendHangupSignal) { 370 371 try { 372 JSONObject data = new JSONObject("{'signalType':'bye','version':'1.0'}"); 373 data.put("target", directConnectionOnly ? "directConnection" : "call"); 374 data.put("sessionId", sessionID); 375 data.put("signalId", Respoke.makeGUID()); 376 377 // Keep a second reference to the listener since the disconnect method will clear it before the success handler is fired 378 final WeakReference<Listener> hangupListener = listenerReference; 379 380 if (null != signalingChannel) { 381 signalingChannel.sendSignal(data, toEndpointId, toConnection, toType, true, new Respoke.TaskCompletionListener() { 382 @Override 383 public void onSuccess() { 384 if (null != hangupListener) { 385 new Handler(Looper.getMainLooper()).post(new Runnable() { 386 public void run() { 387 Listener listener = hangupListener.get(); 388 if (null != listener) { 389 listener.onHangup(RespokeCall.this); 390 } 391 } 392 }); 393 } 394 } 395 396 @Override 397 public void onError(String errorMessage) { 398 postErrorToListener(errorMessage); 399 } 400 }); 401 } 402 } catch (JSONException e) { 403 postErrorToListener("Error encoding signal to json"); 404 } 405 } 406 407 disconnect(); 408 } 409 } 410 411 412 /** 413 * Mute or unmute the local video 414 * 415 * @param mute If true, mute the video. If false, unmute the video 416 */ 417 public void muteVideo(boolean mute) { 418 if (!audioOnly && (null != localStream) && (isActive())) { 419 for (MediaStreamTrack eachTrack : localStream.videoTracks) { 420 eachTrack.setEnabled(!mute); 421 } 422 } 423 } 424 425 426 /** 427 * Indicates if the local video stream is muted 428 * 429 * @return returns true if the local video stream is currently muted 430 */ 431 public boolean videoIsMuted() { 432 boolean isMuted = true; 433 434 if (!audioOnly && (null != localStream)) { 435 for (MediaStreamTrack eachTrack : localStream.videoTracks) { 436 if (eachTrack.enabled()) { 437 isMuted = false; 438 } 439 } 440 } 441 442 return isMuted; 443 } 444 445 446 /** 447 * Mute or unmute the local audio 448 * 449 * @param mute If true, mute the audio. If false, unmute the audio 450 */ 451 public void muteAudio(boolean mute) { 452 if ((null != localStream) && isActive()) { 453 for (MediaStreamTrack eachTrack : localStream.audioTracks) { 454 eachTrack.setEnabled(!mute); 455 } 456 } 457 } 458 459 460 /** 461 * Indicates if the local audio stream is muted 462 * 463 * @return returns true if the local audio stream is currently muted 464 */ 465 public boolean audioIsMuted() { 466 boolean isMuted = true; 467 468 if (null != localStream) { 469 for (MediaStreamTrack eachTrack : localStream.audioTracks) { 470 if (eachTrack.enabled()) { 471 isMuted = false; 472 } 473 } 474 } 475 476 return isMuted; 477 } 478 479 480 /** 481 * Notify the call that the UI controls associated with rendering video are no longer available, such as during activity lifecycle changes 482 */ 483 public void pause() { 484 if (videoSource != null) { 485 videoSource.stop(); 486 videoSourceStopped = true; 487 } 488 } 489 490 491 /** 492 * Notify the call that the UI controls associated with rendering video are available again 493 */ 494 public void resume() { 495 if (videoSource != null && videoSourceStopped) { 496 videoSource.restart(); 497 } 498 } 499 500 501 /** 502 * Process a hangup message received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application. 503 */ 504 public void hangupReceived() { 505 if (!isHangingUp) { 506 isHangingUp = true; 507 508 if (null != listenerReference) { 509 // Disconnect will clear the listenerReference, so grab a reference to the 510 // listener while it's still alive since the listener will be notified in a 511 // different (UI) thread 512 final Listener listener = listenerReference.get(); 513 514 new Handler(Looper.getMainLooper()).post(new Runnable() { 515 public void run() { 516 if (null != listener) { 517 listener.onHangup(RespokeCall.this); 518 } 519 } 520 }); 521 } 522 523 disconnect(); 524 } 525 } 526 527 528 /** 529 * Process an answer message received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application. 530 * 531 * @param remoteSDP Remote SDP data 532 * @param remoteConnection Remote connection that answered the call 533 */ 534 public void answerReceived(JSONObject remoteSDP, String remoteConnection) { 535 if (isActive()) { 536 incomingSDP = remoteSDP; 537 toConnection = remoteConnection; 538 539 try { 540 JSONObject signalData = new JSONObject("{'signalType':'connected','version':'1.0'}"); 541 signalData.put("target", directConnectionOnly ? "directConnection" : "call"); 542 signalData.put("connectionId", toConnection); 543 signalData.put("sessionId", sessionID); 544 signalData.put("signalId", Respoke.makeGUID()); 545 546 if (null != signalingChannel) { 547 signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() { 548 @Override 549 public void onSuccess() { 550 if (isActive()) { 551 processRemoteSDP(); 552 553 if (null != listenerReference) { 554 final Listener listener = listenerReference.get(); 555 if (null != listener) { 556 new Handler(Looper.getMainLooper()).post(new Runnable() { 557 public void run() { 558 if (isActive()) { 559 listener.onConnected(RespokeCall.this); 560 } 561 } 562 }); 563 } 564 } 565 } 566 } 567 568 @Override 569 public void onError(final String errorMessage) { 570 postErrorToListener(errorMessage); 571 } 572 }); 573 } 574 } catch (JSONException e) { 575 postErrorToListener("Error encoding answer signal"); 576 } 577 } 578 } 579 580 581 /** 582 * Process a connected messsage received from the remote endpoint. This is used internally to the SDK and should not be called directly by your client application. 583 */ 584 public void connectedReceived() { 585 if (null != listenerReference) { 586 final Listener listener = listenerReference.get(); 587 if (null != listener) { 588 new Handler(Looper.getMainLooper()).post(new Runnable() { 589 public void run() { 590 if (isActive()) { 591 listener.onConnected(RespokeCall.this); 592 } 593 } 594 }); 595 } 596 } 597 } 598 599 600 /** 601 * Process ICE candidates suggested by the remote endpoint. This is used internally to the SDK and should not be called directly by your client application. 602 * 603 * @param candidates Array of candidates to evaluate 604 */ 605 public void iceCandidatesReceived(JSONArray candidates) { 606 if (isActive()) { 607 for (int ii = 0; ii < candidates.length(); ii++) { 608 try { 609 JSONObject eachCandidate = (JSONObject) candidates.get(ii); 610 String mid = eachCandidate.getString("sdpMid"); 611 int sdpLineIndex = eachCandidate.getInt("sdpMLineIndex"); 612 String sdp = eachCandidate.getString("candidate"); 613 614 IceCandidate rtcCandidate = new IceCandidate(mid, sdpLineIndex, sdp); 615 616 try { 617 // Start critical block 618 queuedRemoteCandidatesSemaphore.acquire(); 619 620 if (null != queuedRemoteCandidates) { 621 queuedRemoteCandidates.add(rtcCandidate); 622 } else { 623 peerConnection.addIceCandidate(rtcCandidate); 624 } 625 626 // End critical block 627 queuedRemoteCandidatesSemaphore.release(); 628 } catch (InterruptedException e) { 629 Log.d(TAG, "Error with remote candidates semaphore"); 630 } 631 632 } catch (JSONException e) { 633 Log.d(TAG, "Error processing remote ice candidate data"); 634 } 635 } 636 } 637 } 638 639 640 /** 641 * Indicates if the local client initiated the call 642 * 643 * @return True if the local client initiated the call 644 */ 645 public boolean isCaller() { 646 return caller; 647 } 648 649 650 /** 651 * Retrieve the WebRTC peer connection handling the call 652 * 653 * @return The WebRTC PeerConnection instance 654 */ 655 public PeerConnection getPeerConnection() { 656 return peerConnection; 657 } 658 659 660 /** 661 * Indicate whether a call is being setup or is in progress. 662 * 663 * @return True if the call is active 664 */ 665 public boolean isActive() { 666 return (!isHangingUp && (null != signalingChannel)); 667 } 668 669 670 //** Private methods 671 672 673 private void disconnect() { 674 localStream = null; 675 localRender = null; 676 remoteRender = null; 677 678 if (peerConnection != null) { 679 peerConnection.dispose(); 680 peerConnection = null; 681 } 682 683 if (videoSource != null) { 684 videoSource.dispose(); 685 videoSource = null; 686 } 687 688 if (null != directConnection) { 689 directConnection.setListener(null); 690 directConnection = null; 691 } 692 693 if (null != signalingChannel) { 694 RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener(); 695 if (null != signalingChannelListener) { 696 signalingChannelListener.callTerminated(this); 697 } 698 } 699 700 listenerReference = null; 701 endpoint = null; 702 toEndpointId = null; 703 toType = null; 704 signalingChannel = null; 705 } 706 707 708 private void processRemoteSDP() { 709 try { 710 String type = incomingSDP.getString("type"); 711 String sdpString = incomingSDP.getString("sdp"); 712 713 SessionDescription sdp = new SessionDescription(SessionDescription.Type.fromCanonicalForm(type), preferISAC(sdpString)); 714 peerConnection.setRemoteDescription(this.sdpObserver, sdp); 715 } catch (JSONException e) { 716 postErrorToListener("Error processing remote SDP."); 717 } 718 } 719 720 721 private void getTurnServerCredentials(final Respoke.TaskCompletionListener completionListener) { 722 if (isActive()) { 723 // get TURN server credentials 724 signalingChannel.sendRESTMessage("get", "/v1/turn", null, new RespokeSignalingChannel.RESTListener() { 725 @Override 726 public void onSuccess(Object response) { 727 if (isActive()) { 728 JSONObject jsonResponse = (JSONObject) response; 729 String username = ""; 730 String password = ""; 731 732 try { 733 username = jsonResponse.getString("username"); 734 password = jsonResponse.getString("password"); 735 } catch (JSONException e) { 736 // No auth info? Must be accessible without TURN 737 } 738 739 try { 740 JSONArray uris = (JSONArray) jsonResponse.get("uris"); 741 742 for (int ii = 0; ii < uris.length(); ii++) { 743 String eachUri = uris.getString(ii); 744 745 PeerConnection.IceServer server = new PeerConnection.IceServer(eachUri, username, password); 746 iceServers.add(server); 747 } 748 749 if (iceServers.size() > 0) { 750 completionListener.onSuccess(); 751 } else { 752 completionListener.onError("No ICE servers were found"); 753 } 754 } catch (JSONException e) { 755 completionListener.onError("Unexpected response from server"); 756 } 757 } 758 } 759 760 @Override 761 public void onError(String errorMessage) { 762 completionListener.onError(errorMessage); 763 } 764 }); 765 } 766 } 767 768 769 private void initializePeerConnection(Context context) { 770 771 if (peerConnectionFactory == null) { 772 // peerConnectionFactory should only be alloc'd and setup once per program lifecycle. 773 774 PeerConnectionFactory.initializeFieldTrials(null); 775 776 if (!PeerConnectionFactory.initializeAndroidGlobals(context, true, true, true, VideoRendererGui.getEGLContext())) { 777 Log.d(TAG, "Failed to initializeAndroidGlobals"); 778 } 779 780 peerConnectionFactory = new PeerConnectionFactory(); 781 } 782 783 if ((null == remoteRender) && (null == localRender)) { 784 // If the client application did not provide UI elements on which to render video, force this to be an audio call 785 audioOnly = true; 786 } 787 788 MediaConstraints sdpMediaConstraints = new MediaConstraints(); 789 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", directConnectionOnly ? "false" : "true")); 790 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", (directConnectionOnly || audioOnly) ? "false" : "true")); 791 sdpMediaConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true")); 792 sdpMediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); 793 794 peerConnection = peerConnectionFactory.createPeerConnection(iceServers, sdpMediaConstraints, pcObserver); 795 } 796 797 798 private void addLocalStreams(Context context) { 799 AudioManager audioManager = ((AudioManager) context.getSystemService(Context.AUDIO_SERVICE)); 800 // TODO(fischman): figure out how to do this Right(tm) and remove the suppression. 801 @SuppressWarnings("deprecation") 802 boolean isWiredHeadsetOn = audioManager.isWiredHeadsetOn(); 803 audioManager.setMode(isWiredHeadsetOn ? AudioManager.MODE_IN_CALL : AudioManager.MODE_IN_COMMUNICATION); 804 audioManager.setSpeakerphoneOn(!isWiredHeadsetOn); 805 806 localStream = peerConnectionFactory.createLocalMediaStream("ARDAMS"); 807 808 if (!audioOnly) { 809 VideoCapturer capturer = getVideoCapturer(); 810 MediaConstraints videoConstraints = new MediaConstraints(); 811 videoSource = peerConnectionFactory.createVideoSource(capturer, videoConstraints); 812 VideoTrack videoTrack = peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); 813 videoTrack.addRenderer(new VideoRenderer(localRender)); 814 localStream.addTrack(videoTrack); 815 } 816 817 localStream.addTrack(peerConnectionFactory.createAudioTrack("ARDAMSa0", peerConnectionFactory.createAudioSource(new MediaConstraints()))); 818 819 peerConnection.addStream(localStream); 820 } 821 822 823 // Cycle through likely device names for the camera and return the first 824 // capturer that works, or crash if none do. 825 private VideoCapturer getVideoCapturer() { 826 String[] cameraFacing = { "front", "back" }; 827 int[] cameraIndex = { 0, 1 }; 828 int[] cameraOrientation = { 0, 90, 180, 270 }; 829 for (String facing : cameraFacing) { 830 for (int index : cameraIndex) { 831 for (int orientation : cameraOrientation) { 832 String name = "Camera " + index + ", Facing " + facing + 833 ", Orientation " + orientation; 834 VideoCapturer capturer = VideoCapturer.create(name); 835 if (capturer != null) { 836 //logAndToast("Using camera: " + name); 837 Log.d(TAG, "Using camera: " + name); 838 return capturer; 839 } 840 } 841 } 842 } 843 throw new RuntimeException("Failed to open capturer"); 844 } 845 846 847 private void createOffer() { 848 MediaConstraints sdpMediaConstraints = new MediaConstraints(); 849 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair( 850 "OfferToReceiveAudio", directConnectionOnly ? "false" : "true")); 851 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair( 852 "OfferToReceiveVideo", (directConnectionOnly || audioOnly) ? "false" : "true")); 853 854 peerConnection.createOffer(sdpObserver, sdpMediaConstraints); 855 } 856 857 858 private void updateVideoViewLayout() { 859 //TODO 860 } 861 862 863 private void actuallyAddDirectConnection() { 864 if ((null != directConnection) && (directConnection.isActive())) { 865 // There is already an active direct connection, so ignore this 866 } else { 867 directConnection = new RespokeDirectConnection(this); 868 endpoint.setDirectConnection(directConnection); 869 870 new Handler(Looper.getMainLooper()).post(new Runnable() { 871 public void run() { 872 if (isActive() && (null != listenerReference)) { 873 Listener listener = listenerReference.get(); 874 if (null != listener) { 875 listener.directConnectionAvailable(directConnection, endpoint); 876 } 877 } 878 } 879 }); 880 881 if ((null != directConnection) && !caller && (null != signalingChannel)) { 882 RespokeSignalingChannel.Listener signalingChannelListener = signalingChannel.GetListener(); 883 if (null != signalingChannelListener) { 884 // Inform the client that a remote endpoint is attempting to open a direct connection 885 signalingChannelListener.directConnectionAvailable(directConnection, endpoint); 886 } 887 } 888 } 889 } 890 891 public void directConnectionDidAccept(final Context context) { 892 getTurnServerCredentials(new Respoke.TaskCompletionListener() { 893 @Override 894 public void onSuccess() { 895 initializePeerConnection(context); 896 897 if (caller) { 898 directConnection.createDataChannel(); 899 createOffer(); 900 } else { 901 processRemoteSDP(); 902 } 903 } 904 905 @Override 906 public void onError(String errorMessage) { 907 postErrorToListener(errorMessage); 908 } 909 }); 910 } 911 912 913 public void directConnectionDidOpen(RespokeDirectConnection sender) { 914 915 } 916 917 918 public void directConnectionDidClose(RespokeDirectConnection sender) { 919 if (sender == directConnection) { 920 directConnection = null; 921 922 if (null != endpoint) { 923 endpoint.setDirectConnection(null); 924 } 925 } 926 } 927 928 929 // Implementation detail: observe ICE & stream changes and react accordingly. 930 private class PCObserver implements PeerConnection.Observer { 931 @Override public void onIceCandidate(final IceCandidate candidate){ 932 new Handler(Looper.getMainLooper()).post(new Runnable() { 933 public void run() { 934 if (isActive()) { 935 Log.d(TAG, "onIceCandidate"); 936 handleLocalCandidate(candidate); 937 } 938 } 939 }); 940 } 941 942 @Override public void onSignalingChange( 943 PeerConnection.SignalingState newState) { 944 } 945 946 @Override public void onIceConnectionChange( 947 PeerConnection.IceConnectionState newState) { 948 if (isActive()) { 949 if (newState == PeerConnection.IceConnectionState.CONNECTED) { 950 Log.d(TAG, "ICE Connection connected"); 951 } else if (newState == PeerConnection.IceConnectionState.FAILED) { 952 Log.d(TAG, "ICE Connection FAILED"); 953 954 if (null != listenerReference) { 955 // Disconnect will clear the listenerReference, so grab a reference to the 956 // listener while it's still alive since the listener will be notified in a 957 // different (UI) thread 958 final Listener listener = listenerReference.get(); 959 960 if (null != listener) { 961 new Handler(Looper.getMainLooper()).post(new Runnable() { 962 public void run() { 963 if (isActive()) { 964 listener.onError("ICE Connection failed!", RespokeCall.this); 965 listener.onHangup(RespokeCall.this); 966 } 967 } 968 }); 969 } 970 } 971 972 disconnect(); 973 } else { 974 Log.d(TAG, "ICE Connection state: " + newState.toString()); 975 } 976 } 977 } 978 979 @Override public void onIceGatheringChange( 980 PeerConnection.IceGatheringState newState) { 981 if (isActive()) { 982 Log.d(TAG, "ICE Gathering state: " + newState.toString()); 983 if (newState == PeerConnection.IceGatheringState.COMPLETE) { 984 sendFinalCandidates(); 985 } 986 } 987 } 988 989 @Override public void onAddStream(final MediaStream stream){ 990 new Handler(Looper.getMainLooper()).post(new Runnable() { 991 public void run() { 992 if (isActive()) { 993 if (stream.audioTracks.size() <= 1 && stream.videoTracks.size() <= 1) { 994 if (stream.videoTracks.size() == 1) { 995 stream.videoTracks.get(0).addRenderer( 996 new VideoRenderer(remoteRender)); 997 } 998 } else { 999 postErrorToListener("An invalid stream was added"); 1000 } 1001 } 1002 } 1003 }); 1004 } 1005 1006 @Override public void onRemoveStream(final MediaStream stream){ 1007 new Handler(Looper.getMainLooper()).post(new Runnable() { 1008 public void run() { 1009 if (isActive()) { 1010 stream.videoTracks.get(0).dispose(); 1011 } 1012 } 1013 }); 1014 } 1015 1016 @Override public void onDataChannel(final DataChannel dc) { 1017 if (isActive()) { 1018 if (null != directConnection) { 1019 directConnection.peerConnectionDidOpenDataChannel(dc); 1020 } else { 1021 Log.d(TAG, "Direct connection opened, but no object to handle it!"); 1022 } 1023 } 1024 } 1025 1026 @Override public void onRenegotiationNeeded() { 1027 // No need to do anything; AppRTC follows a pre-agreed-upon 1028 // signaling/negotiation protocol. 1029 } 1030 } 1031 1032 private void handleLocalCandidate(IceCandidate candidate) { 1033 try { 1034 // Start critical block 1035 localCandidatesSemaphore.acquire(); 1036 1037 // Collect candidates that are generated in addition to sending them immediately. 1038 // This allows us to send a 'finalCandidates' signal when the iceGatheringState has 1039 // changed to COMPLETED. 'finalCandidates' are used by the backend to smooth inter-op 1040 // between clients that generate trickle ice, and clients that do not support trickle ice. 1041 collectedLocalCandidates.add(candidate); 1042 1043 if (null != queuedLocalCandidates) { 1044 queuedLocalCandidates.add(candidate); 1045 } else { 1046 sendLocalCandidate(candidate); 1047 } 1048 1049 // End critical block 1050 localCandidatesSemaphore.release(); 1051 } catch (InterruptedException e) { 1052 Log.d(TAG, "Error with local candidates semaphore"); 1053 } 1054 } 1055 1056 1057 // Implementation detail: handle offer creation/signaling and answer setting, 1058 // as well as adding remote ICE candidates once the answer SDP is set. 1059 private class SDPObserver implements SdpObserver { 1060 1061 @Override public void onCreateSuccess(final SessionDescription origSdp) { 1062 //abortUnless(localSdp == null, "multiple SDP create?!?"); 1063 final SessionDescription sdp = new SessionDescription( 1064 origSdp.type, preferISAC(origSdp.description)); 1065 1066 new Handler(Looper.getMainLooper()).post(new Runnable() { 1067 public void run() { 1068 if (isActive()) { 1069 Log.d(TAG, "onSuccess(Create SDP)"); 1070 peerConnection.setLocalDescription(sdpObserver, sdp); 1071 1072 try { 1073 JSONObject data = new JSONObject("{'version':'1.0'}"); 1074 data.put("target", directConnectionOnly ? "directConnection" : "call"); 1075 String type = sdp.type.toString().toLowerCase(); 1076 data.put("signalType", type); 1077 data.put("sessionId", sessionID); 1078 data.put("signalId", Respoke.makeGUID()); 1079 1080 JSONObject sdpJSON = new JSONObject(); 1081 sdpJSON.put("sdp", sdp.description); 1082 sdpJSON.put("type", type); 1083 1084 data.put("sessionDescription", sdpJSON); 1085 1086 if (null != signalingChannel) { 1087 signalingChannel.sendSignal(data, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() { 1088 @Override 1089 public void onSuccess() { 1090 drainLocalCandidates(); 1091 } 1092 1093 @Override 1094 public void onError(String errorMessage) { 1095 postErrorToListener(errorMessage); 1096 } 1097 }); 1098 } 1099 } catch (JSONException e) { 1100 postErrorToListener("Error encoding sdp"); 1101 } 1102 } 1103 } 1104 }); 1105 } 1106 1107 1108 @Override public void onSetSuccess() { 1109 new Handler(Looper.getMainLooper()).post(new Runnable() { 1110 public void run() { 1111 if (isActive()) { 1112 Log.d(TAG, "onSuccess(Set SDP)"); 1113 if (caller) { 1114 if (peerConnection.getRemoteDescription() != null) { 1115 // We've set our local offer and received & set the remote 1116 // answer, so drain candidates. 1117 drainRemoteCandidates(); 1118 } 1119 } else { 1120 if (peerConnection.getLocalDescription() == null) { 1121 // We just set the remote offer, time to create our answer. 1122 MediaConstraints sdpMediaConstraints = new MediaConstraints(); 1123 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair( 1124 "OfferToReceiveAudio", "true")); 1125 sdpMediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair( 1126 "OfferToReceiveVideo", audioOnly ? "false" : "true")); 1127 1128 peerConnection.createAnswer(SDPObserver.this, sdpMediaConstraints); 1129 } else { 1130 drainRemoteCandidates(); 1131 } 1132 } 1133 } 1134 } 1135 }); 1136 } 1137 1138 @Override public void onCreateFailure(final String error) { 1139 postErrorToListener("createSDP error: " + error); 1140 } 1141 1142 @Override public void onSetFailure(final String error) { 1143 postErrorToListener("setSDP error: " + error); 1144 } 1145 1146 private void drainRemoteCandidates() { 1147 try { 1148 // Start critical block 1149 queuedRemoteCandidatesSemaphore.acquire(); 1150 1151 for (IceCandidate candidate : queuedRemoteCandidates) { 1152 peerConnection.addIceCandidate(candidate); 1153 } 1154 queuedRemoteCandidates = null; 1155 1156 // End critical block 1157 queuedRemoteCandidatesSemaphore.release(); 1158 } catch (InterruptedException e) { 1159 Log.d(TAG, "Error with remote candidates semaphore"); 1160 } 1161 } 1162 1163 private void drainLocalCandidates() { 1164 try { 1165 // Start critical block 1166 localCandidatesSemaphore.acquire(); 1167 1168 for (IceCandidate candidate : queuedLocalCandidates) { 1169 sendLocalCandidate(candidate); 1170 } 1171 queuedLocalCandidates = null; 1172 1173 // End critical block 1174 localCandidatesSemaphore.release(); 1175 } catch (InterruptedException e) { 1176 Log.d(TAG, "Error with local candidates semaphore"); 1177 } 1178 } 1179 } 1180 1181 private JSONObject getCandidateDict(IceCandidate candidate) { 1182 JSONObject result = new JSONObject(); 1183 1184 try { 1185 result.put("sdpMLineIndex", candidate.sdpMLineIndex); 1186 result.put("sdpMid", candidate.sdpMid); 1187 result.put("candidate", candidate.sdp); 1188 } catch (JSONException e) { 1189 postErrorToListener("Unable to encode local candidate"); 1190 } 1191 1192 return result; 1193 } 1194 1195 private JSONArray getLocalCandidateJSONArray() { 1196 JSONArray result = new JSONArray(); 1197 1198 try { 1199 // Begin critical block 1200 localCandidatesSemaphore.acquire(); 1201 1202 for (IceCandidate candidate: collectedLocalCandidates) { 1203 result.put(getCandidateDict(candidate)); 1204 } 1205 1206 // End critical block 1207 localCandidatesSemaphore.release(); 1208 } catch (InterruptedException e) { 1209 e.printStackTrace(); 1210 } 1211 1212 return result; 1213 } 1214 1215 private void sendFinalCandidates() { 1216 Log.d(TAG, "Sending final candidates"); 1217 JSONObject signalData; 1218 try { 1219 signalData = new JSONObject("{ 'signalType': 'iceCandidates', 'version': '1.0' }"); 1220 signalData.put("target", directConnectionOnly ? "directConnection" : "call"); 1221 signalData.put("sessionId", sessionID); 1222 signalData.put("signalId", Respoke.makeGUID()); 1223 signalData.put("iceCandidates", new JSONArray()); 1224 signalData.put("finalCandidates", getLocalCandidateJSONArray()); 1225 1226 if (null != signalingChannel) { 1227 signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() { 1228 @Override 1229 public void onSuccess() { 1230 // Do nothing 1231 } 1232 1233 @Override 1234 public void onError(String errorMessage) { 1235 postErrorToListener(errorMessage); 1236 } 1237 }); 1238 } 1239 } catch (JSONException e) { 1240 postErrorToListener("Error encoding signal to send final candidates"); 1241 } 1242 } 1243 1244 private void sendLocalCandidate(IceCandidate candidate) { 1245 JSONArray candidateArray = new JSONArray(); 1246 try { 1247 candidateArray.put(getCandidateDict(candidate)); 1248 1249 JSONObject signalData = new JSONObject("{'signalType':'iceCandidates','version':'1.0'}"); 1250 signalData.put("target", directConnectionOnly ? "directConnection" : "call"); 1251 signalData.put("sessionId", sessionID); 1252 signalData.put("signalId", Respoke.makeGUID()); 1253 signalData.put("iceCandidates", candidateArray); 1254 1255 if (null != signalingChannel) { 1256 signalingChannel.sendSignal(signalData, toEndpointId, toConnection, toType, false, new Respoke.TaskCompletionListener() { 1257 @Override 1258 public void onSuccess() { 1259 // Do nothing 1260 } 1261 1262 @Override 1263 public void onError(String errorMessage) { 1264 postErrorToListener(errorMessage); 1265 } 1266 }); 1267 } 1268 } catch (JSONException e) { 1269 postErrorToListener("Error encoding signal to send local candidate"); 1270 } 1271 } 1272 1273 1274 // Mangle SDP to prefer ISAC/16000 over any other audio codec. 1275 private static String preferISAC(String sdpDescription) { 1276 String[] lines = sdpDescription.split("\r\n"); 1277 int mLineIndex = -1; 1278 String isac16kRtpMap = null; 1279 Pattern isac16kPattern = 1280 Pattern.compile("^a=rtpmap:(\\d+) ISAC/16000[\r]?$"); 1281 for (int i = 0; 1282 (i < lines.length) && (mLineIndex == -1 || isac16kRtpMap == null); 1283 ++i) { 1284 if (lines[i].startsWith("m=audio ")) { 1285 mLineIndex = i; 1286 continue; 1287 } 1288 Matcher isac16kMatcher = isac16kPattern.matcher(lines[i]); 1289 if (isac16kMatcher.matches()) { 1290 isac16kRtpMap = isac16kMatcher.group(1); 1291 //continue; 1292 } 1293 } 1294 if (mLineIndex == -1) { 1295 //Log.d(TAG, "No m=audio line, so can't prefer iSAC"); 1296 return sdpDescription; 1297 } 1298 if (isac16kRtpMap == null) { 1299 //Log.d(TAG, "No ISAC/16000 line, so can't prefer iSAC"); 1300 return sdpDescription; 1301 } 1302 String[] origMLineParts = lines[mLineIndex].split(" "); 1303 StringBuilder newMLine = new StringBuilder(); 1304 int origPartIndex = 0; 1305 // Format is: m=<media> <port> <proto> <fmt> ... 1306 newMLine.append(origMLineParts[origPartIndex++]).append(" "); 1307 newMLine.append(origMLineParts[origPartIndex++]).append(" "); 1308 newMLine.append(origMLineParts[origPartIndex++]).append(" "); 1309 newMLine.append(isac16kRtpMap); 1310 for (; origPartIndex < origMLineParts.length; ++origPartIndex) { 1311 if (!origMLineParts[origPartIndex].equals(isac16kRtpMap)) { 1312 newMLine.append(" ").append(origMLineParts[origPartIndex]); 1313 } 1314 } 1315 lines[mLineIndex] = newMLine.toString(); 1316 StringBuilder newSdpDescription = new StringBuilder(); 1317 for (String line : lines) { 1318 newSdpDescription.append(line).append("\r\n"); 1319 } 1320 return newSdpDescription.toString(); 1321 } 1322 1323 1324 private void postErrorToListener(final String errorMessage) { 1325 // All listener methods should be called from the UI thread 1326 new Handler(Looper.getMainLooper()).post(new Runnable() { 1327 public void run() { 1328 if ((isActive()) && (null != listenerReference)) { 1329 Listener listener = listenerReference.get(); 1330 if (null != listener) { 1331 listener.onError(errorMessage, RespokeCall.this); 1332 } 1333 } 1334 } 1335 }); 1336 } 1337 1338 1339}