1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "examples/unityplugin/simple_peer_connection.h"
12
13 #include <utility>
14
15 #include "absl/memory/memory.h"
16 #include "api/audio_codecs/builtin_audio_decoder_factory.h"
17 #include "api/audio_codecs/builtin_audio_encoder_factory.h"
18 #include "api/create_peerconnection_factory.h"
19 #include "media/engine/internal_decoder_factory.h"
20 #include "media/engine/internal_encoder_factory.h"
21 #include "media/engine/multiplex_codec_factory.h"
22 #include "modules/audio_device/include/audio_device.h"
23 #include "modules/audio_processing/include/audio_processing.h"
24 #include "modules/video_capture/video_capture_factory.h"
25 #include "pc/video_track_source.h"
26 #include "test/vcm_capturer.h"
27
28 #if defined(WEBRTC_ANDROID)
29 #include "examples/unityplugin/class_reference_holder.h"
30 #include "modules/utility/include/helpers_android.h"
31 #include "sdk/android/src/jni/android_video_track_source.h"
32 #include "sdk/android/src/jni/jni_helpers.h"
33 #endif
34
35 // Names used for media stream ids.
36 const char kAudioLabel[] = "audio_label";
37 const char kVideoLabel[] = "video_label";
38 const char kStreamId[] = "stream_id";
39
40 namespace {
41 static int g_peer_count = 0;
42 static std::unique_ptr<rtc::Thread> g_worker_thread;
43 static std::unique_ptr<rtc::Thread> g_signaling_thread;
44 static rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
45 g_peer_connection_factory;
46 #if defined(WEBRTC_ANDROID)
47 // Android case: the video track does not own the capturer, and it
48 // relies on the app to dispose the capturer when the peerconnection
49 // shuts down.
50 static jobject g_camera = nullptr;
51 #else
52 class CapturerTrackSource : public webrtc::VideoTrackSource {
53 public:
Create()54 static rtc::scoped_refptr<CapturerTrackSource> Create() {
55 const size_t kWidth = 640;
56 const size_t kHeight = 480;
57 const size_t kFps = 30;
58 const size_t kDeviceIndex = 0;
59 std::unique_ptr<webrtc::test::VcmCapturer> capturer = absl::WrapUnique(
60 webrtc::test::VcmCapturer::Create(kWidth, kHeight, kFps, kDeviceIndex));
61 if (!capturer) {
62 return nullptr;
63 }
64 return rtc::make_ref_counted<CapturerTrackSource>(std::move(capturer));
65 }
66
67 protected:
CapturerTrackSource(std::unique_ptr<webrtc::test::VcmCapturer> capturer)68 explicit CapturerTrackSource(
69 std::unique_ptr<webrtc::test::VcmCapturer> capturer)
70 : VideoTrackSource(/*remote=*/false), capturer_(std::move(capturer)) {}
71
72 private:
source()73 rtc::VideoSourceInterface<webrtc::VideoFrame>* source() override {
74 return capturer_.get();
75 }
76 std::unique_ptr<webrtc::test::VcmCapturer> capturer_;
77 };
78
79 #endif
80
GetEnvVarOrDefault(const char * env_var_name,const char * default_value)81 std::string GetEnvVarOrDefault(const char* env_var_name,
82 const char* default_value) {
83 std::string value;
84 const char* env_var = getenv(env_var_name);
85 if (env_var)
86 value = env_var;
87
88 if (value.empty())
89 value = default_value;
90
91 return value;
92 }
93
GetPeerConnectionString()94 std::string GetPeerConnectionString() {
95 return GetEnvVarOrDefault("WEBRTC_CONNECT", "stun:stun.l.google.com:19302");
96 }
97
98 class DummySetSessionDescriptionObserver
99 : public webrtc::SetSessionDescriptionObserver {
100 public:
Create()101 static rtc::scoped_refptr<DummySetSessionDescriptionObserver> Create() {
102 return rtc::make_ref_counted<DummySetSessionDescriptionObserver>();
103 }
OnSuccess()104 virtual void OnSuccess() { RTC_LOG(LS_INFO) << __FUNCTION__; }
OnFailure(webrtc::RTCError error)105 virtual void OnFailure(webrtc::RTCError error) {
106 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": "
107 << error.message();
108 }
109
110 protected:
DummySetSessionDescriptionObserver()111 DummySetSessionDescriptionObserver() {}
~DummySetSessionDescriptionObserver()112 ~DummySetSessionDescriptionObserver() {}
113 };
114
115 } // namespace
116
InitializePeerConnection(const char ** turn_urls,const int no_of_urls,const char * username,const char * credential,bool is_receiver)117 bool SimplePeerConnection::InitializePeerConnection(const char** turn_urls,
118 const int no_of_urls,
119 const char* username,
120 const char* credential,
121 bool is_receiver) {
122 RTC_DCHECK(peer_connection_.get() == nullptr);
123
124 if (g_peer_connection_factory == nullptr) {
125 g_worker_thread = rtc::Thread::Create();
126 g_worker_thread->Start();
127 g_signaling_thread = rtc::Thread::Create();
128 g_signaling_thread->Start();
129
130 g_peer_connection_factory = webrtc::CreatePeerConnectionFactory(
131 g_worker_thread.get(), g_worker_thread.get(), g_signaling_thread.get(),
132 nullptr, webrtc::CreateBuiltinAudioEncoderFactory(),
133 webrtc::CreateBuiltinAudioDecoderFactory(),
134 std::unique_ptr<webrtc::VideoEncoderFactory>(
135 new webrtc::MultiplexEncoderFactory(
136 std::make_unique<webrtc::InternalEncoderFactory>())),
137 std::unique_ptr<webrtc::VideoDecoderFactory>(
138 new webrtc::MultiplexDecoderFactory(
139 std::make_unique<webrtc::InternalDecoderFactory>())),
140 nullptr, nullptr);
141 }
142 if (!g_peer_connection_factory.get()) {
143 DeletePeerConnection();
144 return false;
145 }
146
147 g_peer_count++;
148 if (!CreatePeerConnection(turn_urls, no_of_urls, username, credential)) {
149 DeletePeerConnection();
150 return false;
151 }
152
153 mandatory_receive_ = is_receiver;
154 return peer_connection_.get() != nullptr;
155 }
156
CreatePeerConnection(const char ** turn_urls,const int no_of_urls,const char * username,const char * credential)157 bool SimplePeerConnection::CreatePeerConnection(const char** turn_urls,
158 const int no_of_urls,
159 const char* username,
160 const char* credential) {
161 RTC_DCHECK(g_peer_connection_factory.get() != nullptr);
162 RTC_DCHECK(peer_connection_.get() == nullptr);
163
164 local_video_observer_.reset(new VideoObserver());
165 remote_video_observer_.reset(new VideoObserver());
166
167 // Add the turn server.
168 if (turn_urls != nullptr) {
169 if (no_of_urls > 0) {
170 webrtc::PeerConnectionInterface::IceServer turn_server;
171 for (int i = 0; i < no_of_urls; i++) {
172 std::string url(turn_urls[i]);
173 if (url.length() > 0)
174 turn_server.urls.push_back(turn_urls[i]);
175 }
176
177 std::string user_name(username);
178 if (user_name.length() > 0)
179 turn_server.username = username;
180
181 std::string password(credential);
182 if (password.length() > 0)
183 turn_server.password = credential;
184
185 config_.servers.push_back(turn_server);
186 }
187 }
188
189 // Add the stun server.
190 webrtc::PeerConnectionInterface::IceServer stun_server;
191 stun_server.uri = GetPeerConnectionString();
192 config_.servers.push_back(stun_server);
193
194 auto result = g_peer_connection_factory->CreatePeerConnectionOrError(
195 config_, webrtc::PeerConnectionDependencies(this));
196 if (!result.ok()) {
197 peer_connection_ = nullptr;
198 return false;
199 }
200 peer_connection_ = result.MoveValue();
201 return true;
202 }
203
DeletePeerConnection()204 void SimplePeerConnection::DeletePeerConnection() {
205 g_peer_count--;
206
207 #if defined(WEBRTC_ANDROID)
208 if (g_camera) {
209 JNIEnv* env = webrtc::jni::GetEnv();
210 jclass pc_factory_class =
211 unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
212 jmethodID stop_camera_method = webrtc::GetStaticMethodID(
213 env, pc_factory_class, "StopCamera", "(Lorg/webrtc/VideoCapturer;)V");
214
215 env->CallStaticVoidMethod(pc_factory_class, stop_camera_method, g_camera);
216 CHECK_EXCEPTION(env);
217
218 g_camera = nullptr;
219 }
220 #endif
221
222 CloseDataChannel();
223 peer_connection_ = nullptr;
224 active_streams_.clear();
225
226 if (g_peer_count == 0) {
227 g_peer_connection_factory = nullptr;
228 g_signaling_thread.reset();
229 g_worker_thread.reset();
230 }
231 }
232
CreateOffer()233 bool SimplePeerConnection::CreateOffer() {
234 if (!peer_connection_.get())
235 return false;
236
237 webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options;
238 if (mandatory_receive_) {
239 options.offer_to_receive_audio = true;
240 options.offer_to_receive_video = true;
241 }
242 peer_connection_->CreateOffer(this, options);
243 return true;
244 }
245
CreateAnswer()246 bool SimplePeerConnection::CreateAnswer() {
247 if (!peer_connection_.get())
248 return false;
249
250 webrtc::PeerConnectionInterface::RTCOfferAnswerOptions options;
251 if (mandatory_receive_) {
252 options.offer_to_receive_audio = true;
253 options.offer_to_receive_video = true;
254 }
255 peer_connection_->CreateAnswer(this, options);
256 return true;
257 }
258
OnSuccess(webrtc::SessionDescriptionInterface * desc)259 void SimplePeerConnection::OnSuccess(
260 webrtc::SessionDescriptionInterface* desc) {
261 peer_connection_->SetLocalDescription(
262 DummySetSessionDescriptionObserver::Create().get(), desc);
263
264 std::string sdp;
265 desc->ToString(&sdp);
266
267 if (OnLocalSdpReady)
268 OnLocalSdpReady(desc->type().c_str(), sdp.c_str());
269 }
270
OnFailure(webrtc::RTCError error)271 void SimplePeerConnection::OnFailure(webrtc::RTCError error) {
272 RTC_LOG(LS_ERROR) << ToString(error.type()) << ": " << error.message();
273
274 // TODO(hta): include error.type in the message
275 if (OnFailureMessage)
276 OnFailureMessage(error.message());
277 }
278
OnIceCandidate(const webrtc::IceCandidateInterface * candidate)279 void SimplePeerConnection::OnIceCandidate(
280 const webrtc::IceCandidateInterface* candidate) {
281 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
282
283 std::string sdp;
284 if (!candidate->ToString(&sdp)) {
285 RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
286 return;
287 }
288
289 if (OnIceCandidateReady)
290 OnIceCandidateReady(sdp.c_str(), candidate->sdp_mline_index(),
291 candidate->sdp_mid().c_str());
292 }
293
RegisterOnLocalI420FrameReady(I420FRAMEREADY_CALLBACK callback)294 void SimplePeerConnection::RegisterOnLocalI420FrameReady(
295 I420FRAMEREADY_CALLBACK callback) {
296 if (local_video_observer_)
297 local_video_observer_->SetVideoCallback(callback);
298 }
299
RegisterOnRemoteI420FrameReady(I420FRAMEREADY_CALLBACK callback)300 void SimplePeerConnection::RegisterOnRemoteI420FrameReady(
301 I420FRAMEREADY_CALLBACK callback) {
302 if (remote_video_observer_)
303 remote_video_observer_->SetVideoCallback(callback);
304 }
305
RegisterOnLocalDataChannelReady(LOCALDATACHANNELREADY_CALLBACK callback)306 void SimplePeerConnection::RegisterOnLocalDataChannelReady(
307 LOCALDATACHANNELREADY_CALLBACK callback) {
308 OnLocalDataChannelReady = callback;
309 }
310
RegisterOnDataFromDataChannelReady(DATAFROMEDATECHANNELREADY_CALLBACK callback)311 void SimplePeerConnection::RegisterOnDataFromDataChannelReady(
312 DATAFROMEDATECHANNELREADY_CALLBACK callback) {
313 OnDataFromDataChannelReady = callback;
314 }
315
RegisterOnFailure(FAILURE_CALLBACK callback)316 void SimplePeerConnection::RegisterOnFailure(FAILURE_CALLBACK callback) {
317 OnFailureMessage = callback;
318 }
319
RegisterOnAudioBusReady(AUDIOBUSREADY_CALLBACK callback)320 void SimplePeerConnection::RegisterOnAudioBusReady(
321 AUDIOBUSREADY_CALLBACK callback) {
322 OnAudioReady = callback;
323 }
324
RegisterOnLocalSdpReadytoSend(LOCALSDPREADYTOSEND_CALLBACK callback)325 void SimplePeerConnection::RegisterOnLocalSdpReadytoSend(
326 LOCALSDPREADYTOSEND_CALLBACK callback) {
327 OnLocalSdpReady = callback;
328 }
329
RegisterOnIceCandidateReadytoSend(ICECANDIDATEREADYTOSEND_CALLBACK callback)330 void SimplePeerConnection::RegisterOnIceCandidateReadytoSend(
331 ICECANDIDATEREADYTOSEND_CALLBACK callback) {
332 OnIceCandidateReady = callback;
333 }
334
SetRemoteDescription(const char * type,const char * sdp)335 bool SimplePeerConnection::SetRemoteDescription(const char* type,
336 const char* sdp) {
337 if (!peer_connection_)
338 return false;
339
340 std::string remote_desc(sdp);
341 std::string desc_type(type);
342 webrtc::SdpParseError error;
343 webrtc::SessionDescriptionInterface* session_description(
344 webrtc::CreateSessionDescription(desc_type, remote_desc, &error));
345 if (!session_description) {
346 RTC_LOG(LS_WARNING) << "Can't parse received session description message. "
347 "SdpParseError was: "
348 << error.description;
349 return false;
350 }
351 RTC_LOG(LS_INFO) << " Received session description :" << remote_desc;
352 peer_connection_->SetRemoteDescription(
353 DummySetSessionDescriptionObserver::Create().get(), session_description);
354
355 return true;
356 }
357
AddIceCandidate(const char * candidate,const int sdp_mlineindex,const char * sdp_mid)358 bool SimplePeerConnection::AddIceCandidate(const char* candidate,
359 const int sdp_mlineindex,
360 const char* sdp_mid) {
361 if (!peer_connection_)
362 return false;
363
364 webrtc::SdpParseError error;
365 std::unique_ptr<webrtc::IceCandidateInterface> ice_candidate(
366 webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, candidate, &error));
367 if (!ice_candidate.get()) {
368 RTC_LOG(LS_WARNING) << "Can't parse received candidate message. "
369 "SdpParseError was: "
370 << error.description;
371 return false;
372 }
373 if (!peer_connection_->AddIceCandidate(ice_candidate.get())) {
374 RTC_LOG(LS_WARNING) << "Failed to apply the received candidate";
375 return false;
376 }
377 RTC_LOG(LS_INFO) << " Received candidate :" << candidate;
378 return true;
379 }
380
SetAudioControl(bool is_mute,bool is_record)381 void SimplePeerConnection::SetAudioControl(bool is_mute, bool is_record) {
382 is_mute_audio_ = is_mute;
383 is_record_audio_ = is_record;
384
385 SetAudioControl();
386 }
387
SetAudioControl()388 void SimplePeerConnection::SetAudioControl() {
389 if (!remote_stream_)
390 return;
391 webrtc::AudioTrackVector tracks = remote_stream_->GetAudioTracks();
392 if (tracks.empty())
393 return;
394
395 rtc::scoped_refptr<webrtc::AudioTrackInterface>& audio_track = tracks[0];
396 if (is_record_audio_)
397 audio_track->AddSink(this);
398 else
399 audio_track->RemoveSink(this);
400
401 for (auto& track : tracks) {
402 if (is_mute_audio_)
403 track->set_enabled(false);
404 else
405 track->set_enabled(true);
406 }
407 }
408
OnAddStream(rtc::scoped_refptr<webrtc::MediaStreamInterface> stream)409 void SimplePeerConnection::OnAddStream(
410 rtc::scoped_refptr<webrtc::MediaStreamInterface> stream) {
411 RTC_LOG(LS_INFO) << __FUNCTION__ << " " << stream->id();
412 remote_stream_ = stream;
413 if (remote_video_observer_ && !remote_stream_->GetVideoTracks().empty()) {
414 remote_stream_->GetVideoTracks()[0]->AddOrUpdateSink(
415 remote_video_observer_.get(), rtc::VideoSinkWants());
416 }
417 SetAudioControl();
418 }
419
AddStreams(bool audio_only)420 void SimplePeerConnection::AddStreams(bool audio_only) {
421 if (active_streams_.find(kStreamId) != active_streams_.end())
422 return; // Already added.
423
424 rtc::scoped_refptr<webrtc::MediaStreamInterface> stream =
425 g_peer_connection_factory->CreateLocalMediaStream(kStreamId);
426
427 rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(
428 g_peer_connection_factory->CreateAudioTrack(
429 kAudioLabel,
430 g_peer_connection_factory->CreateAudioSource(cricket::AudioOptions())
431 .get()));
432 stream->AddTrack(audio_track);
433
434 if (!audio_only) {
435 #if defined(WEBRTC_ANDROID)
436 JNIEnv* env = webrtc::jni::GetEnv();
437 jclass pc_factory_class =
438 unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
439 jmethodID load_texture_helper_method = webrtc::GetStaticMethodID(
440 env, pc_factory_class, "LoadSurfaceTextureHelper",
441 "()Lorg/webrtc/SurfaceTextureHelper;");
442 jobject texture_helper = env->CallStaticObjectMethod(
443 pc_factory_class, load_texture_helper_method);
444 CHECK_EXCEPTION(env);
445 RTC_DCHECK(texture_helper != nullptr)
446 << "Cannot get the Surface Texture Helper.";
447
448 auto source = rtc::make_ref_counted<webrtc::jni::AndroidVideoTrackSource>(
449 g_signaling_thread.get(), env, /*is_screencast=*/false,
450 /*align_timestamps=*/true);
451
452 // link with VideoCapturer (Camera);
453 jmethodID link_camera_method = webrtc::GetStaticMethodID(
454 env, pc_factory_class, "LinkCamera",
455 "(JLorg/webrtc/SurfaceTextureHelper;)Lorg/webrtc/VideoCapturer;");
456 jobject camera_tmp =
457 env->CallStaticObjectMethod(pc_factory_class, link_camera_method,
458 (jlong)source.get(), texture_helper);
459 CHECK_EXCEPTION(env);
460 g_camera = (jobject)env->NewGlobalRef(camera_tmp);
461
462 rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
463 g_peer_connection_factory->CreateVideoTrack(kVideoLabel,
464 source.release()));
465 stream->AddTrack(video_track);
466 #else
467 rtc::scoped_refptr<CapturerTrackSource> video_device =
468 CapturerTrackSource::Create();
469 if (video_device) {
470 rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
471 g_peer_connection_factory->CreateVideoTrack(kVideoLabel,
472 video_device.get()));
473
474 stream->AddTrack(video_track);
475 }
476 #endif
477 if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
478 stream->GetVideoTracks()[0]->AddOrUpdateSink(local_video_observer_.get(),
479 rtc::VideoSinkWants());
480 }
481 }
482
483 if (!peer_connection_->AddStream(stream.get())) {
484 RTC_LOG(LS_ERROR) << "Adding stream to PeerConnection failed";
485 }
486
487 typedef std::pair<std::string,
488 rtc::scoped_refptr<webrtc::MediaStreamInterface>>
489 MediaStreamPair;
490 active_streams_.insert(MediaStreamPair(stream->id(), stream));
491 }
492
CreateDataChannel()493 bool SimplePeerConnection::CreateDataChannel() {
494 struct webrtc::DataChannelInit init;
495 init.ordered = true;
496 init.reliable = true;
497 auto result = peer_connection_->CreateDataChannelOrError("Hello", &init);
498 if (result.ok()) {
499 data_channel_ = result.MoveValue();
500 data_channel_->RegisterObserver(this);
501 RTC_LOG(LS_INFO) << "Succeeds to create data channel";
502 return true;
503 } else {
504 RTC_LOG(LS_INFO) << "Fails to create data channel";
505 return false;
506 }
507 }
508
CloseDataChannel()509 void SimplePeerConnection::CloseDataChannel() {
510 if (data_channel_.get()) {
511 data_channel_->UnregisterObserver();
512 data_channel_->Close();
513 }
514 data_channel_ = nullptr;
515 }
516
SendDataViaDataChannel(const std::string & data)517 bool SimplePeerConnection::SendDataViaDataChannel(const std::string& data) {
518 if (!data_channel_.get()) {
519 RTC_LOG(LS_INFO) << "Data channel is not established";
520 return false;
521 }
522 webrtc::DataBuffer buffer(data);
523 data_channel_->Send(buffer);
524 return true;
525 }
526
527 // Peerconnection observer
OnDataChannel(rtc::scoped_refptr<webrtc::DataChannelInterface> channel)528 void SimplePeerConnection::OnDataChannel(
529 rtc::scoped_refptr<webrtc::DataChannelInterface> channel) {
530 channel->RegisterObserver(this);
531 }
532
OnStateChange()533 void SimplePeerConnection::OnStateChange() {
534 if (data_channel_) {
535 webrtc::DataChannelInterface::DataState state = data_channel_->state();
536 if (state == webrtc::DataChannelInterface::kOpen) {
537 if (OnLocalDataChannelReady)
538 OnLocalDataChannelReady();
539 RTC_LOG(LS_INFO) << "Data channel is open";
540 }
541 }
542 }
543
544 // A data buffer was successfully received.
OnMessage(const webrtc::DataBuffer & buffer)545 void SimplePeerConnection::OnMessage(const webrtc::DataBuffer& buffer) {
546 size_t size = buffer.data.size();
547 char* msg = new char[size + 1];
548 memcpy(msg, buffer.data.data(), size);
549 msg[size] = 0;
550 if (OnDataFromDataChannelReady)
551 OnDataFromDataChannelReady(msg);
552 delete[] msg;
553 }
554
555 // AudioTrackSinkInterface implementation.
OnData(const void * audio_data,int bits_per_sample,int sample_rate,size_t number_of_channels,size_t number_of_frames)556 void SimplePeerConnection::OnData(const void* audio_data,
557 int bits_per_sample,
558 int sample_rate,
559 size_t number_of_channels,
560 size_t number_of_frames) {
561 if (OnAudioReady)
562 OnAudioReady(audio_data, bits_per_sample, sample_rate,
563 static_cast<int>(number_of_channels),
564 static_cast<int>(number_of_frames));
565 }
566
GetRemoteAudioTrackSsrcs()567 std::vector<uint32_t> SimplePeerConnection::GetRemoteAudioTrackSsrcs() {
568 std::vector<rtc::scoped_refptr<webrtc::RtpReceiverInterface>> receivers =
569 peer_connection_->GetReceivers();
570
571 std::vector<uint32_t> ssrcs;
572 for (const auto& receiver : receivers) {
573 if (receiver->media_type() != cricket::MEDIA_TYPE_AUDIO)
574 continue;
575
576 std::vector<webrtc::RtpEncodingParameters> params =
577 receiver->GetParameters().encodings;
578
579 for (const auto& param : params) {
580 uint32_t ssrc = param.ssrc.value_or(0);
581 if (ssrc > 0)
582 ssrcs.push_back(ssrc);
583 }
584 }
585
586 return ssrcs;
587 }
588