| /* |
| * Copyright (C) 2019 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| function createDataChannel(pc, label, onMessage) { |
| console.debug('creating data channel: ' + label); |
| let dataChannel = pc.createDataChannel(label); |
| // Return an object with a send function like that of the dataChannel, but |
| // that only actually sends over the data channel once it has connected. |
| return { |
| channelPromise: new Promise((resolve, reject) => { |
| dataChannel.onopen = (event) => { |
| resolve(dataChannel); |
| }; |
| dataChannel.onclose = () => { |
| console.debug( |
| 'Data channel=' + label + ' state=' + dataChannel.readyState); |
| }; |
| dataChannel.onmessage = onMessage ? onMessage : (msg) => { |
| console.debug('Data channel=' + label + ' data="' + msg.data + '"'); |
| }; |
| dataChannel.onerror = err => { |
| reject(err); |
| }; |
| }), |
| send: function(msg) { |
| this.channelPromise = this.channelPromise.then(channel => { |
| channel.send(msg); |
| return channel; |
| }) |
| }, |
| }; |
| } |
| |
| function awaitDataChannel(pc, label, onMessage) { |
| console.debug('expecting data channel: ' + label); |
| // Return an object with a send function like that of the dataChannel, but |
| // that only actually sends over the data channel once it has connected. |
| return { |
| channelPromise: new Promise((resolve, reject) => { |
| let prev_ondatachannel = pc.ondatachannel; |
| pc.ondatachannel = ev => { |
| let dataChannel = ev.channel; |
| if (dataChannel.label == label) { |
| dataChannel.onopen = (event) => { |
| resolve(dataChannel); |
| }; |
| dataChannel.onclose = () => { |
| console.debug( |
| 'Data channel=' + label + ' state=' + dataChannel.readyState); |
| }; |
| dataChannel.onmessage = onMessage ? onMessage : (msg) => { |
| console.debug('Data channel=' + label + ' data="' + msg.data + '"'); |
| }; |
| dataChannel.onerror = err => { |
| reject(err); |
| }; |
| } else if (prev_ondatachannel) { |
| prev_ondatachannel(ev); |
| } |
| }; |
| }), |
| send: function(msg) { |
| this.channelPromise = this.channelPromise.then(channel => { |
| channel.send(msg); |
| return channel; |
| }) |
| }, |
| }; |
| } |
| |
| class DeviceConnection { |
| #pc; |
| #control; |
| #description; |
| |
| #cameraDataChannel; |
| #cameraInputQueue; |
| #controlChannel; |
| #inputChannel; |
| #adbChannel; |
| #bluetoothChannel; |
| |
| #streams; |
| #streamPromiseResolvers; |
| #micSenders = []; |
| #cameraSenders = []; |
| #camera_res_x; |
| #camera_res_y; |
| |
| #onAdbMessage; |
| #onControlMessage; |
| #onBluetoothMessage; |
| |
| constructor(pc, control) { |
| this.#pc = pc; |
| this.#control = control; |
| this.#cameraDataChannel = pc.createDataChannel('camera-data-channel'); |
| this.#cameraDataChannel.binaryType = 'arraybuffer'; |
| this.#cameraInputQueue = new Array(); |
| var self = this; |
| this.#cameraDataChannel.onbufferedamountlow = () => { |
| if (self.#cameraInputQueue.length > 0) { |
| self.sendCameraData(self.#cameraInputQueue.shift()); |
| } |
| }; |
| this.#inputChannel = createDataChannel(pc, 'input-channel'); |
| this.#adbChannel = createDataChannel(pc, 'adb-channel', (msg) => { |
| if (this.#onAdbMessage) { |
| this.#onAdbMessage(msg.data); |
| } else { |
| console.error('Received unexpected ADB message'); |
| } |
| }); |
| this.#controlChannel = awaitDataChannel(pc, 'device-control', (msg) => { |
| if (this.#onControlMessage) { |
| this.#onControlMessage(msg); |
| } else { |
| console.error('Received unexpected Control message'); |
| } |
| }); |
| this.#bluetoothChannel = |
| createDataChannel(pc, 'bluetooth-channel', (msg) => { |
| if (this.#onBluetoothMessage) { |
| this.#onBluetoothMessage(msg.data); |
| } else { |
| console.error('Received unexpected Bluetooth message'); |
| } |
| }); |
| this.#streams = {}; |
| this.#streamPromiseResolvers = {}; |
| |
| pc.addEventListener('track', e => { |
| console.debug('Got remote stream: ', e); |
| for (const stream of e.streams) { |
| this.#streams[stream.id] = stream; |
| if (this.#streamPromiseResolvers[stream.id]) { |
| for (let resolver of this.#streamPromiseResolvers[stream.id]) { |
| resolver(); |
| } |
| delete this.#streamPromiseResolvers[stream.id]; |
| } |
| } |
| }); |
| } |
| |
| set description(desc) { |
| this.#description = desc; |
| } |
| |
| get description() { |
| return this.#description; |
| } |
| |
| get imageCapture() { |
| if (this.#cameraSenders && this.#cameraSenders.length > 0) { |
| let track = this.#cameraSenders[0].track; |
| return new ImageCapture(track); |
| } |
| return undefined; |
| } |
| |
| get cameraWidth() { |
| return this.#camera_res_x; |
| } |
| |
| get cameraHeight() { |
| return this.#camera_res_y; |
| } |
| |
| get cameraEnabled() { |
| return this.#cameraSenders && this.#cameraSenders.length > 0; |
| } |
| |
| getStream(stream_id) { |
| return new Promise((resolve, reject) => { |
| if (this.#streams[stream_id]) { |
| resolve(this.#streams[stream_id]); |
| } else { |
| if (!this.#streamPromiseResolvers[stream_id]) { |
| this.#streamPromiseResolvers[stream_id] = []; |
| } |
| this.#streamPromiseResolvers[stream_id].push(resolve); |
| } |
| }); |
| } |
| |
| #sendJsonInput(evt) { |
| this.#inputChannel.send(JSON.stringify(evt)); |
| } |
| |
| sendMousePosition({x, y, down, display_label}) { |
| this.#sendJsonInput({ |
| type: 'mouse', |
| down: down ? 1 : 0, |
| x, |
| y, |
| display_label, |
| }); |
| } |
| |
| // TODO (b/124121375): This should probably be an array of pointer events and |
| // have different properties. |
| sendMultiTouch({idArr, xArr, yArr, down, slotArr, display_label}) { |
| this.#sendJsonInput({ |
| type: 'multi-touch', |
| id: idArr, |
| x: xArr, |
| y: yArr, |
| down: down ? 1 : 0, |
| slot: slotArr, |
| display_label: display_label, |
| }); |
| } |
| |
| sendKeyEvent(code, type) { |
| this.#sendJsonInput({type: 'keyboard', keycode: code, event_type: type}); |
| } |
| |
| disconnect() { |
| this.#pc.close(); |
| } |
| |
| // Sends binary data directly to the in-device adb daemon (skipping the host) |
| sendAdbMessage(msg) { |
| this.#adbChannel.send(msg); |
| } |
| |
| // Provide a callback to receive data from the in-device adb daemon |
| onAdbMessage(cb) { |
| this.#onAdbMessage = cb; |
| } |
| |
| // Send control commands to the device |
| sendControlMessage(msg) { |
| this.#controlChannel.send(msg); |
| } |
| |
| async #useDevice(in_use, senders_arr, device_opt) { |
| // An empty array means no tracks are currently in use |
| if (senders_arr.length > 0 === !!in_use) { |
| console.warn('Device is already ' + (in_use ? '' : 'not ') + 'in use'); |
| return in_use; |
| } |
| let renegotiation_needed = false; |
| if (in_use) { |
| try { |
| let stream = await navigator.mediaDevices.getUserMedia(device_opt); |
| stream.getTracks().forEach(track => { |
| console.info(`Using ${track.kind} device: ${track.label}`); |
| senders_arr.push(this.#pc.addTrack(track)); |
| renegotiation_needed = true; |
| }); |
| } catch (e) { |
| console.error('Failed to add stream to peer connection: ', e); |
| // Don't return yet, if there were errors some tracks may have been |
| // added so the connection should be renegotiated again. |
| } |
| } else { |
| for (const sender of senders_arr) { |
| console.info( |
| `Removing ${sender.track.kind} device: ${sender.track.label}`); |
| let track = sender.track; |
| track.stop(); |
| this.#pc.removeTrack(sender); |
| renegotiation_needed = true; |
| } |
| // Empty the array passed by reference, just assigning [] won't do that. |
| senders_arr.length = 0; |
| } |
| if (renegotiation_needed) { |
| this.#control.renegotiateConnection(); |
| } |
| // Return the new state |
| return senders_arr.length > 0; |
| } |
| |
| async useMic(in_use) { |
| return this.#useDevice(in_use, this.#micSenders, {audio: true, video: false}); |
| } |
| |
| async useCamera(in_use) { |
| return this.#useDevice(in_use, this.#micSenders, {audio: false, video: true}); |
| } |
| |
| sendCameraResolution(stream) { |
| const cameraTracks = stream.getVideoTracks(); |
| if (cameraTracks.length > 0) { |
| const settings = cameraTracks[0].getSettings(); |
| this.#camera_res_x = settings.width; |
| this.#camera_res_y = settings.height; |
| this.sendControlMessage(JSON.stringify({ |
| command: 'camera_settings', |
| width: settings.width, |
| height: settings.height, |
| frame_rate: settings.frameRate, |
| facing: settings.facingMode |
| })); |
| } |
| } |
| |
| sendOrQueueCameraData(data) { |
| if (this.#cameraDataChannel.bufferedAmount > 0 || |
| this.#cameraInputQueue.length > 0) { |
| this.#cameraInputQueue.push(data); |
| } else { |
| this.sendCameraData(data); |
| } |
| } |
| |
| sendCameraData(data) { |
| const MAX_SIZE = 65535; |
| const END_MARKER = 'EOF'; |
| for (let i = 0; i < data.byteLength; i += MAX_SIZE) { |
| // range is clamped to the valid index range |
| this.#cameraDataChannel.send(data.slice(i, i + MAX_SIZE)); |
| } |
| this.#cameraDataChannel.send(END_MARKER); |
| } |
| |
| // Provide a callback to receive control-related comms from the device |
| onControlMessage(cb) { |
| this.#onControlMessage = cb; |
| } |
| |
| sendBluetoothMessage(msg) { |
| this.#bluetoothChannel.send(msg); |
| } |
| |
| onBluetoothMessage(cb) { |
| this.#onBluetoothMessage = cb; |
| } |
| |
| // Provide a callback to receive connectionstatechange states. |
| onConnectionStateChange(cb) { |
| this.#pc.addEventListener( |
| 'connectionstatechange', evt => cb(this.#pc.connectionState)); |
| } |
| } |
| |
| class Controller { |
| #pc; |
| #serverConnector; |
| |
| constructor(serverConnector) { |
| this.#serverConnector = serverConnector; |
| serverConnector.onDeviceMsg(msg => this.#onDeviceMessage(msg)); |
| } |
| |
| #onDeviceMessage(message) { |
| let type = message.type; |
| switch (type) { |
| case 'offer': |
| this.#onOffer({type: 'offer', sdp: message.sdp}); |
| break; |
| case 'answer': |
| this.#onAnswer({type: 'answer', sdp: message.sdp}); |
| break; |
| case 'ice-candidate': |
| this.#onIceCandidate(new RTCIceCandidate({ |
| sdpMid: message.mid, |
| sdpMLineIndex: message.mLineIndex, |
| candidate: message.candidate |
| })); |
| break; |
| case 'error': |
| console.error('Device responded with error message: ', message.error); |
| break; |
| default: |
| console.error('Unrecognized message type from device: ', type); |
| } |
| } |
| |
| async #sendClientDescription(desc) { |
| console.debug('sendClientDescription'); |
| return this.#serverConnector.sendToDevice({type: 'answer', sdp: desc.sdp}); |
| } |
| |
| async #sendIceCandidate(candidate) { |
| console.debug('sendIceCandidate'); |
| return this.#serverConnector.sendToDevice({type: 'ice-candidate', candidate}); |
| } |
| |
| async #onOffer(desc) { |
| console.debug('Remote description (offer): ', desc); |
| try { |
| await this.#pc.setRemoteDescription(desc); |
| let answer = await this.#pc.createAnswer(); |
| console.debug('Answer: ', answer); |
| await this.#pc.setLocalDescription(answer); |
| await this.#sendClientDescription(answer); |
| } catch (e) { |
| console.error('Error processing remote description (offer)', e) |
| throw e; |
| } |
| } |
| |
| async #onAnswer(answer) { |
| console.debug('Remote description (answer): ', answer); |
| try { |
| await this.#pc.setRemoteDescription(answer); |
| } catch (e) { |
| console.error('Error processing remote description (answer)', e) |
| throw e; |
| } |
| } |
| |
| #onIceCandidate(iceCandidate) { |
| console.debug(`Remote ICE Candidate: `, iceCandidate); |
| this.#pc.addIceCandidate(iceCandidate); |
| } |
| |
| ConnectDevice(pc) { |
| this.#pc = pc; |
| console.debug('ConnectDevice'); |
| // ICE candidates will be generated when we add the offer. Adding it here |
| // instead of in _onOffer because this function is called once per peer |
| // connection, while _onOffer may be called more than once due to |
| // renegotiations. |
| this.#pc.addEventListener('icecandidate', evt => { |
| if (evt.candidate) this.#sendIceCandidate(evt.candidate); |
| }); |
| this.#serverConnector.sendToDevice({type: 'request-offer'}); |
| } |
| |
| async renegotiateConnection() { |
| console.debug('Re-negotiating connection'); |
| let offer = await this.#pc.createOffer(); |
| console.debug('Local description (offer): ', offer); |
| await this.#pc.setLocalDescription(offer); |
| this.#serverConnector.sendToDevice({type: 'offer', sdp: offer.sdp}); |
| } |
| } |
| |
| function createPeerConnection(infra_config) { |
| let pc_config = {iceServers: []}; |
| for (const stun of infra_config.ice_servers) { |
| pc_config.iceServers.push({urls: 'stun:' + stun}); |
| } |
| let pc = new RTCPeerConnection(pc_config); |
| |
| pc.addEventListener('icecandidate', evt => { |
| console.debug('Local ICE Candidate: ', evt.candidate); |
| }); |
| pc.addEventListener('iceconnectionstatechange', evt => { |
| console.debug(`ICE State Change: ${pc.iceConnectionState}`); |
| }); |
| pc.addEventListener( |
| 'connectionstatechange', |
| evt => console.debug( |
| `WebRTC Connection State Change: ${pc.connectionState}`)); |
| return pc; |
| } |
| |
| export async function Connect(deviceId, serverConnector) { |
| let requestRet = await serverConnector.requestDevice(deviceId); |
| let deviceInfo = requestRet.deviceInfo; |
| let infraConfig = requestRet.infraConfig; |
| console.debug('Device available:'); |
| console.debug(deviceInfo); |
| let pc_config = {iceServers: []}; |
| if (infraConfig.ice_servers && infraConfig.ice_servers.length > 0) { |
| for (const server of infraConfig.ice_servers) { |
| pc_config.iceServers.push(server); |
| } |
| } |
| let pc = createPeerConnection(infraConfig); |
| |
| let control = new Controller(serverConnector); |
| let deviceConnection = new DeviceConnection(pc, control); |
| deviceConnection.description = deviceInfo; |
| |
| return new Promise((resolve, reject) => { |
| pc.addEventListener('connectionstatechange', evt => { |
| let state = pc.connectionState; |
| if (state == 'connected') { |
| resolve(deviceConnection); |
| } else if (state == 'failed') { |
| reject(evt); |
| } |
| }); |
| control.ConnectDevice(pc); |
| }); |
| } |