blob: 5c913834024918713a7f93aec46dc447621e4321 [file] [log] [blame]
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
function createDataChannel(pc, label, onMessage) {
console.debug('creating data channel: ' + label);
let dataChannel = pc.createDataChannel(label);
// Return an object with a send function like that of the dataChannel, but
// that only actually sends over the data channel once it has connected.
return {
channelPromise: new Promise((resolve, reject) => {
dataChannel.onopen = (event) => {
resolve(dataChannel);
};
dataChannel.onclose = () => {
console.debug(
'Data channel=' + label + ' state=' + dataChannel.readyState);
};
dataChannel.onmessage = onMessage ? onMessage : (msg) => {
console.debug('Data channel=' + label + ' data="' + msg.data + '"');
};
dataChannel.onerror = err => {
reject(err);
};
}),
send: function(msg) {
this.channelPromise = this.channelPromise.then(channel => {
channel.send(msg);
return channel;
})
},
};
}
function awaitDataChannel(pc, label, onMessage) {
console.debug('expecting data channel: ' + label);
// Return an object with a send function like that of the dataChannel, but
// that only actually sends over the data channel once it has connected.
return {
channelPromise: new Promise((resolve, reject) => {
let prev_ondatachannel = pc.ondatachannel;
pc.ondatachannel = ev => {
let dataChannel = ev.channel;
if (dataChannel.label == label) {
dataChannel.onopen = (event) => {
resolve(dataChannel);
};
dataChannel.onclose = () => {
console.debug(
'Data channel=' + label + ' state=' + dataChannel.readyState);
};
dataChannel.onmessage = onMessage ? onMessage : (msg) => {
console.debug('Data channel=' + label + ' data="' + msg.data + '"');
};
dataChannel.onerror = err => {
reject(err);
};
} else if (prev_ondatachannel) {
prev_ondatachannel(ev);
}
};
}),
send: function(msg) {
this.channelPromise = this.channelPromise.then(channel => {
channel.send(msg);
return channel;
})
},
};
}
class DeviceConnection {
#pc;
#control;
#description;
#cameraDataChannel;
#cameraInputQueue;
#controlChannel;
#inputChannel;
#adbChannel;
#bluetoothChannel;
#streams;
#streamPromiseResolvers;
#micSenders = [];
#cameraSenders = [];
#camera_res_x;
#camera_res_y;
#onAdbMessage;
#onControlMessage;
#onBluetoothMessage;
constructor(pc, control) {
this.#pc = pc;
this.#control = control;
this.#cameraDataChannel = pc.createDataChannel('camera-data-channel');
this.#cameraDataChannel.binaryType = 'arraybuffer';
this.#cameraInputQueue = new Array();
var self = this;
this.#cameraDataChannel.onbufferedamountlow = () => {
if (self.#cameraInputQueue.length > 0) {
self.sendCameraData(self.#cameraInputQueue.shift());
}
};
this.#inputChannel = createDataChannel(pc, 'input-channel');
this.#adbChannel = createDataChannel(pc, 'adb-channel', (msg) => {
if (this.#onAdbMessage) {
this.#onAdbMessage(msg.data);
} else {
console.error('Received unexpected ADB message');
}
});
this.#controlChannel = awaitDataChannel(pc, 'device-control', (msg) => {
if (this.#onControlMessage) {
this.#onControlMessage(msg);
} else {
console.error('Received unexpected Control message');
}
});
this.#bluetoothChannel =
createDataChannel(pc, 'bluetooth-channel', (msg) => {
if (this.#onBluetoothMessage) {
this.#onBluetoothMessage(msg.data);
} else {
console.error('Received unexpected Bluetooth message');
}
});
this.#streams = {};
this.#streamPromiseResolvers = {};
pc.addEventListener('track', e => {
console.debug('Got remote stream: ', e);
for (const stream of e.streams) {
this.#streams[stream.id] = stream;
if (this.#streamPromiseResolvers[stream.id]) {
for (let resolver of this.#streamPromiseResolvers[stream.id]) {
resolver();
}
delete this.#streamPromiseResolvers[stream.id];
}
}
});
}
set description(desc) {
this.#description = desc;
}
get description() {
return this.#description;
}
get imageCapture() {
if (this.#cameraSenders && this.#cameraSenders.length > 0) {
let track = this.#cameraSenders[0].track;
return new ImageCapture(track);
}
return undefined;
}
get cameraWidth() {
return this.#camera_res_x;
}
get cameraHeight() {
return this.#camera_res_y;
}
get cameraEnabled() {
return this.#cameraSenders && this.#cameraSenders.length > 0;
}
getStream(stream_id) {
return new Promise((resolve, reject) => {
if (this.#streams[stream_id]) {
resolve(this.#streams[stream_id]);
} else {
if (!this.#streamPromiseResolvers[stream_id]) {
this.#streamPromiseResolvers[stream_id] = [];
}
this.#streamPromiseResolvers[stream_id].push(resolve);
}
});
}
#sendJsonInput(evt) {
this.#inputChannel.send(JSON.stringify(evt));
}
sendMousePosition({x, y, down, display_label}) {
this.#sendJsonInput({
type: 'mouse',
down: down ? 1 : 0,
x,
y,
display_label,
});
}
// TODO (b/124121375): This should probably be an array of pointer events and
// have different properties.
sendMultiTouch({idArr, xArr, yArr, down, slotArr, display_label}) {
this.#sendJsonInput({
type: 'multi-touch',
id: idArr,
x: xArr,
y: yArr,
down: down ? 1 : 0,
slot: slotArr,
display_label: display_label,
});
}
sendKeyEvent(code, type) {
this.#sendJsonInput({type: 'keyboard', keycode: code, event_type: type});
}
disconnect() {
this.#pc.close();
}
// Sends binary data directly to the in-device adb daemon (skipping the host)
sendAdbMessage(msg) {
this.#adbChannel.send(msg);
}
// Provide a callback to receive data from the in-device adb daemon
onAdbMessage(cb) {
this.#onAdbMessage = cb;
}
// Send control commands to the device
sendControlMessage(msg) {
this.#controlChannel.send(msg);
}
async #useDevice(in_use, senders_arr, device_opt) {
// An empty array means no tracks are currently in use
if (senders_arr.length > 0 === !!in_use) {
console.warn('Device is already ' + (in_use ? '' : 'not ') + 'in use');
return in_use;
}
let renegotiation_needed = false;
if (in_use) {
try {
let stream = await navigator.mediaDevices.getUserMedia(device_opt);
stream.getTracks().forEach(track => {
console.info(`Using ${track.kind} device: ${track.label}`);
senders_arr.push(this.#pc.addTrack(track));
renegotiation_needed = true;
});
} catch (e) {
console.error('Failed to add stream to peer connection: ', e);
// Don't return yet, if there were errors some tracks may have been
// added so the connection should be renegotiated again.
}
} else {
for (const sender of senders_arr) {
console.info(
`Removing ${sender.track.kind} device: ${sender.track.label}`);
let track = sender.track;
track.stop();
this.#pc.removeTrack(sender);
renegotiation_needed = true;
}
// Empty the array passed by reference, just assigning [] won't do that.
senders_arr.length = 0;
}
if (renegotiation_needed) {
this.#control.renegotiateConnection();
}
// Return the new state
return senders_arr.length > 0;
}
async useMic(in_use) {
return this.#useDevice(in_use, this.#micSenders, {audio: true, video: false});
}
async useCamera(in_use) {
return this.#useDevice(in_use, this.#micSenders, {audio: false, video: true});
}
sendCameraResolution(stream) {
const cameraTracks = stream.getVideoTracks();
if (cameraTracks.length > 0) {
const settings = cameraTracks[0].getSettings();
this.#camera_res_x = settings.width;
this.#camera_res_y = settings.height;
this.sendControlMessage(JSON.stringify({
command: 'camera_settings',
width: settings.width,
height: settings.height,
frame_rate: settings.frameRate,
facing: settings.facingMode
}));
}
}
sendOrQueueCameraData(data) {
if (this.#cameraDataChannel.bufferedAmount > 0 ||
this.#cameraInputQueue.length > 0) {
this.#cameraInputQueue.push(data);
} else {
this.sendCameraData(data);
}
}
sendCameraData(data) {
const MAX_SIZE = 65535;
const END_MARKER = 'EOF';
for (let i = 0; i < data.byteLength; i += MAX_SIZE) {
// range is clamped to the valid index range
this.#cameraDataChannel.send(data.slice(i, i + MAX_SIZE));
}
this.#cameraDataChannel.send(END_MARKER);
}
// Provide a callback to receive control-related comms from the device
onControlMessage(cb) {
this.#onControlMessage = cb;
}
sendBluetoothMessage(msg) {
this.#bluetoothChannel.send(msg);
}
onBluetoothMessage(cb) {
this.#onBluetoothMessage = cb;
}
// Provide a callback to receive connectionstatechange states.
onConnectionStateChange(cb) {
this.#pc.addEventListener(
'connectionstatechange', evt => cb(this.#pc.connectionState));
}
}
class Controller {
#pc;
#serverConnector;
constructor(serverConnector) {
this.#serverConnector = serverConnector;
serverConnector.onDeviceMsg(msg => this.#onDeviceMessage(msg));
}
#onDeviceMessage(message) {
let type = message.type;
switch (type) {
case 'offer':
this.#onOffer({type: 'offer', sdp: message.sdp});
break;
case 'answer':
this.#onAnswer({type: 'answer', sdp: message.sdp});
break;
case 'ice-candidate':
this.#onIceCandidate(new RTCIceCandidate({
sdpMid: message.mid,
sdpMLineIndex: message.mLineIndex,
candidate: message.candidate
}));
break;
case 'error':
console.error('Device responded with error message: ', message.error);
break;
default:
console.error('Unrecognized message type from device: ', type);
}
}
async #sendClientDescription(desc) {
console.debug('sendClientDescription');
return this.#serverConnector.sendToDevice({type: 'answer', sdp: desc.sdp});
}
async #sendIceCandidate(candidate) {
console.debug('sendIceCandidate');
return this.#serverConnector.sendToDevice({type: 'ice-candidate', candidate});
}
async #onOffer(desc) {
console.debug('Remote description (offer): ', desc);
try {
await this.#pc.setRemoteDescription(desc);
let answer = await this.#pc.createAnswer();
console.debug('Answer: ', answer);
await this.#pc.setLocalDescription(answer);
await this.#sendClientDescription(answer);
} catch (e) {
console.error('Error processing remote description (offer)', e)
throw e;
}
}
async #onAnswer(answer) {
console.debug('Remote description (answer): ', answer);
try {
await this.#pc.setRemoteDescription(answer);
} catch (e) {
console.error('Error processing remote description (answer)', e)
throw e;
}
}
#onIceCandidate(iceCandidate) {
console.debug(`Remote ICE Candidate: `, iceCandidate);
this.#pc.addIceCandidate(iceCandidate);
}
ConnectDevice(pc) {
this.#pc = pc;
console.debug('ConnectDevice');
// ICE candidates will be generated when we add the offer. Adding it here
// instead of in _onOffer because this function is called once per peer
// connection, while _onOffer may be called more than once due to
// renegotiations.
this.#pc.addEventListener('icecandidate', evt => {
if (evt.candidate) this.#sendIceCandidate(evt.candidate);
});
this.#serverConnector.sendToDevice({type: 'request-offer'});
}
async renegotiateConnection() {
console.debug('Re-negotiating connection');
let offer = await this.#pc.createOffer();
console.debug('Local description (offer): ', offer);
await this.#pc.setLocalDescription(offer);
this.#serverConnector.sendToDevice({type: 'offer', sdp: offer.sdp});
}
}
function createPeerConnection(infra_config) {
let pc_config = {iceServers: []};
for (const stun of infra_config.ice_servers) {
pc_config.iceServers.push({urls: 'stun:' + stun});
}
let pc = new RTCPeerConnection(pc_config);
pc.addEventListener('icecandidate', evt => {
console.debug('Local ICE Candidate: ', evt.candidate);
});
pc.addEventListener('iceconnectionstatechange', evt => {
console.debug(`ICE State Change: ${pc.iceConnectionState}`);
});
pc.addEventListener(
'connectionstatechange',
evt => console.debug(
`WebRTC Connection State Change: ${pc.connectionState}`));
return pc;
}
export async function Connect(deviceId, serverConnector) {
let requestRet = await serverConnector.requestDevice(deviceId);
let deviceInfo = requestRet.deviceInfo;
let infraConfig = requestRet.infraConfig;
console.debug('Device available:');
console.debug(deviceInfo);
let pc_config = {iceServers: []};
if (infraConfig.ice_servers && infraConfig.ice_servers.length > 0) {
for (const server of infraConfig.ice_servers) {
pc_config.iceServers.push(server);
}
}
let pc = createPeerConnection(infraConfig);
let control = new Controller(serverConnector);
let deviceConnection = new DeviceConnection(pc, control);
deviceConnection.description = deviceInfo;
return new Promise((resolve, reject) => {
pc.addEventListener('connectionstatechange', evt => {
let state = pc.connectionState;
if (state == 'connected') {
resolve(deviceConnection);
} else if (state == 'failed') {
reject(evt);
}
});
control.ConnectDevice(pc);
});
}