From db56a264db26f14e31d82cf18a7e872fde306bc6 Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Wed, 21 Aug 2019 12:39:15 -0400 Subject: [PATCH 1/6] Add initial browser bootstrap loading --- index.html | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/index.html b/index.html index 72b3c34..9a8a0e1 100644 --- a/index.html +++ b/index.html @@ -6603,7 +6603,6 @@

{'urls': 'stun:stun.l.google.com:19302'}, ], }; - const peerConnection = new RTCPeerConnection(peerConnectionConfig); peerConnection.ontrack = e => { console.log('got track', e); @@ -6736,6 +6735,7 @@

const connectionId = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 5); const peerConnections = []; +let browserPeerConnection = null; const _rtcConnect = () => { const s = new WebSocket(LAMBDA_URLS.presence); s.onopen = () => { @@ -6745,9 +6745,34 @@

method: 'init', connectionId, })); + + const peerConnectionConfig = { + iceServers: [ + {'urls': 'stun:stun.stunprotocol.org:3478'}, + {'urls': 'stun:stun.l.google.com:19302'}, + ], + }; + browserPeerConnection = new RTCPeerConnection(peerConnectionConfig); + browserPeerConnection.ontrack = e => { + console.log('got browser track', e, e.streams.length); + + const video = document.createElement('video'); + video.style.width = '100%'; + video.srcObject = e.streams[0]; + video.play(); + document.body.querySelector('header').style.display = 'none'; + document.body.querySelector('#canvas').style.display = 'none'; + document.body.appendChild(video); + }; + s.send(JSON.stringify({ + method: 'requestBrowser', + url: 'https://google.com/', + connectionId, + })); + peerConnections.push(browserPeerConnection); }; s.onmessage = e => { - // console.log('got message', e.data); + console.log('got message', e.data); const data = JSON.parse(e.data); const {method} = data; @@ -6837,6 +6862,29 @@

} else { console.warn('no such peer connection', peerConnectionId, peerConnections.map(peerConnection => peerConnection.connectionId)); } + } else if (method === 'respondBrowser') { + const {src: peerConnectionId, offer} = data; + browserPeerConnection.connectionId = peerConnectionId; + browserPeerConnection.onicecandidate = e => { + s.send(JSON.stringify({ + dst: peerConnectionId, + src: connectionId, + method: 'iceCandidate', + candidate: e.candidate, + })); + }; + browserPeerConnection.setRemoteDescription(offer); + browserPeerConnection.createAnswer() + .then(answer => { + browserPeerConnection.setLocalDescription(answer); + + s.send(JSON.stringify({ + dst: peerConnectionId, + src: connectionId, + method: 'answer', + answer, + })); + }); } }; s.onclose = () => { From f6e7799fceac58db92bedeef0eae5c49029cfd2f Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Tue, 27 Aug 2019 00:25:29 -0400 Subject: [PATCH 2/6] Add initial browser intersection tracking flow --- index.html | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/index.html b/index.html index 9a8a0e1..8f2992f 100644 --- a/index.html +++ b/index.html @@ -4370,7 +4370,9 @@

}; mesh.getIntersectionCandidates = () => mesh.visible ? - [mesh, keyboardMesh].concat(appIconMeshes.children) + [mesh, keyboardMesh] + .concat(appIconMeshes.children) + .concat(browserMesh || []) : []; return mesh; }; @@ -5782,6 +5784,8 @@

intersectionSpec.onclose(i); } else if (type === 'screen') { screenMesh.click(intersectionSpec); + } else if (type === 'browser') { + console.log('click browser'); // XXX } else if (type === 'map') { window.document.xrOffset.position[0] += intersectionSpec.x*7 - camera.position.x; window.document.xrOffset.position[2] += intersectionSpec.y*7 - camera.position.z; @@ -6736,6 +6740,8 @@

const connectionId = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 5); const peerConnections = []; let browserPeerConnection = null; +let browserDataChannel = null; +let browserMesh = null; const _rtcConnect = () => { const s = new WebSocket(LAMBDA_URLS.presence); s.onopen = () => { @@ -6760,9 +6766,52 @@

video.style.width = '100%'; video.srcObject = e.streams[0]; video.play(); - document.body.querySelector('header').style.display = 'none'; + + const geometry = new THREE.CylinderBufferGeometry(r, r, s, 10, 10, true, Math.PI*3/2+Math.PI/4, Math.PI/2) + .applyMatrix(new THREE.Matrix4().makeScale(1, 1, -1)) + .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0, r)); + + cost texture = new THREE.VideoTexture(video); + texture.minFilter = THREE.LinearFilter; + texture.magFilter = THREE.LinearFilter; + texture.format = THREE.RGBFormat; + + const material = new THREE.MeshBasicMaterial({ + map: texture, + side: THREE.DoubleSide, + }); + + const mesh = new THREE.Mesh(geometry, material); + mesh.frustumCulled = false; + mesh.intersect = ray => { + localRaycaster.ray.copy(ray); + const intersection = localRaycaster.intersectObject(mesh)[0]; + if (intersection) { + const {uv, distance} = intersection; + uv.y = 1-uv.y; + uv.x *= 1920; + uv.y *= 1080; + return { + type: 'browser', + x: uv.x, + y: uv.y, + distance, + cancel: true, + }; + } else { + return null; + } + }; + scene.add(mesh); + browserMesh = mesh; + + /* document.body.querySelector('header').style.display = 'none'; document.body.querySelector('#canvas').style.display = 'none'; - document.body.appendChild(video); + document.body.appendChild(video); */ + }; + const dataChannel = browserPeerConnection.createDataChannel(); + dataChannel.onopen = () => { + browserDataChannel = dataChannel; }; s.send(JSON.stringify({ method: 'requestBrowser', From be7b21b6c833c483222683661d5400ea0d793acc Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Tue, 27 Aug 2019 13:34:39 -0400 Subject: [PATCH 3/6] Comment out adapter.js --- index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/index.html b/index.html index 8f2992f..393834d 100644 --- a/index.html +++ b/index.html @@ -487,6 +487,7 @@

Loading...

+ From b5346275e665630bd04e25d180230b63252c263a Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Tue, 27 Aug 2019 13:35:08 -0400 Subject: [PATCH 4/6] New browser DataChannel strategy --- index.html | 55 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/index.html b/index.html index 393834d..0253d83 100644 --- a/index.html +++ b/index.html @@ -524,6 +524,7 @@

Loading...

}; const DEFAULT_URL = 'https://google.com'; const PARCEL_SIZE = 8; +const BROWSER_SIZE = 1920; const TOKEN_PRICE = 5e15; const DEFAULT_SKIN_URL = 'skin.png'; const RAY_COLOR = 0x44c2ff; @@ -6614,7 +6615,7 @@

}; let skinMesh = null; - const sendChannel = peerConnection.createDataChannel('sendChannel'); + const sendChannel = peerConnection.createDataChannel('pose'); let pingInterval = 0; let updateInterval = 0; sendChannel.onopen = () => { @@ -6760,6 +6761,11 @@

], }; browserPeerConnection = new RTCPeerConnection(peerConnectionConfig); + const dataChannel = browserPeerConnection.createDataChannel('browser', {id: 0, negotiated: true}); + dataChannel.onopen = () => { + console.log('got browser data channel', dataChannel); + }; + browserDataChannel = dataChannel; browserPeerConnection.ontrack = e => { console.log('got browser track', e, e.streams.length); @@ -6768,11 +6774,13 @@

video.srcObject = e.streams[0]; video.play(); + const s = 2; + const r = s*4/(2*Math.PI); const geometry = new THREE.CylinderBufferGeometry(r, r, s, 10, 10, true, Math.PI*3/2+Math.PI/4, Math.PI/2) .applyMatrix(new THREE.Matrix4().makeScale(1, 1, -1)) .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0, r)); - cost texture = new THREE.VideoTexture(video); + const texture = new THREE.VideoTexture(video); texture.minFilter = THREE.LinearFilter; texture.magFilter = THREE.LinearFilter; texture.format = THREE.RGBFormat; @@ -6784,36 +6792,63 @@

const mesh = new THREE.Mesh(geometry, material); mesh.frustumCulled = false; + const lastCoords = [NaN, NaN]; mesh.intersect = ray => { localRaycaster.ray.copy(ray); const intersection = localRaycaster.intersectObject(mesh)[0]; if (intersection) { const {uv, distance} = intersection; uv.y = 1-uv.y; - uv.x *= 1920; - uv.y *= 1080; + uv.x *= BROWSER_SIZE; + uv.y *= BROWSER_SIZE; + const {x, y} = uv; + if (x !== lastCoords[0] || y !== lastCoords[1]) { + mesh.mousemove(x, y); + + lastCoords[0] = x; + lastCoords[1] = y; + } return { type: 'browser', - x: uv.x, - y: uv.y, + x, + y, distance, + mesh, cancel: true, }; } else { return null; } }; + mesh.mousemove = (x, y) => { + console.log('mouse move', x, y, dataChannel && dataChannel.readyState); + if (dataChannel && dataChannel.readyState === 'open') { + dataChannel.send(JSON.stringify({ + method: 'mousemove', + x, + y: y + 160*(1 - y/BROWSER_SIZE), + })); + } + }; + mesh.click = () => { + if (dataChannel && dataChannel.readyState === 'open') { + dataChannel.send(JSON.stringify({ + method: 'click', + })); + } + }; + mesh.position.y = 2; scene.add(mesh); browserMesh = mesh; + e.track.onended = () => { + console.log('track ended!!!!!!!!!!!!!!!!!!!!!!!!!', e.track); + }; + /* document.body.querySelector('header').style.display = 'none'; document.body.querySelector('#canvas').style.display = 'none'; document.body.appendChild(video); */ }; - const dataChannel = browserPeerConnection.createDataChannel(); - dataChannel.onopen = () => { - browserDataChannel = dataChannel; - }; s.send(JSON.stringify({ method: 'requestBrowser', url: 'https://google.com/', From 41ffec12ba6c8827eb69db00b9d45ba003fa8942 Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Tue, 27 Aug 2019 13:35:38 -0400 Subject: [PATCH 5/6] Small browser click cleanup --- index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.html b/index.html index 0253d83..b5fe91c 100644 --- a/index.html +++ b/index.html @@ -5787,7 +5787,8 @@

} else if (type === 'screen') { screenMesh.click(intersectionSpec); } else if (type === 'browser') { - console.log('click browser'); // XXX + const {mesh: browserMesh} = intersectionSpec; + browserMesh.click(); } else if (type === 'map') { window.document.xrOffset.position[0] += intersectionSpec.x*7 - camera.position.x; window.document.xrOffset.position[2] += intersectionSpec.y*7 - camera.position.z; From ffc3b50f12382c96275478cbc42847be9ba748bd Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Tue, 27 Aug 2019 13:35:50 -0400 Subject: [PATCH 6/6] Remove dead script includes --- index.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/index.html b/index.html index b5fe91c..da3f7e8 100644 --- a/index.html +++ b/index.html @@ -6993,11 +6993,5 @@

})(); - - - - - -