Hi everyone!
I would be very interested in getting feedback on how to make touch intersections work on a VPS mesh, so that I can add a simple 3D box at the intersection coordinates.
Here my component vps-detect-mesh:
import * as ecs from '@8thwall/ecs'
const {THREE} = window as any
const MESH_NAME = 'vps-mesh'
const touchPosition = new THREE.Vector2()
const raycaster = new THREE.Raycaster()
ecs.registerComponent({
name: 'VPS Detect Mesh',
schema: {
},
data: {
state: ecs.string,
},
add: (world, component) => {
/** BASE */
const {eid} = component
const {scene} = world.three
let meshFound = false
let mesh = null
/** VPS DETECT */
const foundMesh = (event) => {
const {bufferGeometry, ...others} = event.data
const existingMesh = scene.getObjectByName(MESH_NAME)
if (existingMesh) {
existingMesh.visible = true // Show the mesh again if it exists
return
}
if (meshFound === true) return
// Construct VPS mesh
const vpsMeshMaterial = new THREE.MeshBasicMaterial({
visible: true,
transparent: true,
opacity: 1,
wireframe: true,
})
mesh = new THREE.Mesh(bufferGeometry, vpsMeshMaterial)
mesh.name = MESH_NAME
// mesh.quaternion.copy(others.rotation)
// mesh.position.copy(others.position)
// const vpsObject = world.three.entityToObject.get(eid)
// vpsObject.add(mesh) // Test adding mesh as an element's child
scene.add(mesh) // Test adding mesh in the scene
meshFound = true
world.events.dispatch(eid, 'start_drawing')
const axesHelper = new THREE.AxesHelper(5)
mesh.add(axesHelper) // Debug VPS position
console.log('-> VPS Mesh Found', {others})
}
const meshLost = () => {
const existingMesh = scene.getObjectByName(MESH_NAME)
if (existingMesh) {
existingMesh.visible = false // Hide the mesh until we find it again
world.events.dispatch(component.eid, 'end_drawing')
console.log('-> VPS Mesh Lost')
}
}
const locationFound = () => {
console.log('-> VPS Location Found')
}
const locationLost = (event) => {
console.log('-> VPS Location Lost. Please look around to relocate yourself.')
}
// Set up listeners for all our VPS events
world.events.addListener(world.events.globalId, 'reality.meshfound', foundMesh)
world.events.addListener(world.events.globalId, 'reality.meshlost', meshLost)
world.events.addListener(world.events.globalId, 'reality.locationfound', locationFound)
world.events.addListener(world.events.globalId, 'reality.locationlost', locationLost)
},
stateMachine: ({world, eid, dataAttribute, schemaAttribute}) => {
const {renderer: {domElement}, scene, activeCamera} = world.three
// UPDATE STATE--------------------------------------
const updateState = (newState) => {
dataAttribute.set(eid, {state: newState})
}
// DEBUG BOX-----------------------------------------
const debugAddBox = (event) => {
const vpsMesh = scene.getObjectByName(MESH_NAME)
if (!vpsMesh) return
event.preventDefault()
// Use the renderer's canvas bounds for proper NDC conversion.
const rect = domElement.getBoundingClientRect()
touchPosition.x = ((event.touches[0].clientX - rect.left) / rect.width) * 2 - 1
touchPosition.y = -((event.touches[0].clientY - rect.top) / rect.height) * 2 + 1
activeCamera.updateMatrixWorld()
vpsMesh.updateMatrixWorld()
raycaster.setFromCamera(touchPosition, activeCamera)
const intersects = raycaster.intersectObject(vpsMesh, true)
if (intersects.length > 0) {
const {point} = intersects[0]
console.log('World Intersection:', point)
// Small debug box:
const geometry = new THREE.BoxGeometry(0.05, 0.05, 0.05)
const material = new THREE.MeshBasicMaterial({color: 'hotpink'})
const box = new THREE.Mesh(geometry, material)
box.position.copy(point)
scene.add(box)
} else {
console.log('No intersection found for debug box.')
}
}
// CALLBACKS-------------------------------------------
const handleTouchStart = (event) => {
debugAddBox(event)
}
// STATES-------------------------------------------
ecs.defineState('idle')
.initial()
.onEnter(() => updateState('idle'))
.onEvent('start_drawing', 'drawing')
ecs.defineState('drawing')
.onEnter(() => {
updateState('drawing')
domElement.addEventListener('touchstart', handleTouchStart)
})
.onExit(() => {
domElement.removeEventListener('touchstart', handleTouchStart)
})
.onEvent('end_drawing', 'idle')
// .listen(eid, ecs.input.SCREEN_TOUCH_START, (event) => {
// console.log('SCREEN_TOUCH_START', {event})
// })
// .listen(eid, ecs.input.SCREEN_TOUCH_MOVE, (event) => {
// console.log('SCREEN_TOUCH_MOVE', {event})
// })
// .listen(eid, ecs.input.SCREEN_TOUCH_END, (event) => {
// console.log('SCREEN_TOUCH_END', {event})
// })
},
remove: (world) => {
const mesh = world.three.scene.getObjectByName(MESH_NAME)
if (mesh) {
world.three.scene.remove(mesh)
mesh.material.dispose()
mesh.geometry.dispose()
}
},
})
This code uses the logic of the VPS mesh being found, and once the mesh is available, I simply want to add a 3D box at the intersection point of my finger touch event and the VPS mesh.
- I commented out the listeners from
ecs.input.SCREEN_TOUCH_...because they weren’t working. I believe this is because the VPS mesh is being added directly to the scene usingscene.add(mesh). - I created touch events based on the domElement/canvas from the renderer.
The main issue I’m facing right now is that this code isn’t working as expected, only one specific area of the VPS mesh responds to the raycaster intersection, and when the box is placed, it doesn’t align with the finger’s intersection point.
Although, this code is working fine with 8thWall + A-FRAME and with 8thWall + THREE.JS but it is not working with Studio.
If anyone can give me feedback on what I did wrong and help me understand how to make the touch intersection work across the entire mesh and adding the box correctly it would be really helpful.
You can easily replicate this issue without a VPS mesh: (1) by creating an empty object in your scene, (2) attaching a custom component to it, but not setting a mesh or geometry initially. (3) Instead, create the mesh by code inside the component. For example inside your add function add these lines of code:
const geometry = new THREE.BoxGeometry(0.05, 0.05, 0.05)
const material = new THREE.MeshBasicMaterial({color: 'hotpink'})
const box = new THREE.Mesh(geometry, material)
and then either one or the other way to add your mesh in the scene:
const object = world.three.entityToObject.get(eid)
object.add(box)
or
scene.add(box)
In this case, events like ecs.input.SCREEN_TOUCH_... won’t work, and if you create a THREE.Raycaster, the intersection won’t work across the entire mesh either.
I’m sorry for the long message, I just wanted to share everything I’ve experienced in case someone can help. Any feedback would be greatly appreciated. Thank you!
