Hi all! I’m looking to combine face tracking and Camera feed shaders.
Studio: Got face tracking set up, but unsure how I’d go about adding the camera shaders
Editor: Got the Camera shader project set up, but when I add to the head.html the project fails to build.
Thoughts??
Here’s an example CameraPipelineModule I created in a Studio component: 
import * as ecs from '@8thwall/ecs'
ecs.registerComponent({
name: 'warp',
schema: {
},
schemaDefaults: {
},
data: {
},
stateMachine: ({world, eid}) => {
ecs.defineState('default')
.initial()
.onEnter(() => {
window.addEventListener('xrloaded', (e) => {
const {XR8, THREE} = window as any
const threeTexture = new THREE.Texture()
XR8.addCameraPipelineModule({
name: 'warp',
onStart: ({canvasWidth, canvasHeight}) => {
console.log('started')
},
onProcessGpu: ({frameStartResult}) => {
const {cameraTexture, GLctx, textureWidth, textureHeight} = frameStartResult
threeTexture.isDataTexture = true
threeTexture.image = {width: textureWidth, height: textureHeight}
threeTexture.needsUpdate = true
threeTexture.encoding = THREE.sRGBEncoding
threeTexture.onUpdate = () => {
GLctx.bindTexture(GLctx.TEXTURE_2D, cameraTexture)
}
const object3D = world.three.entityToObject.get(eid)
object3D.material.envMap = threeTexture
object3D.material.map = threeTexture
object3D.traverse((node: any) => {
if (node.isMesh) {
node.material.envMap = threeTexture
node.material.map = threeTexture
}
})
},
})
})
})
},
add: (world, component) => {
},
tick: (world, component) => {
},
remove: (world, component) => {
},
})
Does that help?
Hey, thanks for the reply / code. I’m also finding out two more thing i need to figure out.
I’ve gotten it successfully working with this code:
import * as ecs from '@8thwall/ecs'
ecs.registerComponent({
name: 'Camera Shaders',
schema: {
},
schemaDefaults: {
},
data: {
},
stateMachine: ({world, eid}) => {
ecs.defineState('default')
.initial()
.onEnter(() => {
window.addEventListener('xrloaded', (e) => {
const {XR8} = window as any
const fragmentShaders = [ // Define some simple shaders to apply to the camera feed.
` precision mediump float; // Just the camera feed.
varying vec2 texUv;
uniform sampler2D sampler;
void main() { gl_FragColor = texture2D(sampler, texUv); }`,
` precision mediump float; // Color boost.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
float y = dot(c.rgb, vec3(0.299, 0.587, 0.114));
float u = dot(c.rgb, vec3(-.159, -.331, .5)) * 6.0;
float v = dot(c.rgb, vec3(.5, -.419, -.081)) * 3.0;
gl_FragColor = vec4(y + 1.4 * v, y - .343 * u - .711 * v, y + 1.765 * u, c.a);
}`,
` precision mediump float; // Vignette.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
float x = texUv.x - .5;
float y = texUv.y - .5;
float v = 1.5 - sqrt(x * x + y * y) * 2.5;
vec4 c = texture2D(sampler, texUv);
gl_FragColor = vec4(c.rgb * (v > 1.0 ? 1.0 : v), c.a);
}`,
` precision mediump float; // Black and white.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
gl_FragColor = vec4(vec3(dot(c.rgb, vec3(0.299, 0.587, 0.114))), c.a);
}`,
` precision mediump float; // Sepia.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
gl_FragColor.r = dot(c.rgb, vec3(.393, .769, .189));
gl_FragColor.g = dot(c.rgb, vec3(.349, .686, .168));
gl_FragColor.b = dot(c.rgb, vec3(.272, .534, .131));
gl_FragColor.a = c.a;
}`,
` precision mediump float; // Purple.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
float y = dot(c.rgb, vec3(0.299, 0.587, 0.114));
vec3 p = vec3(.463, .067, .712);
vec3 rgb = y < .25 ? (y * 4.0) * p : ((y - .25) * 1.333) * (vec3(1.0, 1.0, 1.0) - p) + p;
gl_FragColor = vec4(rgb, c.a);
}`,
` precision mediump float; // Shader with time variable
uniform float time;
varying vec2 texUv;
uniform sampler2D sampler;
void main( void ) {
vec2 position = texUv;
float color = 0.0;
color += sin( position.x * cos( time / 15.0 ) * 80.0 ) + cos( position.y * cos( time / 15.0 ) * 10.0 );
color += sin( position.y * sin( time / 10.0 ) * 40.0 ) + cos( position.x * sin( time / 25.0 ) * 40.0 );
color += sin( position.x * sin( time / 5.0 ) * 10.0 ) + sin( position.y * sin( time / 35.0 ) * 80.0 );
color *= sin( time / 10.0 ) * 0.5;
gl_FragColor = texture2D(sampler, texUv) + vec4( vec3( color, color * 0.5, sin( color + time / 3.0 ) * 0.75 ), 1.0 );
}`,
]
const vertexShaders = [ // Vertex shaders for swapping whether or not drawing is mirrored.
` attribute vec3 position; // Normal drawing
attribute vec2 uv;
varying vec2 texUv;
void main() {
gl_Position = vec4(position, 1.0);
texUv = uv;
}`,
` attribute vec3 position; // Mirrored drawing
attribute vec2 uv;
varying vec2 texUv;
void main() {
gl_Position = vec4(position, 1.0);
texUv = vec2(1.0 - uv.x, uv.y);
}`,
]
let uniforms = 1.0
XR8.addCameraPipelineModule({
name: 'Custom Shaders',
onStart: ({canvasWidth, canvasHeight}) => {
console.log('started')
XR8.GlTextureRenderer.configure({fragmentSource: fragmentShaders[1]})
},
onUpdate: ({frameStartResult, processGpuResult}) => {
if (processGpuResult.gltexturerenderer) {
const {shader} = processGpuResult.gltexturerenderer
const {GLctx} = frameStartResult
const timeLoc = GLctx.getUniformLocation(shader, 'time')
if (timeLoc) {
const p = XR8.GlTextureRenderer.getGLctxParameters(GLctx)
GLctx.useProgram(shader)
GLctx.uniform1f(timeLoc, uniforms)
XR8.GlTextureRenderer.setGLctxParameters(GLctx, p)
uniforms += 0.1
}
}
},
})
})
})
},
add: (world, component) => {
},
tick: (world, component) => {
},
remove: (world, component) => {
},
})
This is a component that I’ve attached to the camera in my scene. As you can see from the screenshot it using the second shader in the list, as the first shader is just the normal camera feed.
How can I add this in a World Camera?
You can apply this component to any camera. You simply need to change the camera type to ‘World’.
Huh, you’re right. I created a new project and was able to add it easily. If I injected HTML and CSS into my game, could that interfere with shader rendering? I can’t see the effect in my main game.
Strange, If you remove the custom CSS does it work?
It didn’t work. I reopened the test project I created for this, and I noticed that the shader appears and disappears in World Camera mode whenever I reload the game in the simulator.
When I switch the camera to Face mode and test the game in the editor, the shader works. Then, if I switch it back to World Camera, it also works in the editor. However, if the camera is set to World and I refresh the project in the browser, the effect disappears.
By the way, I also tested this on my phone using both Safari and Chrome with Face and World Camera modes, but the shader doesn’t appear at all.
Update.
Now, it sometimes works in my phone. I don’t know why it isn’t consistent though. 
Likely it has to do with the xrloaded event not being dispatched correctly. I’ll let the engineering team know.
@Herman_Ptacnik & @GeorgeButler here is a slightly modified script that works for me every time in studio. First we check if XR8 is attached the window before adding an event listener for the xrloaded event. Seems you need both cases for now. import * as ecs from ‘@8thwall/ecs’
import * as ecs from '@8thwall/ecs'
ecs.registerComponent({
name: 'Camera Shaders',
schema: {
},
schemaDefaults: {
},
data: {
},
stateMachine: ({world, eid}) => {
ecs.defineState('default')
.initial()
.onEnter(() => {
// window.addEventListener('xrloaded', () => {})
const onxrloaded = () => {
console.log('xr8 loaded')
const {XR8} = window as any
const fragmentShaders = [ // Define some simple shaders to apply to the camera feed.
` precision mediump float; // Just the camera feed.
varying vec2 texUv;
uniform sampler2D sampler;
void main() { gl_FragColor = texture2D(sampler, texUv); }`,
` precision mediump float; // Color boost.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
float y = dot(c.rgb, vec3(0.299, 0.587, 0.114));
float u = dot(c.rgb, vec3(-.159, -.331, .5)) * 6.0;
float v = dot(c.rgb, vec3(.5, -.419, -.081)) * 3.0;
gl_FragColor = vec4(y + 1.4 * v, y - .343 * u - .711 * v, y + 1.765 * u, c.a);
}`,
` precision mediump float; // Vignette.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
float x = texUv.x - .5;
float y = texUv.y - .5;
float v = 1.5 - sqrt(x * x + y * y) * 2.5;
vec4 c = texture2D(sampler, texUv);
gl_FragColor = vec4(c.rgb * (v > 1.0 ? 1.0 : v), c.a);
}`,
` precision mediump float; // Black and white.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
gl_FragColor = vec4(vec3(dot(c.rgb, vec3(0.299, 0.587, 0.114))), c.a);
}`,
` precision mediump float; // Sepia.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
gl_FragColor.r = dot(c.rgb, vec3(.393, .769, .189));
gl_FragColor.g = dot(c.rgb, vec3(.349, .686, .168));
gl_FragColor.b = dot(c.rgb, vec3(.272, .534, .131));
gl_FragColor.a = c.a;
}`,
` precision mediump float; // Purple.
varying vec2 texUv;
uniform sampler2D sampler;
void main() {
vec4 c = texture2D(sampler, texUv);
float y = dot(c.rgb, vec3(0.299, 0.587, 0.114));
vec3 p = vec3(.463, .067, .712);
vec3 rgb = y < .25 ? (y * 4.0) * p : ((y - .25) * 1.333) * (vec3(1.0, 1.0, 1.0) - p) + p;
gl_FragColor = vec4(rgb, c.a);
}`,
` precision mediump float; // Shader with time variable
uniform float time;
varying vec2 texUv;
uniform sampler2D sampler;
void main( void ) {
vec2 position = texUv;
float color = 0.0;
color += sin( position.x * cos( time / 15.0 ) * 80.0 ) + cos( position.y * cos( time / 15.0 ) * 10.0 );
color += sin( position.y * sin( time / 10.0 ) * 40.0 ) + cos( position.x * sin( time / 25.0 ) * 40.0 );
color += sin( position.x * sin( time / 5.0 ) * 10.0 ) + sin( position.y * sin( time / 35.0 ) * 80.0 );
color *= sin( time / 10.0 ) * 0.5;
gl_FragColor = texture2D(sampler, texUv) + vec4( vec3( color, color * 0.5, sin( color + time / 3.0 ) * 0.75 ), 1.0 );
}`,
]
const vertexShaders = [ // Vertex shaders for swapping whether or not drawing is mirrored.
` attribute vec3 position; // Normal drawing
attribute vec2 uv;
varying vec2 texUv;
void main() {
gl_Position = vec4(position, 1.0);
texUv = uv;
}`,
` attribute vec3 position; // Mirrored drawing
attribute vec2 uv;
varying vec2 texUv;
void main() {
gl_Position = vec4(position, 1.0);
texUv = vec2(1.0 - uv.x, uv.y);
}`,
]
let uniforms = 1.0
XR8.addCameraPipelineModule({
name: 'Custom Shaders',
onStart: ({canvasWidth, canvasHeight}) => {
console.log('started')
XR8.GlTextureRenderer.configure({fragmentSource: fragmentShaders[1]})
},
onUpdate: ({frameStartResult, processGpuResult}) => {
if (processGpuResult.gltexturerenderer) {
const {shader} = processGpuResult.gltexturerenderer
const {GLctx} = frameStartResult
const timeLoc = GLctx.getUniformLocation(shader, 'time')
if (timeLoc) {
const p = XR8.GlTextureRenderer.getGLctxParameters(GLctx)
GLctx.useProgram(shader)
GLctx.uniform1f(timeLoc, uniforms)
XR8.GlTextureRenderer.setGLctxParameters(GLctx, p)
uniforms += 0.1
}
}
},
})
}
// @ts-ignore
window.XR8 ? onxrloaded() : window.addEventListener('xrloaded', onxrloaded)
})
},
add: (world, component) => {
},
tick: (world, component) => {
},
remove: (world, component) => {
},
})