javascript - Decode images in web worker

Question

In our WebGL application I'm trying to load and decode texture images in a web worker in order to avoid rendering hick-ups in the main thread. Using createImageBitmap in the worker and transferring the image bitmap back to the main thread works well, but in Chrome this will use three or more (maybe depending on number of cores?) separate workers (ThreadPoolForegroundWorker) which together with the main thread and my own worker will result in five threads.

I'm guessing this causes my remaining rendering disturbances on my quad core since I can see some inexplicable long times in the Performance feature of Chrome's DevTools.

So, can I limit the number of workers used by createImageBitmap somehow? Even if I transfer the images as blobs or array buffers to the main thread and activate createImageBitmap from there, its workers will compete with my own worker and the main thread.

I have tried creating regular Images in the worker instead to explicitly decode them there, but Image is not defined in the worker context, neither is document if I'd like to create them as elements. And regular Images are not transferable either, so creating them on the main thread and transferring them to the worker doesn't seem feasible either.

Looking forward to any suggestions...

Answer 1

There's no reason to use createImageBitmap in a worker (well, see bottom). The browser already decodes the image in a separate thread. Doing it in a worker doesn't get you anything. The bigger issue is there's no way for ImageBitmap to know how you are going to use the image when you finally pass it to WebGL. If you ask for a format that's different than what ImageBitmap decoded then WebGL has to convert and/or decode it again and you can't give ImageBitmap enough info to tell it the format you want it to decode in.

On top of that WebGL in Chrome has to transfer the data of the image from the render process to the GPU process which for a large image is a relatively big copy (1024x1024 by RGBA is 4meg)

A better API IMO would have allowed you to tell ImageBitmap what format you want and where you want it (CPU, GPU). That way the browser could prep the image asynchonously and have it require no heavy work when done.

In any case, here's a test. If you uncheck "update texture" then it's still downloading and decoding textures but it's just not calling gl.texImage2D to upload the texture. In that case I see no jank (not proof that's the issue but that's where I think it is)

const m4 = twgl.m4;
const gl = document.querySelector('#webgl').getContext('webgl');
const ctx = document.querySelector('#graph').getContext('2d');

let update = true;
document.querySelector('#update').addEventListener('change', function() {
  update = this.checked;
});

const vs = `
attribute vec4 position;
uniform mat4 matrix;
varying vec2 v_texcoord;
void main() {
  gl_Position = matrix * position;
  v_texcoord = position.xy;
}
`

const fs = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D tex;
void main() {
  gl_FragColor = texture2D(tex, v_texcoord);
}
`;

const program = twgl.createProgram(gl, [vs, fs]);
const posLoc = gl.getAttribLocation(program, 'position');
const matLoc = gl.getUniformLocation(program, 'matrix');

const buf = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, buf);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
  0, 0,
  1, 0,
  0, 1,
  0, 1,
  1, 0,
  1, 1,
]), gl.STATIC_DRAW);

gl.enableVertexAttribArray(posLoc);
gl.vertexAttribPointer(posLoc, 2, gl.FLOAT, false, 0, 0);

const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texImage2D(
    gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE,
    new Uint8Array([0, 0, 255, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);

const m = m4.identity();
let frameCount = 0;
let previousTime = 0;
let imgNdx = 0;
let imgAspect = 1;

const imageUrls = [
  'https://i.imgur.com/KjUybBD.png',
  'https://i.imgur.com/AyOufBk.jpg',
  'https://i.imgur.com/UKBsvV0.jpg',
  'https://i.imgur.com/TSiyiJv.jpg',
];

async function loadNextImage() {
  const url = `${imageUrls[imgNdx]}?cachebust=${performance.now()}`;
  imgNdx = (imgNdx + 1) % imageUrls.length;
  const res = await fetch(url, {mode: 'cors'});
  const blob = await res.blob();
  const bitmap = await createImageBitmap(blob, {
    premultiplyAlpha: 'none',
    colorSpaceConversion: 'none',
  });
  if (update) {
    gl.bindTexture(gl.TEXTURE_2D, tex);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, bitmap);
    imgAspect = bitmap.width / bitmap.height;
  }
  setTimeout(loadNextImage, 1000);
}
loadNextImage();

function render(currentTime) {
  const deltaTime = currentTime - previousTime;
  previousTime = currentTime;
  
  {
    const {width, height} = ctx.canvas;
    const x = frameCount % width;
    const y = 1000 / deltaTime / 60 * height / 2;
    ctx.fillStyle = frameCount % (width * 2) < width ? 'red' : 'blue';
    ctx.clearRect(x, 0, 1, height);
    ctx.fillRect(x, y, 1, height);
    ctx.clearRect(0, 0, 30, 15);
    ctx.fillText((1000 / deltaTime).toFixed(1), 2, 10);
  }

  gl.useProgram(program);
  const dispAspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
  m4.scaling([1 / dispAspect, 1, 1], m);
  m4.rotateZ(m, currentTime * 0.001, m);
  m4.scale(m, [imgAspect, 1, 1], m);
  m4.translate(m, [-0.5, -0.5, 0], m);
  gl.uniformMatrix4fv(matLoc, false, m);
  gl.drawArrays(gl.TRIANGLES, 0, 6);
  
  ++frameCount;
  requestAnimationFrame(render);
}
requestAnimationFrame(render);

canvas { border: 1px solid black; margin: 2px; }
#ui { position: absolute; }

<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<div id="ui"><input type="checkbox" id="update" checked><label for="update">Update Texture</label></div>
<canvas id="webgl"></canvas>
<canvas id="graph"></canvas>