fix(engine): only create gl shaders for linked shader program entry pointsHEADmaster

1 files changed, 86 insertions, 70 deletions

diff --git a/engine/src/rendering/backend/opengl.rs b/engine/src/rendering/backend/opengl.rs
index 2926cde..0a58417 100644
--- a/engine/src/rendering/backend/opengl.rs
+++ b/engine/src/rendering/backend/opengl.rs
@@ -62,7 +62,7 @@ use opengl_bindings::misc::{
 };
 use opengl_bindings::shader::{
     Error as GlShaderError,
-    Kind as ShaderKind,
+    Kind as GlShaderKind,
     Program as GlShaderProgram,
     Shader as GlShader,
 };
@@ -940,12 +940,12 @@ fn handle_commands(
             }
             Command::CreateShaderProgram(object_id, shader_program) => {
                 if let Err(err) = backend_resources
-                    .try_create_resource::<CreateShaderError>(
+                    .try_create_resource::<GlShaderCreationError>(
                         object_store,
                         object_id,
                         || {
                             Ok(BackendResource::Shader {
-                                program: create_shader_program(
+                                program: create_and_init_gl_shader_program(
                                     gl_context,
                                     &shader_program,
                                     backend_properties,
@@ -1861,107 +1861,123 @@ fn cube_map_texture_face_to_gl(face: CubeMapTextureFace) -> GlCubeMapTextureFace
     }
 }
 
-fn create_shader_program(
+fn create_and_init_gl_shader_program(
     current_context: &MaybeCurrentContextWithFns,
     shader_program: &ShaderProgram,
     backend_properties: &BackendProperties,
-) -> Result<GlShaderProgram, CreateShaderError>
+) -> Result<GlShaderProgram, GlShaderCreationError>
 {
+    let gl_shader_program = GlShaderProgram::new(current_context);
+
     let shader_program_reflection = shader_program
         .reflection(backend_properties.shader_target_id)
         .expect("Not possible");
 
-    let (vs_entry_point_index, vs_entry_point_reflection) = shader_program_reflection
-        .entry_points()
-        .enumerate()
-        .find(|(_, entry_point)| entry_point.stage() == ShaderStage::Vertex)
-        .ok_or_else(|| {
-            CreateShaderError::NoShaderStageEntrypointFound(ShaderStage::Vertex)
-        })?;
+    if shader_program
+        .metadata()
+        .link_entrypoints
+        .contains(ShaderEntrypointFlags::VERTEX)
+    {
+        let Some(vs_entry_point_index) = shader_program_reflection
+            .entry_points()
+            .position(|entry_point| entry_point.stage() == ShaderStage::Vertex)
+        else {
+            unreachable!();
+        };
 
-    let vertex_shader_entry_point_code = shader_program
-        .get_entry_point_code(
-            vs_entry_point_index.try_into().expect(
-                "Vertex shader entry point index does not fit in 32-bit unsigned int",
-            ),
-            backend_properties.shader_target_id,
-        )
-        .map_err(|err| CreateShaderError::GetShaderEntryPointCodeFailed {
-            err,
-            stage: ShaderStage::Vertex,
-            entrypoint: vs_entry_point_reflection
-                .name()
-                .map_or("(none)".into(), |name| name.to_string().into()),
-        })?;
+        let gl_vertex_shader = create_and_init_gl_shader(
+            current_context,
+            shader_program,
+            vs_entry_point_index,
+            GlShaderKind::Vertex,
+            backend_properties,
+        )?;
 
-    let (fs_entry_point_index, fs_entry_point_reflection) = shader_program_reflection
-        .entry_points()
-        .enumerate()
-        .find(|(_, entry_point)| entry_point.stage() == ShaderStage::Fragment)
-        .ok_or_else(|| {
-            CreateShaderError::NoShaderStageEntrypointFound(ShaderStage::Fragment)
-        })?;
+        gl_shader_program.attach(current_context, &gl_vertex_shader);
+    }
 
-    let fragment_shader_entry_point_code = shader_program
-        .get_entry_point_code(
-            fs_entry_point_index.try_into().expect(
-                "Fragment shader entry point index does not fit in 32-bit unsigned int",
-            ),
-            backend_properties.shader_target_id,
-        )
-        .map_err(|err| CreateShaderError::GetShaderEntryPointCodeFailed {
-            err,
-            stage: ShaderStage::Fragment,
-            entrypoint: fs_entry_point_reflection
-                .name()
-                .map_or("(none)".into(), |name| name.to_string().into()),
-        })?;
+    if shader_program
+        .metadata()
+        .link_entrypoints
+        .contains(ShaderEntrypointFlags::FRAGMENT)
+    {
+        let Some(fs_entry_point_index) = shader_program_reflection
+            .entry_points()
+            .position(|entry_point| entry_point.stage() == ShaderStage::Fragment)
+        else {
+            unreachable!();
+        };
 
-    let vertex_shader = GlShader::new(current_context, ShaderKind::Vertex);
+        let gl_fragment_shader = create_and_init_gl_shader(
+            current_context,
+            shader_program,
+            fs_entry_point_index,
+            GlShaderKind::Fragment,
+            backend_properties,
+        )?;
 
-    vertex_shader.set_source(
-        current_context,
-        vertex_shader_entry_point_code.as_str().unwrap(),
-    )?;
+        gl_shader_program.attach(current_context, &gl_fragment_shader);
+    }
 
-    vertex_shader.compile(current_context)?;
+    gl_shader_program.link(current_context)?;
 
-    let fragment_shader = GlShader::new(current_context, ShaderKind::Fragment);
+    Ok(gl_shader_program)
+}
 
-    fragment_shader.set_source(
-        current_context,
-        fragment_shader_entry_point_code.as_str().unwrap(),
-    )?;
+fn create_and_init_gl_shader(
+    current_context: &MaybeCurrentContextWithFns,
+    shader_program: &ShaderProgram,
+    entry_point_index: usize,
+    gl_shader_kind: GlShaderKind,
+    backend_properties: &BackendProperties,
+) -> Result<GlShader, GlShaderCreationError>
+{
+    let entry_point_index = u32::try_from(entry_point_index)
+        .expect("Shader entry point index does not fit in 32-bit unsigned int");
 
-    fragment_shader.compile(current_context)?;
+    let entry_point_code = shader_program
+        .get_entry_point_code(entry_point_index, backend_properties.shader_target_id)
+        .map_err(|err| GlShaderCreationError::GetShaderEntryPointCodeFailed {
+            err,
+            stage: match gl_shader_kind {
+                GlShaderKind::Vertex => ShaderStage::Vertex,
+                GlShaderKind::Fragment => ShaderStage::Fragment,
+            },
+            entry_point: shader_program
+                .reflection(backend_properties.shader_target_id)
+                .ok()
+                .and_then(|program_reflection| {
+                    program_reflection
+                        .get_entry_point_by_index(entry_point_index)?
+                        .name()
+                        .map(ToOwned::to_owned)
+                })
+                .map_or("<unknown>".into(), Into::into),
+        })?;
 
-    let gl_shader_program = GlShaderProgram::new(current_context);
+    let gl_shader = GlShader::new(current_context, gl_shader_kind);
 
-    gl_shader_program.attach(current_context, &vertex_shader);
-    gl_shader_program.attach(current_context, &fragment_shader);
+    gl_shader.set_source(current_context, entry_point_code.as_str().unwrap())?;
 
-    gl_shader_program.link(current_context)?;
+    gl_shader.compile(current_context)?;
 
-    Ok(gl_shader_program)
+    Ok(gl_shader)
 }
 
 #[derive(Debug, thiserror::Error)]
-enum CreateShaderError
+enum GlShaderCreationError
 {
     #[error(
-        "Failed to get code of shader program entry point {entrypoint} of stage {stage:?}"
+        "Failed to get code of shader program entry point {entry_point} of stage {stage:?}"
     )]
     GetShaderEntryPointCodeFailed
     {
         #[source]
         err: ShaderError,
         stage: ShaderStage,
-        entrypoint: Cow<'static, str>,
+        entry_point: Cow<'static, str>,
     },
 
-    #[error("No entrypoint was found for shader stage {0:?}")]
-    NoShaderStageEntrypointFound(ShaderStage),
-
     #[error("Failed to create shader vertex description")]
     VertexDescCreationFailed(#[from] ShaderVertexDescCreationError),