chore(engine): add shader preprocessor

3 files changed, 692 insertions, 5 deletions

diff --git a/engine/src/lib.rs b/engine/src/lib.rs
index ae603b0..0eca914 100644
--- a/engine/src/lib.rs
+++ b/engine/src/lib.rs
@@ -15,6 +15,7 @@ mod matrix;
 mod opengl;
 mod projection;
 mod renderer;
+mod shader_preprocessor;
 mod transform;
 
 pub mod camera;
diff --git a/engine/src/object.rs b/engine/src/object.rs
index a8cd076..4112bcb 100644
--- a/engine/src/object.rs
+++ b/engine/src/object.rs
@@ -1,3 +1,7 @@
+use std::fs::read_to_string;
+use std::io::Error as IoError;
+use std::path::{Path, PathBuf};
+
 use crate::opengl::shader::{
     Error as ShaderError,
     Kind as ShaderKind,
@@ -5,13 +9,17 @@ use crate::opengl::shader::{
     Shader,
 };
 use crate::renderer::Renderable;
+use crate::shader_preprocessor::{Error as ShaderPreprocessorError, ShaderPreprocessor};
 use crate::texture::Texture;
 use crate::transform::Transform;
 use crate::vector::Vec3;
 use crate::vertex::Vertex;
 
-const FRAG_SHADER_COLOR: &str = include_str!("../fragment-color.glsl");
-const FRAG_SHADER_TEXTURE: &str = include_str!("../fragment-texture.glsl");
+const SHADER_DIR: &str = "engine";
+
+const VERTEX_SHADER_FILE: &str = "vertex.glsl";
+const FRAG_SHADER_COLOR_FILE: &str = "fragment-color.glsl";
+const FRAG_SHADER_TEXTURE_FILE: &str = "fragment-texture.glsl";
 
 #[derive(Debug)]
 pub struct Object
@@ -114,16 +122,72 @@ impl Builder
     /// - Linking the shader program fails
     pub fn build(self, id: Id) -> Result<Object, Error>
     {
+        let vertex_shader_file = Path::new(SHADER_DIR).join(VERTEX_SHADER_FILE);
+
+        let fragment_shader_color_file =
+            Path::new(SHADER_DIR).join(FRAG_SHADER_COLOR_FILE);
+
+        let fragment_shader_texture_file =
+            Path::new(SHADER_DIR).join(FRAG_SHADER_TEXTURE_FILE);
+
+        let vertex_shader_content =
+            read_to_string(&vertex_shader_file).map_err(|err| {
+                Error::ReadShaderFailed {
+                    source: err,
+                    shader_file: vertex_shader_file.clone(),
+                }
+            })?;
+
+        let fragment_shader_color_content = read_to_string(&fragment_shader_color_file)
+            .map_err(|err| Error::ReadShaderFailed {
+            source: err,
+            shader_file: fragment_shader_color_file.clone(),
+        })?;
+
+        let fragment_shader_texture_content =
+            read_to_string(&fragment_shader_texture_file).map_err(|err| {
+                Error::ReadShaderFailed {
+                    source: err,
+                    shader_file: fragment_shader_texture_file.clone(),
+                }
+            })?;
+
+        let shader_preprocessor = ShaderPreprocessor::new(PathBuf::from(SHADER_DIR));
+
+        let vertex_shader_content_preprocessed = shader_preprocessor
+            .preprocess(vertex_shader_content, &vertex_shader_file)
+            .map_err(|err| Error::PreprocessShaderFailed {
+                source: err,
+                shader_file: vertex_shader_file,
+            })?;
+
+        let frag_shader_color_content_preprocessed = shader_preprocessor
+            .preprocess(fragment_shader_color_content, &fragment_shader_color_file)
+            .map_err(|err| Error::PreprocessShaderFailed {
+                source: err,
+                shader_file: fragment_shader_color_file,
+            })?;
+
+        let frag_shader_texture_content_preprocessed = shader_preprocessor
+            .preprocess(
+                fragment_shader_texture_content,
+                &fragment_shader_texture_file,
+            )
+            .map_err(|err| Error::PreprocessShaderFailed {
+                source: err,
+                shader_file: fragment_shader_texture_file,
+            })?;
+
         let vertex_shader =
-            Self::create_shader(ShaderKind::Vertex, include_str!("../vertex.glsl"))
+            Self::create_shader(ShaderKind::Vertex, &vertex_shader_content_preprocessed)
                 .map_err(Error::CreateVertexShaderFailed)?;
 
         let fragment_shader = Self::create_shader(
             ShaderKind::Fragment,
             if self.texture.is_some() {
-                FRAG_SHADER_TEXTURE
+                &frag_shader_texture_content_preprocessed
             } else {
-                FRAG_SHADER_COLOR
+                &frag_shader_color_content_preprocessed
             },
         )
         .map_err(Error::CreateFragmentShaderFailed)?;
@@ -173,6 +237,22 @@ pub enum Error
 
     #[error("Failed to link shader program")]
     LinkShaderProgramFailed(#[source] ShaderError),
+
+    #[error("Failed to read shader {}", shader_file.display())]
+    ReadShaderFailed
+    {
+        #[source]
+        source: IoError,
+        shader_file: PathBuf,
+    },
+
+    #[error("Failed to preprocess shader {}", shader_file.display())]
+    PreprocessShaderFailed
+    {
+        #[source]
+        source: ShaderPreprocessorError,
+        shader_file: PathBuf,
+    },
 }
 
 /// Object ID.
diff --git a/engine/src/shader_preprocessor.rs b/engine/src/shader_preprocessor.rs
new file mode 100644
index 0000000..479c5b1
--- /dev/null
+++ b/engine/src/shader_preprocessor.rs
@@ -0,0 +1,606 @@
+use std::path::{Path, PathBuf};
+use std::string::FromUtf8Error;
+
+const PREINCLUDE_DIRECTIVE: &str = "#preinclude";
+
+/// Preprocessor for shaders written in the OpenGL Shading Language.
+pub struct ShaderPreprocessor
+{
+    read_file: fn(&Path) -> Result<Vec<u8>, std::io::Error>,
+    base_dir: PathBuf,
+}
+
+impl ShaderPreprocessor
+{
+    pub fn new(base_dir: PathBuf) -> Self
+    {
+        Self {
+            read_file: |path| std::fs::read(path),
+            base_dir,
+        }
+    }
+
+    pub fn preprocess(
+        &self,
+        shader_content: String,
+        shader_file_path: &Path,
+    ) -> Result<String, Error>
+    {
+        let mut preincludes = shader_content
+            .match_indices(PREINCLUDE_DIRECTIVE)
+            .peekable();
+
+        if preincludes.peek().is_none() {
+            // Shader content contains no preincludes
+            return Ok(shader_content);
+        };
+
+        let mut preprocessed = shader_content.clone();
+
+        let mut curr = shader_content.find(PREINCLUDE_DIRECTIVE);
+
+        let mut last_start = 0;
+        let mut span_line_offset = 0;
+
+        while let Some(preinclude_start) = curr {
+            let replacement_job = self.handle_preinclude(
+                &preprocessed,
+                shader_file_path,
+                preinclude_start,
+                span_line_offset,
+            )?;
+
+            let path = replacement_job.path.clone();
+
+            let mut included = String::from_utf8(
+                (self.read_file)(&self.base_dir.join(replacement_job.path.clone()))
+                    .map_err(|err| Error::ReadIncludedShaderFailed {
+                        source: err,
+                        path: replacement_job.path.clone(),
+                    })?,
+            )
+            .map_err(|err| Error::IncludedShaderInvalidUtf8 {
+                source: err,
+                path: path.clone(),
+            })?;
+
+            let mut included_lines = included.lines();
+
+            if let Some(first_line) = included_lines.next() {
+                if first_line.starts_with("#version") {
+                    included = included_lines.collect();
+                }
+            }
+
+            let included_preprocessed =
+                self.preprocess(included, &replacement_job.path)?;
+
+            let start = replacement_job.start_index;
+            let end = replacement_job.end_index;
+
+            preprocessed.replace_range(start..end, &included_preprocessed);
+
+            curr = preprocessed[last_start + 1..]
+                .find(PREINCLUDE_DIRECTIVE)
+                .map(|index| index + 1);
+
+            last_start = preinclude_start + included_preprocessed.len();
+
+            span_line_offset += included_preprocessed.lines().count();
+        }
+
+        Ok(preprocessed)
+    }
+
+    fn handle_preinclude(
+        &self,
+        shader_content: &str,
+        shader_file_path: &Path,
+        preinclude_start_index: usize,
+        span_line_offset: usize,
+    ) -> Result<ReplacementJob, Error>
+    {
+        let expect_token = |token: char, index: usize| {
+            let token_found = shader_content.chars().nth(index).ok_or_else(|| {
+                Error::ExpectedToken {
+                    expected: token,
+                    span: Span::new(
+                        shader_content,
+                        &self.base_dir.join(shader_file_path),
+                        index,
+                        span_line_offset,
+                        preinclude_start_index,
+                    ),
+                }
+            })?;
+
+            if token_found != token {
+                return Err(Error::InvalidToken {
+                    expected: token,
+                    found: token_found,
+                    span: Span::new(
+                        shader_content,
+                        &self.base_dir.join(shader_file_path),
+                        index,
+                        span_line_offset,
+                        preinclude_start_index,
+                    ),
+                });
+            }
+
+            Ok(())
+        };
+
+        let space_index = preinclude_start_index + PREINCLUDE_DIRECTIVE.len();
+        let quote_open_index = space_index + 1;
+
+        expect_token(' ', space_index)?;
+        expect_token('"', quote_open_index)?;
+
+        let buf = shader_content[quote_open_index + 1..]
+            .chars()
+            .take_while(|character| *character != '"')
+            .map(|character| character as u8)
+            .collect::<Vec<_>>();
+
+        if buf.is_empty() {
+            return Err(Error::ExpectedToken {
+                expected: '"',
+                span: Span::new(
+                    shader_content,
+                    &self.base_dir.join(shader_file_path),
+                    shader_content.len() - 1,
+                    span_line_offset,
+                    preinclude_start_index,
+                ),
+            });
+        }
+
+        let path_len = buf.len();
+
+        let path = PathBuf::from(String::from_utf8(buf).map_err(|err| {
+            Error::PreincludePathInvalidUtf8 {
+                source: err,
+                span: Span::new(
+                    shader_content,
+                    &self.base_dir.join(shader_file_path),
+                    quote_open_index + 1,
+                    span_line_offset,
+                    preinclude_start_index,
+                ),
+            }
+        })?);
+
+        Ok(ReplacementJob {
+            start_index: preinclude_start_index,
+            end_index: quote_open_index + 1 + path_len + 1,
+            path,
+        })
+    }
+}
+
+struct ReplacementJob
+{
+    start_index: usize,
+    end_index: usize,
+    path: PathBuf,
+}
+
+/// Shader preprocessing error.
+#[derive(Debug, thiserror::Error)]
+pub enum Error
+{
+    #[error(
+        "Invalid token at line {}, column {} of {}. Expected '{}', found '{}'",
+        span.line,
+        span.column,
+        span.file.display(),
+        expected,
+        found
+    )]
+    InvalidToken
+    {
+        expected: char,
+        found: char,
+        span: Span,
+    },
+
+    #[error(
+        "Expected token '{}' at line {}, column {} of {}. Found eof",
+        expected,
+        span.line,
+        span.column,
+        span.file.display(),
+    )]
+    ExpectedToken
+    {
+        expected: char, span: Span
+    },
+
+    #[error(
+        "Preinclude path at line {}, column {} of {} is invalid UTF-8",
+        span.line,
+        span.column,
+        span.file.display(),
+    )]
+    PreincludePathInvalidUtf8
+    {
+        #[source]
+        source: FromUtf8Error,
+        span: Span,
+    },
+
+    #[error("Failed to read included shader")]
+    ReadIncludedShaderFailed
+    {
+        #[source]
+        source: std::io::Error,
+        path: PathBuf,
+    },
+
+    #[error("Included shader is not valid UTF-8")]
+    IncludedShaderInvalidUtf8
+    {
+        #[source]
+        source: FromUtf8Error,
+        path: PathBuf,
+    },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Span
+{
+    line: usize,
+    column: usize,
+    file: PathBuf,
+}
+
+impl Span
+{
+    fn new(
+        file_content: &str,
+        file_path: &Path,
+        char_index: usize,
+        line_offset: usize,
+        line_start_index: usize,
+    ) -> Self
+    {
+        let line = find_line_of_index(file_content, char_index) + 1
+            - line_offset.saturating_sub(1);
+
+        Self {
+            line,
+            column: char_index - line_start_index + 1,
+            file: file_path.to_path_buf(),
+        }
+    }
+}
+
+fn find_line_of_index(text: &str, index: usize) -> usize
+{
+    text.chars()
+        .take(index + 1)
+        .enumerate()
+        .filter(|(_, character)| *character == '\n')
+        .count()
+}
+
+#[cfg(test)]
+mod tests
+{
+    use std::ffi::OsStr;
+    use std::path::{Path, PathBuf};
+
+    use super::{Error, ShaderPreprocessor};
+
+    #[test]
+    fn preprocess_no_directives_is_same()
+    {
+        assert_eq!(
+            ShaderPreprocessor {
+                read_file: |_| { unreachable!() },
+                base_dir: PathBuf::new()
+            }
+            .preprocess("#version 330 core\n".to_string(), Path::new("foo.glsl"),)
+            .unwrap(),
+            "#version 330 core\n".to_string()
+        );
+    }
+
+    #[test]
+    fn preprocess_with_directives_works()
+    {
+        assert_eq!(
+            ShaderPreprocessor {
+                read_file: |_| { Ok(b"out vec4 FragColor;".to_vec()) },
+                base_dir: PathBuf::new()
+            }
+            .preprocess(
+                concat!(
+                    "#version 330 core\n",
+                    "\n",
+                    "#preinclude \"foo.glsl\"\n",
+                    "\n",
+                    "void main() {}",
+                )
+                .to_string(),
+                Path::new("foo.glsl"),
+            )
+            .unwrap(),
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "out vec4 FragColor;\n",
+                "\n",
+                "void main() {}",
+            )
+        );
+
+        assert_eq!(
+            ShaderPreprocessor {
+                read_file: |_| { Ok(b"out vec4 FragColor;".to_vec()) },
+                base_dir: PathBuf::new()
+            }
+            .preprocess(
+                concat!(
+                    "#version 330 core\n",
+                    "\n",
+                    "#preinclude \"bar.glsl\"\n",
+                    "\n",
+                    "in vec3 in_frag_color;\n",
+                    "\n",
+                    "void main() {}",
+                )
+                .to_string(),
+                Path::new("foo.glsl"),
+            )
+            .unwrap(),
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "out vec4 FragColor;\n",
+                "\n",
+                "in vec3 in_frag_color;\n",
+                "\n",
+                "void main() {}",
+            )
+        );
+
+        assert_eq!(
+            ShaderPreprocessor {
+                read_file: |path| {
+                    if path == OsStr::new("bar.glsl") {
+                        Ok(b"out vec4 FragColor;".to_vec())
+                    } else {
+                        Ok(concat!(
+                            "uniform sampler2D input_texture;\n",
+                            "in vec2 in_texture_coords;"
+                        )
+                        .as_bytes()
+                        .to_vec())
+                    }
+                },
+                base_dir: PathBuf::new()
+            }
+            .preprocess(
+                concat!(
+                    "#version 330 core\n",
+                    "\n",
+                    "#preinclude \"bar.glsl\"\n",
+                    "\n",
+                    "in vec3 in_frag_color;\n",
+                    "\n",
+                    "#preinclude \"foo.glsl\"\n",
+                    "\n",
+                    "void main() {}",
+                )
+                .to_string(),
+                Path::new("foo.glsl"),
+            )
+            .unwrap(),
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "out vec4 FragColor;\n",
+                "\n",
+                "in vec3 in_frag_color;\n",
+                "\n",
+                "uniform sampler2D input_texture;\n",
+                "in vec2 in_texture_coords;\n",
+                "\n",
+                "void main() {}",
+            )
+        );
+    }
+
+    #[test]
+    fn preprocess_invalid_shader_does_not_work()
+    {
+        let path = Path::new("foo.glsl");
+
+        let res = ShaderPreprocessor {
+            read_file: |_| Ok(b"out vec4 FragColor;".to_vec()),
+            base_dir: PathBuf::new(),
+        }
+        .preprocess(
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                // Missing "
+                "#preinclude foo.glsl\"\n",
+                "\n",
+                "void main() {}",
+            )
+            .to_string(),
+            path,
+        );
+
+        let Err(Error::InvalidToken { expected, found, span }) = res else {
+            panic!(
+                "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+            );
+        };
+
+        assert_eq!(expected, '"');
+        assert_eq!(found, 'f');
+        assert_eq!(span.line, 3);
+        assert_eq!(span.column, 13);
+        assert_eq!(span.file, path);
+    }
+
+    #[test]
+    fn preprocess_error_has_correct_span()
+    {
+        let path = Path::new("foo.glsl");
+
+        let res = ShaderPreprocessor {
+            read_file: |path| {
+                if path == OsStr::new("bar.glsl") {
+                    Ok(concat!(
+                        "out vec4 FragColor;\n",
+                        "in vec2 in_texture_coords;\n",
+                        "in float foo;"
+                    )
+                    .as_bytes()
+                    .to_vec())
+                } else {
+                    Ok(b"uniform sampler2D input_texture;".to_vec())
+                }
+            },
+            base_dir: PathBuf::new(),
+        }
+        .preprocess(
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "#preinclude \"bar.glsl\"\n",
+                "\n",
+                "in vec3 in_frag_color;\n",
+                "\n",
+                "#preinclude\"foo.glsl\"\n",
+                "\n",
+                "void main() {}",
+            )
+            .to_string(),
+            Path::new("foo.glsl"),
+        );
+
+        let Err(Error::InvalidToken { expected, found, span }) = res else {
+            panic!(
+                "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+            );
+        };
+
+        assert_eq!(expected, ' ');
+        assert_eq!(found, '"');
+        assert_eq!(span.line, 7);
+        assert_eq!(span.column, 12);
+        assert_eq!(span.file, path);
+    }
+
+    #[test]
+    fn preprocess_included_shader_with_include_works()
+    {
+        assert_eq!(
+            ShaderPreprocessor {
+                read_file: |path| {
+                    if path == OsStr::new("bar.glsl") {
+                        Ok(concat!(
+                            "#preinclude \"foo.glsl\"\n",
+                            "\n",
+                            "out vec4 FragColor;"
+                        )
+                        .as_bytes()
+                        .to_vec())
+                    } else {
+                        Ok(concat!(
+                            "uniform sampler2D input_texture;\n",
+                            "in vec2 in_texture_coords;"
+                        )
+                        .as_bytes()
+                        .to_vec())
+                    }
+                },
+                base_dir: PathBuf::new()
+            }
+            .preprocess(
+                concat!(
+                    "#version 330 core\n",
+                    "\n",
+                    "#preinclude \"bar.glsl\"\n",
+                    "\n",
+                    "in vec3 in_frag_color;\n",
+                    "\n",
+                    "void main() {}",
+                )
+                .to_string(),
+                Path::new("foo.glsl"),
+            )
+            .unwrap(),
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "uniform sampler2D input_texture;\n",
+                "in vec2 in_texture_coords;\n",
+                "\n",
+                "out vec4 FragColor;\n",
+                "\n",
+                "in vec3 in_frag_color;\n",
+                "\n",
+                "void main() {}",
+            )
+        );
+    }
+
+    #[test]
+    fn preprocess_included_shader_with_include_error_span_is_correct()
+    {
+        let res = ShaderPreprocessor {
+            read_file: |path| {
+                if path == OsStr::new("bar.glsl") {
+                    Ok(concat!(
+                        // ' instead of "
+                        "#preinclude 'foo.glsl\"\n",
+                        "\n",
+                        "out vec4 FragColor;"
+                    )
+                    .as_bytes()
+                    .to_vec())
+                } else {
+                    Ok(concat!(
+                        "uniform sampler2D input_texture;\n",
+                        "in vec2 in_texture_coords;"
+                    )
+                    .as_bytes()
+                    .to_vec())
+                }
+            },
+            base_dir: PathBuf::new(),
+        }
+        .preprocess(
+            concat!(
+                "#version 330 core\n",
+                "\n",
+                "#preinclude \"bar.glsl\"\n",
+                "\n",
+                "in vec3 in_frag_color;\n",
+                "\n",
+                "void main() {}",
+            )
+            .to_string(),
+            Path::new("foo.glsl"),
+        );
+
+        let Err(Error::InvalidToken { expected, found, span }) = res else {
+            panic!(
+                "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+            );
+        };
+
+        assert_eq!(expected, '"');
+        assert_eq!(found, '\'');
+        assert_eq!(span.line, 1);
+        assert_eq!(span.column, 13);
+        assert_eq!(span.file, Path::new("bar.glsl"));
+    }
+}