From 7d01458a6ea591f1d6215c6c9b247410807ed60d Mon Sep 17 00:00:00 2001
From: HampusM <hampus@hampusmat.com>
Date: Fri, 10 Nov 2023 20:19:37 +0100
Subject: chore(engine): add shader preprocessor
---
engine/src/lib.rs | 1 +
engine/src/object.rs | 90 +++++-
engine/src/shader_preprocessor.rs | 606 ++++++++++++++++++++++++++++++++++++++
3 files changed, 692 insertions(+), 5 deletions(-)
create mode 100644 engine/src/shader_preprocessor.rs
(limited to 'engine/src')
diff --git a/engine/src/lib.rs b/engine/src/lib.rs
index ae603b0..0eca914 100644
--- a/engine/src/lib.rs
+++ b/engine/src/lib.rs
@@ -15,6 +15,7 @@ mod matrix;
mod opengl;
mod projection;
mod renderer;
+mod shader_preprocessor;
mod transform;
pub mod camera;
diff --git a/engine/src/object.rs b/engine/src/object.rs
index a8cd076..4112bcb 100644
--- a/engine/src/object.rs
+++ b/engine/src/object.rs
@@ -1,3 +1,7 @@
+use std::fs::read_to_string;
+use std::io::Error as IoError;
+use std::path::{Path, PathBuf};
+
use crate::opengl::shader::{
Error as ShaderError,
Kind as ShaderKind,
@@ -5,13 +9,17 @@ use crate::opengl::shader::{
Shader,
};
use crate::renderer::Renderable;
+use crate::shader_preprocessor::{Error as ShaderPreprocessorError, ShaderPreprocessor};
use crate::texture::Texture;
use crate::transform::Transform;
use crate::vector::Vec3;
use crate::vertex::Vertex;
-const FRAG_SHADER_COLOR: &str = include_str!("../fragment-color.glsl");
-const FRAG_SHADER_TEXTURE: &str = include_str!("../fragment-texture.glsl");
+const SHADER_DIR: &str = "engine";
+
+const VERTEX_SHADER_FILE: &str = "vertex.glsl";
+const FRAG_SHADER_COLOR_FILE: &str = "fragment-color.glsl";
+const FRAG_SHADER_TEXTURE_FILE: &str = "fragment-texture.glsl";
#[derive(Debug)]
pub struct Object
@@ -114,16 +122,72 @@ impl Builder
/// - Linking the shader program fails
pub fn build(self, id: Id) -> Result<Object, Error>
{
+ let vertex_shader_file = Path::new(SHADER_DIR).join(VERTEX_SHADER_FILE);
+
+ let fragment_shader_color_file =
+ Path::new(SHADER_DIR).join(FRAG_SHADER_COLOR_FILE);
+
+ let fragment_shader_texture_file =
+ Path::new(SHADER_DIR).join(FRAG_SHADER_TEXTURE_FILE);
+
+ let vertex_shader_content =
+ read_to_string(&vertex_shader_file).map_err(|err| {
+ Error::ReadShaderFailed {
+ source: err,
+ shader_file: vertex_shader_file.clone(),
+ }
+ })?;
+
+ let fragment_shader_color_content = read_to_string(&fragment_shader_color_file)
+ .map_err(|err| Error::ReadShaderFailed {
+ source: err,
+ shader_file: fragment_shader_color_file.clone(),
+ })?;
+
+ let fragment_shader_texture_content =
+ read_to_string(&fragment_shader_texture_file).map_err(|err| {
+ Error::ReadShaderFailed {
+ source: err,
+ shader_file: fragment_shader_texture_file.clone(),
+ }
+ })?;
+
+ let shader_preprocessor = ShaderPreprocessor::new(PathBuf::from(SHADER_DIR));
+
+ let vertex_shader_content_preprocessed = shader_preprocessor
+ .preprocess(vertex_shader_content, &vertex_shader_file)
+ .map_err(|err| Error::PreprocessShaderFailed {
+ source: err,
+ shader_file: vertex_shader_file,
+ })?;
+
+ let frag_shader_color_content_preprocessed = shader_preprocessor
+ .preprocess(fragment_shader_color_content, &fragment_shader_color_file)
+ .map_err(|err| Error::PreprocessShaderFailed {
+ source: err,
+ shader_file: fragment_shader_color_file,
+ })?;
+
+ let frag_shader_texture_content_preprocessed = shader_preprocessor
+ .preprocess(
+ fragment_shader_texture_content,
+ &fragment_shader_texture_file,
+ )
+ .map_err(|err| Error::PreprocessShaderFailed {
+ source: err,
+ shader_file: fragment_shader_texture_file,
+ })?;
+
let vertex_shader =
- Self::create_shader(ShaderKind::Vertex, include_str!("../vertex.glsl"))
+ Self::create_shader(ShaderKind::Vertex, &vertex_shader_content_preprocessed)
.map_err(Error::CreateVertexShaderFailed)?;
let fragment_shader = Self::create_shader(
ShaderKind::Fragment,
if self.texture.is_some() {
- FRAG_SHADER_TEXTURE
+ &frag_shader_texture_content_preprocessed
} else {
- FRAG_SHADER_COLOR
+ &frag_shader_color_content_preprocessed
},
)
.map_err(Error::CreateFragmentShaderFailed)?;
@@ -173,6 +237,22 @@ pub enum Error
#[error("Failed to link shader program")]
LinkShaderProgramFailed(#[source] ShaderError),
+
+ #[error("Failed to read shader {}", shader_file.display())]
+ ReadShaderFailed
+ {
+ #[source]
+ source: IoError,
+ shader_file: PathBuf,
+ },
+
+ #[error("Failed to preprocess shader {}", shader_file.display())]
+ PreprocessShaderFailed
+ {
+ #[source]
+ source: ShaderPreprocessorError,
+ shader_file: PathBuf,
+ },
}
/// Object ID.
diff --git a/engine/src/shader_preprocessor.rs b/engine/src/shader_preprocessor.rs
new file mode 100644
index 0000000..479c5b1
--- /dev/null
+++ b/engine/src/shader_preprocessor.rs
@@ -0,0 +1,606 @@
+use std::path::{Path, PathBuf};
+use std::string::FromUtf8Error;
+
+const PREINCLUDE_DIRECTIVE: &str = "#preinclude";
+
+/// Preprocessor for shaders written in the OpenGL Shading Language.
+pub struct ShaderPreprocessor
+{
+ read_file: fn(&Path) -> Result<Vec<u8>, std::io::Error>,
+ base_dir: PathBuf,
+}
+
+impl ShaderPreprocessor
+{
+ pub fn new(base_dir: PathBuf) -> Self
+ {
+ Self {
+ read_file: |path| std::fs::read(path),
+ base_dir,
+ }
+ }
+
+ pub fn preprocess(
+ &self,
+ shader_content: String,
+ shader_file_path: &Path,
+ ) -> Result<String, Error>
+ {
+ let mut preincludes = shader_content
+ .match_indices(PREINCLUDE_DIRECTIVE)
+ .peekable();
+
+ if preincludes.peek().is_none() {
+ // Shader content contains no preincludes
+ return Ok(shader_content);
+ };
+
+ let mut preprocessed = shader_content.clone();
+
+ let mut curr = shader_content.find(PREINCLUDE_DIRECTIVE);
+
+ let mut last_start = 0;
+ let mut span_line_offset = 0;
+
+ while let Some(preinclude_start) = curr {
+ let replacement_job = self.handle_preinclude(
+ &preprocessed,
+ shader_file_path,
+ preinclude_start,
+ span_line_offset,
+ )?;
+
+ let path = replacement_job.path.clone();
+
+ let mut included = String::from_utf8(
+ (self.read_file)(&self.base_dir.join(replacement_job.path.clone()))
+ .map_err(|err| Error::ReadIncludedShaderFailed {
+ source: err,
+ path: replacement_job.path.clone(),
+ })?,
+ )
+ .map_err(|err| Error::IncludedShaderInvalidUtf8 {
+ source: err,
+ path: path.clone(),
+ })?;
+
+ let mut included_lines = included.lines();
+
+ if let Some(first_line) = included_lines.next() {
+ if first_line.starts_with("#version") {
+ included = included_lines.collect();
+ }
+ }
+
+ let included_preprocessed =
+ self.preprocess(included, &replacement_job.path)?;
+
+ let start = replacement_job.start_index;
+ let end = replacement_job.end_index;
+
+ preprocessed.replace_range(start..end, &included_preprocessed);
+
+ curr = preprocessed[last_start + 1..]
+ .find(PREINCLUDE_DIRECTIVE)
+ .map(|index| index + 1);
+
+ last_start = preinclude_start + included_preprocessed.len();
+
+ span_line_offset += included_preprocessed.lines().count();
+ }
+
+ Ok(preprocessed)
+ }
+
+ fn handle_preinclude(
+ &self,
+ shader_content: &str,
+ shader_file_path: &Path,
+ preinclude_start_index: usize,
+ span_line_offset: usize,
+ ) -> Result<ReplacementJob, Error>
+ {
+ let expect_token = |token: char, index: usize| {
+ let token_found = shader_content.chars().nth(index).ok_or_else(|| {
+ Error::ExpectedToken {
+ expected: token,
+ span: Span::new(
+ shader_content,
+ &self.base_dir.join(shader_file_path),
+ index,
+ span_line_offset,
+ preinclude_start_index,
+ ),
+ }
+ })?;
+
+ if token_found != token {
+ return Err(Error::InvalidToken {
+ expected: token,
+ found: token_found,
+ span: Span::new(
+ shader_content,
+ &self.base_dir.join(shader_file_path),
+ index,
+ span_line_offset,
+ preinclude_start_index,
+ ),
+ });
+ }
+
+ Ok(())
+ };
+
+ let space_index = preinclude_start_index + PREINCLUDE_DIRECTIVE.len();
+ let quote_open_index = space_index + 1;
+
+ expect_token(' ', space_index)?;
+ expect_token('"', quote_open_index)?;
+
+ let buf = shader_content[quote_open_index + 1..]
+ .chars()
+ .take_while(|character| *character != '"')
+ .map(|character| character as u8)
+ .collect::<Vec<_>>();
+
+ if buf.is_empty() {
+ return Err(Error::ExpectedToken {
+ expected: '"',
+ span: Span::new(
+ shader_content,
+ &self.base_dir.join(shader_file_path),
+ shader_content.len() - 1,
+ span_line_offset,
+ preinclude_start_index,
+ ),
+ });
+ }
+
+ let path_len = buf.len();
+
+ let path = PathBuf::from(String::from_utf8(buf).map_err(|err| {
+ Error::PreincludePathInvalidUtf8 {
+ source: err,
+ span: Span::new(
+ shader_content,
+ &self.base_dir.join(shader_file_path),
+ quote_open_index + 1,
+ span_line_offset,
+ preinclude_start_index,
+ ),
+ }
+ })?);
+
+ Ok(ReplacementJob {
+ start_index: preinclude_start_index,
+ end_index: quote_open_index + 1 + path_len + 1,
+ path,
+ })
+ }
+}
+
+struct ReplacementJob
+{
+ start_index: usize,
+ end_index: usize,
+ path: PathBuf,
+}
+
+/// Shader preprocessing error.
+#[derive(Debug, thiserror::Error)]
+pub enum Error
+{
+ #[error(
+ "Invalid token at line {}, column {} of {}. Expected '{}', found '{}'",
+ span.line,
+ span.column,
+ span.file.display(),
+ expected,
+ found
+ )]
+ InvalidToken
+ {
+ expected: char,
+ found: char,
+ span: Span,
+ },
+
+ #[error(
+ "Expected token '{}' at line {}, column {} of {}. Found eof",
+ expected,
+ span.line,
+ span.column,
+ span.file.display(),
+ )]
+ ExpectedToken
+ {
+ expected: char, span: Span
+ },
+
+ #[error(
+ "Preinclude path at line {}, column {} of {} is invalid UTF-8",
+ span.line,
+ span.column,
+ span.file.display(),
+ )]
+ PreincludePathInvalidUtf8
+ {
+ #[source]
+ source: FromUtf8Error,
+ span: Span,
+ },
+
+ #[error("Failed to read included shader")]
+ ReadIncludedShaderFailed
+ {
+ #[source]
+ source: std::io::Error,
+ path: PathBuf,
+ },
+
+ #[error("Included shader is not valid UTF-8")]
+ IncludedShaderInvalidUtf8
+ {
+ #[source]
+ source: FromUtf8Error,
+ path: PathBuf,
+ },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Span
+{
+ line: usize,
+ column: usize,
+ file: PathBuf,
+}
+
+impl Span
+{
+ fn new(
+ file_content: &str,
+ file_path: &Path,
+ char_index: usize,
+ line_offset: usize,
+ line_start_index: usize,
+ ) -> Self
+ {
+ let line = find_line_of_index(file_content, char_index) + 1
+ - line_offset.saturating_sub(1);
+
+ Self {
+ line,
+ column: char_index - line_start_index + 1,
+ file: file_path.to_path_buf(),
+ }
+ }
+}
+
+fn find_line_of_index(text: &str, index: usize) -> usize
+{
+ text.chars()
+ .take(index + 1)
+ .enumerate()
+ .filter(|(_, character)| *character == '\n')
+ .count()
+}
+
+#[cfg(test)]
+mod tests
+{
+ use std::ffi::OsStr;
+ use std::path::{Path, PathBuf};
+
+ use super::{Error, ShaderPreprocessor};
+
+ #[test]
+ fn preprocess_no_directives_is_same()
+ {
+ assert_eq!(
+ ShaderPreprocessor {
+ read_file: |_| { unreachable!() },
+ base_dir: PathBuf::new()
+ }
+ .preprocess("#version 330 core\n".to_string(), Path::new("foo.glsl"),)
+ .unwrap(),
+ "#version 330 core\n".to_string()
+ );
+ }
+
+ #[test]
+ fn preprocess_with_directives_works()
+ {
+ assert_eq!(
+ ShaderPreprocessor {
+ read_file: |_| { Ok(b"out vec4 FragColor;".to_vec()) },
+ base_dir: PathBuf::new()
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"foo.glsl\"\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ )
+ .unwrap(),
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "out vec4 FragColor;\n",
+ "\n",
+ "void main() {}",
+ )
+ );
+
+ assert_eq!(
+ ShaderPreprocessor {
+ read_file: |_| { Ok(b"out vec4 FragColor;".to_vec()) },
+ base_dir: PathBuf::new()
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"bar.glsl\"\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ )
+ .unwrap(),
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "out vec4 FragColor;\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "void main() {}",
+ )
+ );
+
+ assert_eq!(
+ ShaderPreprocessor {
+ read_file: |path| {
+ if path == OsStr::new("bar.glsl") {
+ Ok(b"out vec4 FragColor;".to_vec())
+ } else {
+ Ok(concat!(
+ "uniform sampler2D input_texture;\n",
+ "in vec2 in_texture_coords;"
+ )
+ .as_bytes()
+ .to_vec())
+ }
+ },
+ base_dir: PathBuf::new()
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"bar.glsl\"\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "#preinclude \"foo.glsl\"\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ )
+ .unwrap(),
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "out vec4 FragColor;\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "uniform sampler2D input_texture;\n",
+ "in vec2 in_texture_coords;\n",
+ "\n",
+ "void main() {}",
+ )
+ );
+ }
+
+ #[test]
+ fn preprocess_invalid_shader_does_not_work()
+ {
+ let path = Path::new("foo.glsl");
+
+ let res = ShaderPreprocessor {
+ read_file: |_| Ok(b"out vec4 FragColor;".to_vec()),
+ base_dir: PathBuf::new(),
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ // Missing "
+ "#preinclude foo.glsl\"\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ path,
+ );
+
+ let Err(Error::InvalidToken { expected, found, span }) = res else {
+ panic!(
+ "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+ );
+ };
+
+ assert_eq!(expected, '"');
+ assert_eq!(found, 'f');
+ assert_eq!(span.line, 3);
+ assert_eq!(span.column, 13);
+ assert_eq!(span.file, path);
+ }
+
+ #[test]
+ fn preprocess_error_has_correct_span()
+ {
+ let path = Path::new("foo.glsl");
+
+ let res = ShaderPreprocessor {
+ read_file: |path| {
+ if path == OsStr::new("bar.glsl") {
+ Ok(concat!(
+ "out vec4 FragColor;\n",
+ "in vec2 in_texture_coords;\n",
+ "in float foo;"
+ )
+ .as_bytes()
+ .to_vec())
+ } else {
+ Ok(b"uniform sampler2D input_texture;".to_vec())
+ }
+ },
+ base_dir: PathBuf::new(),
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"bar.glsl\"\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "#preinclude\"foo.glsl\"\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ );
+
+ let Err(Error::InvalidToken { expected, found, span }) = res else {
+ panic!(
+ "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+ );
+ };
+
+ assert_eq!(expected, ' ');
+ assert_eq!(found, '"');
+ assert_eq!(span.line, 7);
+ assert_eq!(span.column, 12);
+ assert_eq!(span.file, path);
+ }
+
+ #[test]
+ fn preprocess_included_shader_with_include_works()
+ {
+ assert_eq!(
+ ShaderPreprocessor {
+ read_file: |path| {
+ if path == OsStr::new("bar.glsl") {
+ Ok(concat!(
+ "#preinclude \"foo.glsl\"\n",
+ "\n",
+ "out vec4 FragColor;"
+ )
+ .as_bytes()
+ .to_vec())
+ } else {
+ Ok(concat!(
+ "uniform sampler2D input_texture;\n",
+ "in vec2 in_texture_coords;"
+ )
+ .as_bytes()
+ .to_vec())
+ }
+ },
+ base_dir: PathBuf::new()
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"bar.glsl\"\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ )
+ .unwrap(),
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "uniform sampler2D input_texture;\n",
+ "in vec2 in_texture_coords;\n",
+ "\n",
+ "out vec4 FragColor;\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "void main() {}",
+ )
+ );
+ }
+
+ #[test]
+ fn preprocess_included_shader_with_include_error_span_is_correct()
+ {
+ let res = ShaderPreprocessor {
+ read_file: |path| {
+ if path == OsStr::new("bar.glsl") {
+ Ok(concat!(
+ // ' instead of "
+ "#preinclude 'foo.glsl\"\n",
+ "\n",
+ "out vec4 FragColor;"
+ )
+ .as_bytes()
+ .to_vec())
+ } else {
+ Ok(concat!(
+ "uniform sampler2D input_texture;\n",
+ "in vec2 in_texture_coords;"
+ )
+ .as_bytes()
+ .to_vec())
+ }
+ },
+ base_dir: PathBuf::new(),
+ }
+ .preprocess(
+ concat!(
+ "#version 330 core\n",
+ "\n",
+ "#preinclude \"bar.glsl\"\n",
+ "\n",
+ "in vec3 in_frag_color;\n",
+ "\n",
+ "void main() {}",
+ )
+ .to_string(),
+ Path::new("foo.glsl"),
+ );
+
+ let Err(Error::InvalidToken { expected, found, span }) = res else {
+ panic!(
+ "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}"
+ );
+ };
+
+ assert_eq!(expected, '"');
+ assert_eq!(found, '\'');
+ assert_eq!(span.line, 1);
+ assert_eq!(span.column, 13);
+ assert_eq!(span.file, Path::new("bar.glsl"));
+ }
+}
--
cgit v1.2.3-18-g5258