From 999264a46f9a545771c6710b0893ca32cf6e7ee3 Mon Sep 17 00:00:00 2001 From: HampusM Date: Sat, 23 Nov 2024 20:20:24 +0100 Subject: refactor(engine): move & improve glsl preprocessing --- engine/src/opengl/glsl.rs | 616 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 616 insertions(+) create mode 100644 engine/src/opengl/glsl.rs (limited to 'engine/src/opengl/glsl.rs') diff --git a/engine/src/opengl/glsl.rs b/engine/src/opengl/glsl.rs new file mode 100644 index 0000000..6fd5638 --- /dev/null +++ b/engine/src/opengl/glsl.rs @@ -0,0 +1,616 @@ +use std::borrow::Cow; +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +use std::string::FromUtf8Error; + +const PREINCLUDE_DIRECTIVE: &str = "#preinclude"; + +pub fn preprocess<'content>( + shader_content: impl Into>, + read_file: &impl Fn(&Path) -> Result, std::io::Error>, +) -> Result, PreprocessingError> +{ + do_preprocess(shader_content, SpanPath::Original, read_file) +} + +fn do_preprocess<'content>( + shader_content: impl Into>, + shader_path: SpanPath<'_>, + read_file: &impl Fn(&Path) -> Result, std::io::Error>, +) -> Result, PreprocessingError> +{ + let shader_content = shader_content.into(); + + let mut preincludes = shader_content + .match_indices(PREINCLUDE_DIRECTIVE) + .peekable(); + + if preincludes.peek().is_none() { + // Shader content contains no preincludes + return Ok(shader_content.into()); + }; + + let mut preprocessed = shader_content.to_string(); + + let mut curr = shader_content.find(PREINCLUDE_DIRECTIVE); + + let mut last_start = 0; + let mut span_line_offset = 0; + + while let Some(preinclude_start) = curr { + let replacement_job = handle_preinclude( + &preprocessed, + &shader_path, + preinclude_start, + span_line_offset, + )?; + + let path = replacement_job.path.clone(); + + let mut included = + String::from_utf8(read_file(&replacement_job.path).map_err(|err| { + PreprocessingError::ReadIncludedShaderFailed { + source: err, + path: replacement_job.path.clone(), + } + })?) + .map_err(|err| { + PreprocessingError::IncludedShaderInvalidUtf8 { + source: err, + path: path.clone(), + } + })?; + + if let Some(first_line) = included.lines().next() { + if first_line.starts_with("#version") { + included = included + .chars() + .skip_while(|character| *character != '\n') + .collect(); + } + } + + let included_preprocessed = do_preprocess( + &included, + SpanPath::Path(replacement_job.path.as_path().into()), + read_file, + )?; + + let start = replacement_job.start_index; + let end = replacement_job.end_index; + + preprocessed.replace_range(start..end, &included_preprocessed); + + curr = preprocessed[last_start + 1..] + .find(PREINCLUDE_DIRECTIVE) + .map(|index| index + 1); + + last_start = preinclude_start + included_preprocessed.len(); + + span_line_offset += included_preprocessed.lines().count(); + } + + Ok(preprocessed.into()) +} + +fn handle_preinclude( + shader_content: &str, + shader_path: &SpanPath<'_>, + preinclude_start_index: usize, + span_line_offset: usize, +) -> Result +{ + let expect_token = |token: char, index: usize| { + let token_found = shader_content.chars().nth(index).ok_or_else(|| { + PreprocessingError::ExpectedToken { + expected: token, + span: Span::new( + shader_content, + shader_path.to_owned(), + index, + span_line_offset, + preinclude_start_index, + ), + } + })?; + + if token_found != token { + return Err(PreprocessingError::InvalidToken { + expected: token, + found: token_found, + span: Span::new( + shader_content, + shader_path.to_owned(), + index, + span_line_offset, + preinclude_start_index, + ), + }); + } + + Ok(()) + }; + + let space_index = preinclude_start_index + PREINCLUDE_DIRECTIVE.len(); + let quote_open_index = space_index + 1; + + expect_token(' ', space_index)?; + expect_token('"', quote_open_index)?; + + let buf = shader_content[quote_open_index + 1..] + .chars() + .take_while(|character| *character != '"') + .map(|character| character as u8) + .collect::>(); + + if buf.is_empty() { + return Err(PreprocessingError::ExpectedToken { + expected: '"', + span: Span::new( + shader_content, + shader_path.to_owned(), + shader_content.len() - 1, + span_line_offset, + preinclude_start_index, + ), + }); + } + + let path_len = buf.len(); + + let path = PathBuf::from(String::from_utf8(buf).map_err(|err| { + PreprocessingError::PreincludePathInvalidUtf8 { + source: err, + span: Span::new( + shader_content, + shader_path.to_owned(), + quote_open_index + 1, + span_line_offset, + preinclude_start_index, + ), + } + })?); + + Ok(ReplacementJob { + start_index: preinclude_start_index, + end_index: quote_open_index + 1 + path_len + 1, + path, + }) +} + +struct ReplacementJob +{ + start_index: usize, + end_index: usize, + path: PathBuf, +} + +#[derive(Debug, thiserror::Error)] +pub enum PreprocessingError +{ + #[error( + "Invalid token at line {}, column {} of {}. Expected '{}', found '{}'", + span.line, + span.column, + span.path, + expected, + found + )] + InvalidToken + { + expected: char, + found: char, + span: Span, + }, + + #[error( + "Expected token '{}' at line {}, column {} of {}. Found eof", + expected, + span.line, + span.column, + span.path + )] + ExpectedToken + { + expected: char, span: Span + }, + + #[error( + "Preinclude path at line {}, column {} of {} is invalid UTF-8", + span.line, + span.column, + span.path + )] + PreincludePathInvalidUtf8 + { + #[source] + source: FromUtf8Error, + span: Span, + }, + + #[error("Failed to read included shader")] + ReadIncludedShaderFailed + { + #[source] + source: std::io::Error, + path: PathBuf, + }, + + #[error("Included shader is not valid UTF-8")] + IncludedShaderInvalidUtf8 + { + #[source] + source: FromUtf8Error, + path: PathBuf, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct Span +{ + pub line: usize, + pub column: usize, + pub path: SpanPath<'static>, +} + +impl Span +{ + fn new( + file_content: &str, + path: SpanPath<'static>, + char_index: usize, + line_offset: usize, + line_start_index: usize, + ) -> Self + { + let line = find_line_of_index(file_content, char_index) + 1 + - line_offset.saturating_sub(1); + + Self { + line, + column: char_index - line_start_index + 1, + path, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpanPath<'a> +{ + Original, + Path(Cow<'a, Path>), +} + +impl<'a> SpanPath<'a> +{ + fn to_owned(&self) -> SpanPath<'static> + { + match self { + Self::Original => SpanPath::Original, + Self::Path(path) => SpanPath::Path(Cow::Owned(path.to_path_buf().into())), + } + } +} + +impl<'a> Display for SpanPath<'a> +{ + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result + { + match self { + Self::Original => write!(formatter, "original file"), + Self::Path(path) => write!(formatter, "file {}", path.display()), + } + } +} + +impl<'a, PathLike> PartialEq for SpanPath<'a> +where + PathLike: AsRef, +{ + fn eq(&self, other: &PathLike) -> bool + { + match self { + Self::Original => false, + Self::Path(path) => path == other.as_ref(), + } + } +} + +fn find_line_of_index(text: &str, index: usize) -> usize +{ + text.chars() + .take(index + 1) + .enumerate() + .filter(|(_, character)| *character == '\n') + .count() +} + +#[cfg(test)] +mod tests +{ + use std::ffi::OsStr; + use std::path::Path; + + use super::{preprocess, PreprocessingError}; + use crate::opengl::glsl::SpanPath; + + #[test] + fn preprocess_no_directives_is_same() + { + assert_eq!( + preprocess("#version 330 core\n", &|_| { unreachable!() }).unwrap(), + "#version 330 core\n" + ); + } + + #[test] + fn preprocess_with_directives_works() + { + assert_eq!( + preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"foo.glsl\"\n", + "\n", + "void main() {}", + ), + &|_| { Ok(b"out vec4 FragColor;".to_vec()) } + ) + .unwrap(), + concat!( + "#version 330 core\n", + "\n", + "out vec4 FragColor;\n", + "\n", + "void main() {}", + ) + ); + + assert_eq!( + preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"bar.glsl\"\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ), + &|_| { Ok(b"out vec4 FragColor;".to_vec()) } + ) + .unwrap(), + concat!( + "#version 330 core\n", + "\n", + "out vec4 FragColor;\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ) + ); + + assert_eq!( + preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"bar.glsl\"\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "#preinclude \"foo.glsl\"\n", + "\n", + "void main() {}", + ), + &|path| { + if path == OsStr::new("bar.glsl") { + Ok(b"out vec4 FragColor;".to_vec()) + } else { + Ok(concat!( + "uniform sampler2D input_texture;\n", + "in vec2 in_texture_coords;" + ) + .as_bytes() + .to_vec()) + } + }, + ) + .unwrap(), + concat!( + "#version 330 core\n", + "\n", + "out vec4 FragColor;\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "uniform sampler2D input_texture;\n", + "in vec2 in_texture_coords;\n", + "\n", + "void main() {}", + ) + ); + } + + #[test] + fn preprocess_invalid_directive_does_not_work() + { + let res = preprocess( + concat!( + "#version 330 core\n", + "\n", + // Missing " + "#preinclude foo.glsl\"\n", + "\n", + "void main() {}", + ), + &|_| Ok(b"out vec4 FragColor;".to_vec()), + ); + + let Err(PreprocessingError::InvalidToken { expected, found, span }) = res else { + panic!( + "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}" + ); + }; + + assert_eq!(expected, '"'); + assert_eq!(found, 'f'); + assert_eq!(span.line, 3); + assert_eq!(span.column, 13); + assert_eq!(span.path, SpanPath::Original); + } + + #[test] + fn preprocess_error_has_correct_span() + { + let res = preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"bar.glsl\"\n", + "\n", + "#preinclude \"foo.glsl\"\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ), + &|path| { + if path == OsStr::new("bar.glsl") { + Ok(concat!( + "out vec4 FragColor;\n", + "in vec2 in_texture_coords;\n", + "in float foo;" + ) + .as_bytes() + .to_vec()) + } else if path == OsStr::new("foo.glsl") { + Ok(concat!( + "uniform sampler2D input_texture;\n", + "\n", + // Missing space before first " + "#preinclude\"shared_types.glsl\"\n", + ) + .as_bytes() + .to_vec()) + } else { + panic!(concat!( + "Expected read function to be called with ", + "either path bar.glsl or foo.glsl" + )); + } + }, + ); + + let Err(PreprocessingError::InvalidToken { expected, found, span }) = res else { + panic!( + "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}" + ); + }; + + assert_eq!(expected, ' '); + assert_eq!(found, '"'); + assert_eq!(span.line, 3); + assert_eq!(span.column, 12); + assert_eq!(span.path, SpanPath::Path(Path::new("foo.glsl").into())); + } + + #[test] + fn preprocess_included_shader_with_include_works() + { + assert_eq!( + preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"bar.glsl\"\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ), + &|path| { + if path == OsStr::new("bar.glsl") { + Ok(concat!( + "#preinclude \"foo.glsl\"\n", + "\n", + "out vec4 FragColor;" + ) + .as_bytes() + .to_vec()) + } else { + Ok(concat!( + "uniform sampler2D input_texture;\n", + "in vec2 in_texture_coords;" + ) + .as_bytes() + .to_vec()) + } + } + ) + .unwrap(), + concat!( + "#version 330 core\n", + "\n", + "uniform sampler2D input_texture;\n", + "in vec2 in_texture_coords;\n", + "\n", + "out vec4 FragColor;\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ) + ); + } + + #[test] + fn preprocess_included_shader_with_include_error_span_is_correct() + { + let res = preprocess( + concat!( + "#version 330 core\n", + "\n", + "#preinclude \"bar.glsl\"\n", + "\n", + "in vec3 in_frag_color;\n", + "\n", + "void main() {}", + ), + &|path| { + if path == OsStr::new("bar.glsl") { + Ok(concat!( + // ' instead of " + "#preinclude 'foo.glsl\"\n", + "\n", + "out vec4 FragColor;" + ) + .as_bytes() + .to_vec()) + } else { + Ok(concat!( + "uniform sampler2D input_texture;\n", + "in vec2 in_texture_coords;" + ) + .as_bytes() + .to_vec()) + } + }, + ); + + let Err(PreprocessingError::InvalidToken { expected, found, span }) = res else { + panic!( + "Expected result to be Err(Error::InvalidToken {{ ... }}), is {res:?}" + ); + }; + + assert_eq!(expected, '"'); + assert_eq!(found, '\''); + assert_eq!(span.line, 1); + assert_eq!(span.column, 13); + assert_eq!(span.path, Path::new("bar.glsl")); + } +} -- cgit v1.2.3-18-g5258