feat: 新版 FX 解析器 (p2)

src/Magpie.Core/ShaderEffectParser.cpp

@@ -4,11 +4,12 @@
 #include "StrHelper.h"
 #include "Logger.h"
 #include "ShaderEffectDesc.h"
+#include <bitset>
 
 namespace Magpie {
 
 // 当前 MagpieFX 版本
-// static constexpr uint32_t MAGPIE_FX_VERSION = 4;
+static constexpr uint32_t MAGPIE_FX_VERSION = 5;
 
 // 必须出现在一行的开头才视为指令
 static const char* META_INDICATOR = "//!";
@@ -19,6 +20,70 @@ struct ParserState {
 	bool isNewLine;
 };
 
+static bool RemoveLeadingComment(
+	std::string_view& source,
+	ParserState& state,
+	size_t& i,
+	bool& isComment,
+	bool& isMetaIdicator
+) noexcept {
+	assert(source[i] == '/');
+
+	isComment = false;
+	isMetaIdicator = false;
+
+	// source 以换行符结尾，因此可以安全提取非换行符的下一个的字符
+	char nextChar = source[i + 1];
+
+	if (nextChar == '/') {
+		if (state.isNewLine && source[i + 2] == '!') {
+			// 指令
+			isMetaIdicator = true;
+		} else {
+			// 行注释
+			isComment = true;
+
+			i += 2;
+			while (source[i] != '\n') {
+				++i;
+			}
+
+			++state.lineNumber;
+			state.isNewLine = true;
+		}
+	} else if (nextChar == '*') {
+		// 块注释
+		isComment = true;
+		i += 2;
+
+		while (true) {
+			char c = source[i];
+
+			if (c == '*') {
+				if (source[i + 1] == '/') {
+					// 块注释结束
+					++i;
+					state.isNewLine = false;
+					break;
+				}
+			} else if (c == '\n') {
+				// 块注释中换行不记为新行
+				++state.lineNumber;
+
+				// 提取换行符的后一个字符需检查文件结尾
+				if (i + 1 == source.size()) {
+					state.errorMsg = "块注释未闭合";
+					return false;
+				}
+			}
+
+			++i;
+		}
+	}
+
+	return true;
+}
+
 static void RemoveLeadingSpaces(std::string_view& source, ParserState& state) noexcept {
 	size_t i = 0;
 	for (; i < source.size(); ++i) {
@@ -40,64 +105,24 @@ static bool RemoveLeadingBlanks(std::string_view& source, ParserState& state) no
 
 		if (c == ' ' || c == '\t') {
 			state.isNewLine = false;
-			continue;
-		}
-
-		if (c == '\n') {
+		} else if (c == '\n') {
 			++state.lineNumber;
 			state.isNewLine = true;
-			continue;
-		}
+		} else {
+			if (c == '/') {
+				bool isComment;
+				bool isMetaIdicator;
+				if (!RemoveLeadingComment(source, state, i, isComment, isMetaIdicator)) {
+					return false;
+				}
 
-		if (c == '/') {
-			// source 以换行符结尾，因此可以安全提取非换行符的下一个的字符
-			char nextChar = source[i + 1];
-
-			if (nextChar == '/') {
-				if (!state.isNewLine || source[i + 2] != '!') {
-					// 行注释而非指令
-					i += 2;
-					while (source[i] != '\n') {
-						++i;
-					}
-
-					++state.lineNumber;
-					state.isNewLine = true;
+				if (isComment) {
 					continue;
 				}
-			} else if (nextChar == '*') {
-				// 块注释
-				i += 2;
-
-				while (true) {
-					c = source[i];
-
-					if (c == '*') {
-						if (source[i + 1] == '/') {
-							// 块注释结束
-							++i;
-							state.isNewLine = false;
-							break;
-						}
-					} else if (c == '\n') {
-						// 块注释中换行不记为新行
-						++state.lineNumber;
-
-						// 提取换行符的后一个字符需检查文件结尾
-						if (i + 1 == source.size()) {
-							state.errorMsg = "块注释未闭合";
-							return false;
-						}
-					}
-
-					++i;
-				}
-
-				continue;
 			}
-		}
 
-		break;
+			break;
+		}
 	}
 
 	source.remove_prefix(i);
@@ -129,12 +154,15 @@ static bool GetNextToken(std::string_view& source, ParserState& state, std::stri
 
 	// 可以包含字母、数字或下划线
 	size_t i = 1;
-	for (; i < source.size(); ++i) {
+	while(true) {
 		c = source[i];
 
 		if (!StrHelper::isalnum(c) && c != '_') {
 			break;
 		}
+
+		// source 以换行符结尾，因此无需检查越界
+		++i;
 	}
 
 	result = source.substr(0, i);
@@ -158,22 +186,34 @@ static bool CheckNextToken(std::string_view& source, ParserState& state, std::st
 	}
 }
 
-static bool CheckMetaIndicator(std::string_view& source, ParserState& state) noexcept {
+static bool CheckMetaIndicator(std::string_view& source, ParserState& state, bool& result) noexcept {
 	if (!RemoveLeadingBlanks(source, state)) {
 		return false;
 	}
 
-	if (state.isNewLine && source.starts_with(META_INDICATOR)) {
+	result = state.isNewLine && source.starts_with(META_INDICATOR);
+	if (result) {
 		source.remove_prefix(StrHelper::StrLen(META_INDICATOR));
 		state.isNewLine = false;
-		return true;
-	} else {
-		state.errorMsg = fmt::format("Unexpected character \"{}\" in line {}.", source[0], state.lineNumber);
-		return false;
 	}
+
+	return true;
 }
 
-static bool CheckLineEnd(std::string_view& source, ParserState& state) noexcept {
+static bool RequireMetaIndicator(std::string_view& source, ParserState& state) noexcept {
+	bool result = false;
+	if (!CheckMetaIndicator(source, state, result)) {
+		return false;
+	}
+
+	if (!result) {
+		state.errorMsg = fmt::format("Unexpected character \"{}\" in line {}.", source[0], state.lineNumber);
+	}
+
+	return result;
+}
+
+static bool RequireLineEnd(std::string_view& source, ParserState& state) noexcept {
 	RemoveLeadingSpaces(source, state);
 
 	if (source.empty() || source[0] == '\n') {
@@ -185,7 +225,7 @@ static bool CheckLineEnd(std::string_view& source, ParserState& state) noexcept
 }
 
 static bool CheckMagic(std::string_view& source, ParserState& state) noexcept {
-	if (!CheckMetaIndicator(source, state)) {
+	if (!RequireMetaIndicator(source, state)) {
 		return false;
 	}
 
@@ -196,7 +236,234 @@ static bool CheckMagic(std::string_view& source, ParserState& state) noexcept {
 		return false;
 	}
 
-	return CheckLineEnd(source, state);
+	return RequireLineEnd(source, state);
+}
+
+template <bool AllowEmpty>
+static bool GetNextStringUntilLineEnd(
+	std::string_view& source,
+	ParserState& state,
+	std::string_view& value
+) noexcept {
+	assert(value.empty());
+
+	RemoveLeadingSpaces(source, state);
+
+	if constexpr (AllowEmpty) {
+		if (source.empty()) {
+			return true;
+		}
+	}
+
+	size_t pos = source.find('\n');
+
+	value = source.substr(0, pos);
+	StrHelper::Trim(value);
+
+	if constexpr (!AllowEmpty) {
+		if (value.empty()) {
+			return false;
+		}
+	}
+
+	// 源码的最后一个字符必为换行，允许空字符串时已检查 source 不为空，不允许空字符串
+	// 时已检查 value 不为空。
+	assert(pos != std::string_view::npos);
+	// 不删除换行符
+	source.remove_prefix(pos);
+	state.isNewLine = false;
+
+	return true;
+}
+
+template <typename T>
+static bool GetNextNumber(std::string_view& source, ParserState& state, T& value) noexcept {
+	RemoveLeadingSpaces(source, state);
+
+	const auto& result = std::from_chars(source.data(), source.data() + source.size(), value);
+	if ((int)result.ec) {
+		return false;
+	}
+
+	source.remove_prefix(result.ptr - source.data());
+	state.isNewLine = false;
+	return true;
+}
+
+static bool ResolveHeaderVersion(
+	std::string_view& source,
+	ParserState& state,
+	EffectInfo2&
+) noexcept {
+	uint32_t version;
+	if (!GetNextNumber(source, state, version)) {
+		return false;
+	}
+
+	if (version != MAGPIE_FX_VERSION) {
+		return false;
+	}
+
+	if (!RequireLineEnd(source, state)) {
+		return false;
+	}
+
+	return true;
+}
+
+static bool ResolveHeaderSortName(
+	std::string_view& source,
+	ParserState& state,
+	EffectInfo2& effectInfo
+) noexcept {
+	std::string_view sortName;
+	if (!GetNextStringUntilLineEnd<false>(source, state, sortName)) {
+		return false;
+	}
+
+	effectInfo.sortName = sortName;
+	return true;
+}
+
+static bool ResolveHeaderUse(
+	std::string_view& source,
+	ParserState& state,
+	EffectInfo2&
+) noexcept {
+	std::string_view flags;
+	return GetNextStringUntilLineEnd<false>(source, state, flags);
+}
+
+static bool ResolveHeaderCapability(
+	std::string_view& source,
+	ParserState& state,
+	EffectInfo2& effectInfo
+) noexcept {
+	std::string_view flags;
+	if (!GetNextStringUntilLineEnd<false>(source, state, flags)) {
+		return false;
+	}
+
+	static constexpr std::array FLAG_INFOS = {
+		// 以下为必需
+		std::make_pair("FP16", EffectInfoFlags2::SupportFP16),
+		// 以下为可选
+		std::make_pair("ADVANCEDCOLOR", EffectInfoFlags2::SupportAdvancedColor)
+	};
+
+	std::bitset<FLAG_INFOS.size()> processed;
+
+	for (std::string_view& token : StrHelper::Split(flags, ',')) {
+		StrHelper::Trim(token);
+		std::string flag = StrHelper::ToUpperCase(token);
+
+		auto it = std::find_if(FLAG_INFOS.begin(), FLAG_INFOS.end(), [&](const auto& flagInfo) {
+			return flagInfo.first == flag;
+		});
+		if (it != FLAG_INFOS.end()) {
+			size_t idx = it - FLAG_INFOS.begin();
+
+			if (processed[idx]) {
+				return false;
+			}
+			processed[idx] = true;
+
+			effectInfo.flags |= it->second;
+		} else {
+			Logger::Get().Warn(StrHelper::Concat("使用了未知 CAPABILITY 标志: ", token));
+		}
+	}
+
+	return true;
+}
+
+static bool ResolveHeader(
+	std::string_view source,
+	uint32_t startLineNumer,
+	EffectInfo2& effectInfo
+) noexcept {
+	static constexpr std::array COMMAND_INFOS = {
+		// 以下为必需
+		std::make_pair("VERSION", ResolveHeaderVersion),
+		// 以下为可选
+		std::make_pair("SORT_NAME", ResolveHeaderSortName),
+		std::make_pair("USE", ResolveHeaderUse),
+		std::make_pair("CAPABILITY", ResolveHeaderCapability)
+	};
+
+	ParserState state = {
+		.lineNumber = startLineNumer,
+		.isNewLine = false
+	};
+
+	std::bitset<COMMAND_INFOS.size()> processed;
+
+	std::string_view token;
+
+	while (true) {
+		bool isMetaIndicator = false;
+		if (!CheckMetaIndicator(source, state, isMetaIndicator)) {
+			return false;
+		}
+
+		if (!isMetaIndicator) {
+			break;
+		}
+
+		if (!GetNextToken<false>(source, state, token)) {
+			return false;
+		}
+		std::string command = StrHelper::ToUpperCase(token);
+
+		auto it = std::find_if(COMMAND_INFOS.begin(), COMMAND_INFOS.end(), [&](const auto& commandInfo) {
+			return commandInfo.first == command;
+		});
+		if (it != COMMAND_INFOS.end()) {
+			size_t idx = it - COMMAND_INFOS.begin();
+
+			if (processed[idx]) {
+				return false;
+			}
+			processed[idx] = true;
+
+			if (!it->second(source, state, effectInfo)) {
+				return false;
+			}
+		} else {
+			Logger::Get().Warn(StrHelper::Concat("解析头时遇到未知指令: ", token));
+
+			std::string_view unused;
+			if (!GetNextStringUntilLineEnd<true>(source, state, unused)) {
+				return false;
+			}
+		}
+	}
+
+	if (!RemoveLeadingBlanks(source, state)) {
+		return false;
+	}
+
+	if (source.empty()) {
+		return true;
+	}
+
+	// HEADER 只能有 #include
+	if (!source.starts_with("#include")) {
+		return false;
+	}
+
+	// 跳过整行
+	std::string_view unused;
+	if (!GetNextStringUntilLineEnd<false>(source, state, unused)) {
+		return false;
+	}
+
+	// 之后不允许有内容
+	if (!RemoveLeadingBlanks(source, state)) {
+		return false;
+	}
+
+	return source.empty();
 }
 
 std::string ShaderEffectParser::ParseForInfo(
@@ -224,12 +491,115 @@ std::string ShaderEffectParser::ParseForInfo(
 		return std::move(state.errorMsg);
 	}
 
+	enum class BlockType {
+		Header,
+		Parameter,
+		Texture,
+		Sampler,
+		Common,
+		Pass
+	};
+
+	struct BlockData {
+		std::string_view source;
+		uint32_t startLineNumer;
+	};
+
+	BlockData headerBlock{};
+	SmallVector<BlockData> paramBlocks;
+
+	BlockType curBlockType = BlockType::Header;
+	size_t curBlockOffset = 0;
+	uint32_t curBlockStartLineNumber = state.lineNumber;
+
+	const auto completeCurrentBlock = [&](
+		BlockType newBlockType,
+		size_t curBlockEnd,
+		size_t newBlockOffset
+	) {
+		assert(curBlockOffset < curBlockEnd && curBlockEnd < newBlockOffset);
+		assert(sourceView[curBlockEnd - 1] == '\n');
+
+		switch (curBlockType) {
+		case BlockType::Header:
+			headerBlock = BlockData{
+				sourceView.substr(curBlockOffset, curBlockEnd - curBlockOffset),curBlockStartLineNumber };
+			break;
+		case BlockType::Parameter:
+			paramBlocks.push_back(BlockData{
+				sourceView.substr(curBlockOffset, curBlockEnd - curBlockOffset),curBlockStartLineNumber });
+			break;
+		case BlockType::Texture:
+		case BlockType::Sampler:
+		case BlockType::Common:
+		case BlockType::Pass:
+			break;
+		default:
+			assert(false);
+			break;
+		}
+
+		curBlockType = newBlockType;
+		curBlockOffset = newBlockOffset;
+		curBlockStartLineNumber = state.lineNumber;
+	};
+
+	for (size_t i = 0; i < sourceView.size(); ++i) {
+		char c = sourceView[i];
+
+		if (c == '\n') {
+			++state.lineNumber;
+			state.isNewLine = true;
+		} else if (c == '/') {
+			bool isComment;
+			bool isMetaIdicator;
+			if (!RemoveLeadingComment(sourceView, state, i, isComment, isMetaIdicator)) {
+				return std::move(state.errorMsg);
+			}
+
+			if (isMetaIdicator) {
+				std::string_view tempSource = sourceView.substr(i + 3);
+				std::string_view token;
+				if (!GetNextToken<false>(tempSource, state, token)) {
+					return std::move(state.errorMsg);
+				}
+
+				std::string newBlockType = StrHelper::ToUpperCase(token);
+				size_t newBlockOffset = tempSource.data() - sourceView.data();
+
+				// sourceView[i - 1] 是换行符，因此每个区块都以换行结尾。区块开头不包含声明该区块的
+				// 指令，如果该指令没有参数，此区块就以换行符开头。
+				if (newBlockType == "PARAMETER") {
+					completeCurrentBlock(BlockType::Parameter, i, newBlockOffset);
+				} else if (newBlockType == "TEXTURE") {
+					completeCurrentBlock(BlockType::Texture, i, newBlockOffset);
+				} else if (newBlockType == "SAMPLER") {
+					completeCurrentBlock(BlockType::Sampler, i, newBlockOffset);
+				} else if (newBlockType == "COMMON") {
+					completeCurrentBlock(BlockType::Common, i, newBlockOffset);
+				} else if (newBlockType == "PASS") {
+					completeCurrentBlock(BlockType::Pass, i, newBlockOffset);
+				}
+
+				i = newBlockOffset;
+				state.isNewLine = false;
+			}
+		}
+	}
+
+	// 结束最后一个区块。source 以换行符结尾，因此最后一个区块也以换行符结尾。
+	completeCurrentBlock(BlockType::Header, sourceView.size(), std::numeric_limits<size_t>::max());
+
+	if (!ResolveHeader(headerBlock.source, headerBlock.startLineNumer, effectInfo)) {
+		return std::move(state.errorMsg);
+	}
+
 	return std::move(state.errorMsg);
 }
 
 bool ShaderEffectParser::ParseForDesc(
 	std::string&& name,
-	std::string_view source,
+	std::string&& source,
 	std::string&& workingFolder,
 	const ShaderEffectParserOptions& /*options*/,
 	struct ShaderEffectDesc& effectDesc,
@@ -239,6 +609,18 @@ bool ShaderEffectParser::ParseForDesc(
 
 	effectDesc.name = std::move(name);
 	effectSource.workingFolder = std::move(workingFolder);
+
+	ParserState state = {
+		.lineNumber = 1,
+		.isNewLine = true
+	};
+
+	// 确保源代码以换行结尾，这可以有效简化文件结尾检查
+	if (!source.ends_with('\n')) {
+		source.push_back('\n');
+	}
+	std::string_view sourceView(source);
+
 	return false;
 }
 

src/Magpie.Core/include/ShaderEffectParser.h

@@ -38,7 +38,7 @@ struct ShaderEffectParser {
 
 	static bool ParseForDesc(
 		std::string&& name,
-		std::string_view source,
+		std::string&& source,
 		std::string&& workingFolder,
 		const ShaderEffectParserOptions& options,
 		struct ShaderEffectDesc& effectDesc,

src/Magpie/EffectsService.cpp

@@ -1,7 +1,9 @@
 #include "pch.h"
 #include "CommonSharedConstants.h"
 #include "EffectCompiler.h"
+#include "ShaderEffectParser.h"
 #include "EffectDesc.h"
+#include "EffectInfo.h"
 #include "EffectsService.h"
 #include "Logger.h"
 #include "StrHelper.h"
@@ -64,6 +66,16 @@ fire_and_forget EffectsService::Initialize() {
 	Win32Helper::RunParallel([&](uint32_t id) {
 		EffectDesc effectDesc;
 
+		{
+			std::wstring fileName = StrHelper::Concat(
+				CommonSharedConstants::EFFECTS_DIR, L"\\", effectNames[id], L".hlsl");
+			std::string source;
+			Win32Helper::ReadTextFile(fileName.c_str(), source);
+			EffectInfo2 effectInfo;
+			ShaderEffectParser::ParseForInfo(
+				StrHelper::UTF16ToUTF8(effectNames[id]), std::move(source), effectInfo);
+		}
+		
 		effectDesc.name = StrHelper::UTF16ToUTF8(effectNames[id]);
 		if (EffectCompiler::Compile(effectDesc, EffectCompilerFlags::NoCompile)) {
 			return;