gl_shader_decompiler: Add skeleton code from Citra for shader analysis.

2 files changed, 142 insertions, 44 deletions

diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp
index 564ea8f9e..3fc420649 100644
--- a/src/video_core/renderer_opengl/gl_shader_decompiler.cpp
+++ b/src/video_core/renderer_opengl/gl_shader_decompiler.cpp
@@ -2,57 +2,158 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <map>
+#include <set>
 #include <string>
-#include <queue>
 #include "common/assert.h"
 #include "common/common_types.h"
+#include "video_core/engines/shader_bytecode.h"
 #include "video_core/renderer_opengl/gl_shader_decompiler.h"
 
-namespace Maxwell3D {
+namespace Tegra {
 namespace Shader {
 namespace Decompiler {
 
 constexpr u32 PROGRAM_END = MAX_PROGRAM_CODE_LENGTH;
 
-class Impl {
+class DecompileFail : public std::runtime_error {
 public:
-    Impl(const std::array<u32, MAX_PROGRAM_CODE_LENGTH>& program_code,
-         const std::array<u32, MAX_SWIZZLE_DATA_LENGTH>& swizzle_data, u32 main_offset,
-         const std::function<std::string(u32)>& inputreg_getter,
-         const std::function<std::string(u32)>& outputreg_getter, bool sanitize_mul,
-         const std::string& emit_cb, const std::string& setemit_cb)
-        : program_code(program_code), swizzle_data(swizzle_data), main_offset(main_offset),
-          inputreg_getter(inputreg_getter), outputreg_getter(outputreg_getter),
-          sanitize_mul(sanitize_mul), emit_cb(emit_cb), setemit_cb(setemit_cb) {}
+    using std::runtime_error::runtime_error;
+};
+
+/// Describes the behaviour of code path of a given entry point and a return point.
+enum class ExitMethod {
+    Undetermined, ///< Internal value. Only occur when analyzing JMP loop.
+    AlwaysReturn, ///< All code paths reach the return point.
+    Conditional,  ///< Code path reaches the return point or an END instruction conditionally.
+    AlwaysEnd,    ///< All code paths reach a END instruction.
+};
+
+/// A subroutine is a range of code refereced by a CALL, IF or LOOP instruction.
+struct Subroutine {
+    /// Generates a name suitable for GLSL source code.
+    std::string GetName() const {
+        return "sub_" + std::to_string(begin) + "_" + std::to_string(end);
+    }
+
+    u32 begin;              ///< Entry point of the subroutine.
+    u32 end;                ///< Return point of the subroutine.
+    ExitMethod exit_method; ///< Exit method of the subroutine.
+    std::set<u32> labels;   ///< Addresses refereced by JMP instructions.
+
+    bool operator<(const Subroutine& rhs) const {
+        return std::tie(begin, end) < std::tie(rhs.begin, rhs.end);
+    }
+};
+
+/// Analyzes shader code and produces a set of subroutines.
+class ControlFlowAnalyzer {
+public:
+    ControlFlowAnalyzer(const ProgramCode& program_code, u32 main_offset)
+        : program_code(program_code) {
+
+        // Recursively finds all subroutines.
+        const Subroutine& program_main = AddSubroutine(main_offset, PROGRAM_END);
+        if (program_main.exit_method != ExitMethod::AlwaysEnd)
+            throw DecompileFail("Program does not always end");
+    }
+
+    std::set<Subroutine> GetSubroutines() {
+        return std::move(subroutines);
+    }
+
+private:
+    const ProgramCode& program_code;
+    std::set<Subroutine> subroutines;
+    std::map<std::pair<u32, u32>, ExitMethod> exit_method_map;
+
+    /// Adds and analyzes a new subroutine if it is not added yet.
+    const Subroutine& AddSubroutine(u32 begin, u32 end) {
+        auto iter = subroutines.find(Subroutine{begin, end});
+        if (iter != subroutines.end())
+            return *iter;
+
+        Subroutine subroutine{begin, end};
+        subroutine.exit_method = Scan(begin, end, subroutine.labels);
+        if (subroutine.exit_method == ExitMethod::Undetermined)
+            throw DecompileFail("Recursive function detected");
+        return *subroutines.insert(std::move(subroutine)).first;
+    }
 
-    std::string Decompile() {
-        UNREACHABLE();
-        return {};
+    /// Scans a range of code for labels and determines the exit method.
+    ExitMethod Scan(u32 begin, u32 end, std::set<u32>& labels) {
+        auto [iter, inserted] =
+            exit_method_map.emplace(std::make_pair(begin, end), ExitMethod::Undetermined);
+        ExitMethod& exit_method = iter->second;
+        if (!inserted)
+            return exit_method;
+
+        for (u32 offset = begin; offset != end && offset != PROGRAM_END; ++offset) {
+            const Instruction instr = {program_code[offset]};
+            switch (instr.opcode.Value().EffectiveOpCode()) {
+            case OpCode::Id::EXIT: {
+                return exit_method = ExitMethod::AlwaysEnd;
+            }
+            }
+        }
+        return exit_method = ExitMethod::AlwaysReturn;
+    }
+};
+
+class ShaderWriter {
+public:
+    void AddLine(const std::string& text) {
+        DEBUG_ASSERT(scope >= 0);
+        if (!text.empty()) {
+            shader_source += std::string(static_cast<size_t>(scope) * 4, ' ');
+        }
+        shader_source += text + '\n';
+    }
+
+    std::string GetResult() {
+        return std::move(shader_source);
+    }
+
+    int scope = 0;
+
+private:
+    std::string shader_source;
+};
+
+class GLSLGenerator {
+public:
+    GLSLGenerator(const std::set<Subroutine>& subroutines, const ProgramCode& program_code,
+                  u32 main_offset)
+        : subroutines(subroutines), program_code(program_code), main_offset(main_offset) {
+
+        Generate();
+    }
+
+    std::string GetShaderCode() {
+        return shader.GetResult();
     }
 
 private:
-    const std::array<u32, MAX_PROGRAM_CODE_LENGTH>& program_code;
-    const std::array<u32, MAX_SWIZZLE_DATA_LENGTH>& swizzle_data;
-    u32 main_offset;
-    const std::function<std::string(u32)>& inputreg_getter;
-    const std::function<std::string(u32)>& outputreg_getter;
-    bool sanitize_mul;
-    const std::string& emit_cb;
-    const std::string& setemit_cb;
+    const std::set<Subroutine>& subroutines;
+    const ProgramCode& program_code;
+    const u32 main_offset;
+
+    ShaderWriter shader;
+
+    void Generate() {}
 };
 
-std::string DecompileProgram(const std::array<u32, MAX_PROGRAM_CODE_LENGTH>& program_code,
-                             const std::array<u32, MAX_SWIZZLE_DATA_LENGTH>& swizzle_data,
-                             u32 main_offset,
-                             const std::function<std::string(u32)>& inputreg_getter,
-                             const std::function<std::string(u32)>& outputreg_getter,
-                             bool sanitize_mul, const std::string& emit_cb,
-                             const std::string& setemit_cb) {
-    Impl impl(program_code, swizzle_data, main_offset, inputreg_getter, outputreg_getter,
-              sanitize_mul, emit_cb, setemit_cb);
-    return impl.Decompile();
+boost::optional<std::string> DecompileProgram(const ProgramCode& program_code, u32 main_offset) {
+    try {
+        auto subroutines = ControlFlowAnalyzer(program_code, main_offset).GetSubroutines();
+        GLSLGenerator generator(subroutines, program_code, main_offset);
+        return generator.GetShaderCode();
+    } catch (const DecompileFail& exception) {
+        LOG_ERROR(HW_GPU, "Shader decompilation failed: %s", exception.what());
+    }
+    return boost::none;
 }
 
 } // namespace Decompiler
 } // namespace Shader
-} // namespace Maxwell3D
+} // namespace Tegra
diff --git a/src/video_core/renderer_opengl/gl_shader_decompiler.h b/src/video_core/renderer_opengl/gl_shader_decompiler.h
index 02ebfcbe8..628f02c93 100644
--- a/src/video_core/renderer_opengl/gl_shader_decompiler.h
+++ b/src/video_core/renderer_opengl/gl_shader_decompiler.h
@@ -5,23 +5,20 @@
 #include <array>
 #include <functional>
 #include <string>
+#include <boost/optional.hpp>
 #include "common/common_types.h"
 
-namespace Maxwell3D {
+namespace Tegra {
 namespace Shader {
 namespace Decompiler {
 
-constexpr size_t MAX_PROGRAM_CODE_LENGTH{0x100000};
-constexpr size_t MAX_SWIZZLE_DATA_LENGTH{0x100000};
+constexpr size_t MAX_PROGRAM_CODE_LENGTH{0x100};
+constexpr size_t MAX_SWIZZLE_DATA_LENGTH{0x100};
 
-std::string DecompileProgram(const std::array<u32, MAX_PROGRAM_CODE_LENGTH>& program_code,
-                             const std::array<u32, MAX_SWIZZLE_DATA_LENGTH>& swizzle_data,
-                             u32 main_offset,
-                             const std::function<std::string(u32)>& inputreg_getter,
-                             const std::function<std::string(u32)>& outputreg_getter,
-                             bool sanitize_mul, const std::string& emit_cb = "",
-                             const std::string& setemit_cb = "");
+using ProgramCode = std::array<u64, MAX_PROGRAM_CODE_LENGTH>;
+
+boost::optional<std::string> DecompileProgram(const ProgramCode& program_code, u32 main_offset);
 
 } // namespace Decompiler
 } // namespace Shader
-} // namespace Maxwell3D
+} // namespace Tegra