diff options
| author | 2020-11-04 18:36:55 +1100 | |
|---|---|---|
| committer | 2020-11-04 18:36:55 +1100 | |
| commit | 6bbbbe8f85369dfc7a67441e5f7f6ab7a6484ae1 (patch) | |
| tree | a97a13d00eaae7e81f028a57fef7dadf9d96a27c /src/core/hle | |
| parent | Merge pull request #4874 from lioncash/nodiscard2 (diff) | |
| parent | fixup! hle service: nvdrv: nvhost_gpu: Update to use SyncpointManager and ot... (diff) | |
| download | yuzu-6bbbbe8f85369dfc7a67441e5f7f6ab7a6484ae1.tar.gz yuzu-6bbbbe8f85369dfc7a67441e5f7f6ab7a6484ae1.tar.xz yuzu-6bbbbe8f85369dfc7a67441e5f7f6ab7a6484ae1.zip | |
Merge pull request #4869 from bunnei/improve-gpu-sync
Improvements to GPU synchronization & various refactoring
Diffstat (limited to 'src/core/hle')
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp | 33 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_ctrl.h | 4 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp | 135 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_gpu.h | 20 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/nvdrv.cpp | 17 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/nvdrv.h | 14 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/syncpoint_manager.cpp | 39 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/syncpoint_manager.h | 85 | ||||
| -rw-r--r-- | src/core/hle/service/nvflinger/nvflinger.cpp | 4 |
9 files changed, 291 insertions, 60 deletions
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp index 75d9191ff..8356a8139 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp | |||
| @@ -15,8 +15,9 @@ | |||
| 15 | 15 | ||
| 16 | namespace Service::Nvidia::Devices { | 16 | namespace Service::Nvidia::Devices { |
| 17 | 17 | ||
| 18 | nvhost_ctrl::nvhost_ctrl(Core::System& system, EventInterface& events_interface) | 18 | nvhost_ctrl::nvhost_ctrl(Core::System& system, EventInterface& events_interface, |
| 19 | : nvdevice(system), events_interface{events_interface} {} | 19 | SyncpointManager& syncpoint_manager) |
| 20 | : nvdevice(system), events_interface{events_interface}, syncpoint_manager{syncpoint_manager} {} | ||
| 20 | nvhost_ctrl::~nvhost_ctrl() = default; | 21 | nvhost_ctrl::~nvhost_ctrl() = default; |
| 21 | 22 | ||
| 22 | u32 nvhost_ctrl::ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, | 23 | u32 nvhost_ctrl::ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, |
| @@ -70,19 +71,33 @@ u32 nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector<u8>& | |||
| 70 | return NvResult::BadParameter; | 71 | return NvResult::BadParameter; |
| 71 | } | 72 | } |
| 72 | 73 | ||
| 74 | if (syncpoint_manager.IsSyncpointExpired(params.syncpt_id, params.threshold)) { | ||
| 75 | params.value = syncpoint_manager.GetSyncpointMin(params.syncpt_id); | ||
| 76 | std::memcpy(output.data(), ¶ms, sizeof(params)); | ||
| 77 | return NvResult::Success; | ||
| 78 | } | ||
| 79 | |||
| 80 | if (const auto new_value = syncpoint_manager.RefreshSyncpoint(params.syncpt_id); | ||
| 81 | syncpoint_manager.IsSyncpointExpired(params.syncpt_id, params.threshold)) { | ||
| 82 | params.value = new_value; | ||
| 83 | std::memcpy(output.data(), ¶ms, sizeof(params)); | ||
| 84 | return NvResult::Success; | ||
| 85 | } | ||
| 86 | |||
| 73 | auto event = events_interface.events[event_id]; | 87 | auto event = events_interface.events[event_id]; |
| 74 | auto& gpu = system.GPU(); | 88 | auto& gpu = system.GPU(); |
| 89 | |||
| 75 | // This is mostly to take into account unimplemented features. As synced | 90 | // This is mostly to take into account unimplemented features. As synced |
| 76 | // gpu is always synced. | 91 | // gpu is always synced. |
| 77 | if (!gpu.IsAsync()) { | 92 | if (!gpu.IsAsync()) { |
| 78 | event.writable->Signal(); | 93 | event.event.writable->Signal(); |
| 79 | return NvResult::Success; | 94 | return NvResult::Success; |
| 80 | } | 95 | } |
| 81 | auto lock = gpu.LockSync(); | 96 | auto lock = gpu.LockSync(); |
| 82 | const u32 current_syncpoint_value = gpu.GetSyncpointValue(params.syncpt_id); | 97 | const u32 current_syncpoint_value = event.fence.value; |
| 83 | const s32 diff = current_syncpoint_value - params.threshold; | 98 | const s32 diff = current_syncpoint_value - params.threshold; |
| 84 | if (diff >= 0) { | 99 | if (diff >= 0) { |
| 85 | event.writable->Signal(); | 100 | event.event.writable->Signal(); |
| 86 | params.value = current_syncpoint_value; | 101 | params.value = current_syncpoint_value; |
| 87 | std::memcpy(output.data(), ¶ms, sizeof(params)); | 102 | std::memcpy(output.data(), ¶ms, sizeof(params)); |
| 88 | return NvResult::Success; | 103 | return NvResult::Success; |
| @@ -109,7 +124,7 @@ u32 nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector<u8>& | |||
| 109 | params.value = ((params.syncpt_id & 0xfff) << 16) | 0x10000000; | 124 | params.value = ((params.syncpt_id & 0xfff) << 16) | 0x10000000; |
| 110 | } | 125 | } |
| 111 | params.value |= event_id; | 126 | params.value |= event_id; |
| 112 | event.writable->Clear(); | 127 | event.event.writable->Clear(); |
| 113 | gpu.RegisterSyncptInterrupt(params.syncpt_id, target_value); | 128 | gpu.RegisterSyncptInterrupt(params.syncpt_id, target_value); |
| 114 | if (!is_async && ctrl.fresh_call) { | 129 | if (!is_async && ctrl.fresh_call) { |
| 115 | ctrl.must_delay = true; | 130 | ctrl.must_delay = true; |
| @@ -157,15 +172,19 @@ u32 nvhost_ctrl::IocCtrlEventUnregister(const std::vector<u8>& input, std::vecto | |||
| 157 | u32 nvhost_ctrl::IocCtrlClearEventWait(const std::vector<u8>& input, std::vector<u8>& output) { | 172 | u32 nvhost_ctrl::IocCtrlClearEventWait(const std::vector<u8>& input, std::vector<u8>& output) { |
| 158 | IocCtrlEventSignalParams params{}; | 173 | IocCtrlEventSignalParams params{}; |
| 159 | std::memcpy(¶ms, input.data(), sizeof(params)); | 174 | std::memcpy(¶ms, input.data(), sizeof(params)); |
| 175 | |||
| 160 | u32 event_id = params.event_id & 0x00FF; | 176 | u32 event_id = params.event_id & 0x00FF; |
| 161 | LOG_WARNING(Service_NVDRV, "cleared event wait on, event_id: {:X}", event_id); | 177 | LOG_WARNING(Service_NVDRV, "cleared event wait on, event_id: {:X}", event_id); |
| 178 | |||
| 162 | if (event_id >= MaxNvEvents) { | 179 | if (event_id >= MaxNvEvents) { |
| 163 | return NvResult::BadParameter; | 180 | return NvResult::BadParameter; |
| 164 | } | 181 | } |
| 165 | if (events_interface.status[event_id] == EventState::Waiting) { | 182 | if (events_interface.status[event_id] == EventState::Waiting) { |
| 166 | events_interface.LiberateEvent(event_id); | 183 | events_interface.LiberateEvent(event_id); |
| 167 | events_interface.events[event_id].writable->Signal(); | ||
| 168 | } | 184 | } |
| 185 | |||
| 186 | syncpoint_manager.RefreshSyncpoint(events_interface.events[event_id].fence.id); | ||
| 187 | |||
| 169 | return NvResult::Success; | 188 | return NvResult::Success; |
| 170 | } | 189 | } |
| 171 | 190 | ||
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h index f7b04d9f1..24ad96cb9 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h | |||
| @@ -14,7 +14,8 @@ namespace Service::Nvidia::Devices { | |||
| 14 | 14 | ||
| 15 | class nvhost_ctrl final : public nvdevice { | 15 | class nvhost_ctrl final : public nvdevice { |
| 16 | public: | 16 | public: |
| 17 | explicit nvhost_ctrl(Core::System& system, EventInterface& events_interface); | 17 | explicit nvhost_ctrl(Core::System& system, EventInterface& events_interface, |
| 18 | SyncpointManager& syncpoint_manager); | ||
| 18 | ~nvhost_ctrl() override; | 19 | ~nvhost_ctrl() override; |
| 19 | 20 | ||
| 20 | u32 ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, | 21 | u32 ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, |
| @@ -145,6 +146,7 @@ private: | |||
| 145 | u32 IocCtrlClearEventWait(const std::vector<u8>& input, std::vector<u8>& output); | 146 | u32 IocCtrlClearEventWait(const std::vector<u8>& input, std::vector<u8>& output); |
| 146 | 147 | ||
| 147 | EventInterface& events_interface; | 148 | EventInterface& events_interface; |
| 149 | SyncpointManager& syncpoint_manager; | ||
| 148 | }; | 150 | }; |
| 149 | 151 | ||
| 150 | } // namespace Service::Nvidia::Devices | 152 | } // namespace Service::Nvidia::Devices |
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index f1966ac0e..152019548 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp | |||
| @@ -7,14 +7,20 @@ | |||
| 7 | #include "common/logging/log.h" | 7 | #include "common/logging/log.h" |
| 8 | #include "core/core.h" | 8 | #include "core/core.h" |
| 9 | #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" | 9 | #include "core/hle/service/nvdrv/devices/nvhost_gpu.h" |
| 10 | #include "core/hle/service/nvdrv/syncpoint_manager.h" | ||
| 10 | #include "core/memory.h" | 11 | #include "core/memory.h" |
| 11 | #include "video_core/gpu.h" | 12 | #include "video_core/gpu.h" |
| 12 | #include "video_core/memory_manager.h" | 13 | #include "video_core/memory_manager.h" |
| 13 | 14 | ||
| 14 | namespace Service::Nvidia::Devices { | 15 | namespace Service::Nvidia::Devices { |
| 15 | 16 | ||
| 16 | nvhost_gpu::nvhost_gpu(Core::System& system, std::shared_ptr<nvmap> nvmap_dev) | 17 | nvhost_gpu::nvhost_gpu(Core::System& system, std::shared_ptr<nvmap> nvmap_dev, |
| 17 | : nvdevice(system), nvmap_dev(std::move(nvmap_dev)) {} | 18 | SyncpointManager& syncpoint_manager) |
| 19 | : nvdevice(system), nvmap_dev(std::move(nvmap_dev)), syncpoint_manager{syncpoint_manager} { | ||
| 20 | channel_fence.id = syncpoint_manager.AllocateSyncpoint(); | ||
| 21 | channel_fence.value = system.GPU().GetSyncpointValue(channel_fence.id); | ||
| 22 | } | ||
| 23 | |||
| 18 | nvhost_gpu::~nvhost_gpu() = default; | 24 | nvhost_gpu::~nvhost_gpu() = default; |
| 19 | 25 | ||
| 20 | u32 nvhost_gpu::ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, | 26 | u32 nvhost_gpu::ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, |
| @@ -126,10 +132,10 @@ u32 nvhost_gpu::AllocGPFIFOEx2(const std::vector<u8>& input, std::vector<u8>& ou | |||
| 126 | params.num_entries, params.flags, params.unk0, params.unk1, params.unk2, | 132 | params.num_entries, params.flags, params.unk0, params.unk1, params.unk2, |
| 127 | params.unk3); | 133 | params.unk3); |
| 128 | 134 | ||
| 129 | auto& gpu = system.GPU(); | 135 | channel_fence.value = system.GPU().GetSyncpointValue(channel_fence.id); |
| 130 | params.fence_out.id = assigned_syncpoints; | 136 | |
| 131 | params.fence_out.value = gpu.GetSyncpointValue(assigned_syncpoints); | 137 | params.fence_out = channel_fence; |
| 132 | assigned_syncpoints++; | 138 | |
| 133 | std::memcpy(output.data(), ¶ms, output.size()); | 139 | std::memcpy(output.data(), ¶ms, output.size()); |
| 134 | return 0; | 140 | return 0; |
| 135 | } | 141 | } |
| @@ -145,39 +151,100 @@ u32 nvhost_gpu::AllocateObjectContext(const std::vector<u8>& input, std::vector< | |||
| 145 | return 0; | 151 | return 0; |
| 146 | } | 152 | } |
| 147 | 153 | ||
| 148 | u32 nvhost_gpu::SubmitGPFIFO(const std::vector<u8>& input, std::vector<u8>& output) { | 154 | static std::vector<Tegra::CommandHeader> BuildWaitCommandList(Fence fence) { |
| 149 | if (input.size() < sizeof(IoctlSubmitGpfifo)) { | 155 | return { |
| 150 | UNIMPLEMENTED(); | 156 | Tegra::BuildCommandHeader(Tegra::BufferMethods::FenceValue, 1, |
| 157 | Tegra::SubmissionMode::Increasing), | ||
| 158 | {fence.value}, | ||
| 159 | Tegra::BuildCommandHeader(Tegra::BufferMethods::FenceAction, 1, | ||
| 160 | Tegra::SubmissionMode::Increasing), | ||
| 161 | Tegra::GPU::FenceAction::Build(Tegra::GPU::FenceOperation::Acquire, fence.id), | ||
| 162 | }; | ||
| 163 | } | ||
| 164 | |||
| 165 | static std::vector<Tegra::CommandHeader> BuildIncrementCommandList(Fence fence, u32 add_increment) { | ||
| 166 | std::vector<Tegra::CommandHeader> result{ | ||
| 167 | Tegra::BuildCommandHeader(Tegra::BufferMethods::FenceValue, 1, | ||
| 168 | Tegra::SubmissionMode::Increasing), | ||
| 169 | {}}; | ||
| 170 | |||
| 171 | for (u32 count = 0; count < add_increment; ++count) { | ||
| 172 | result.emplace_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::FenceAction, 1, | ||
| 173 | Tegra::SubmissionMode::Increasing)); | ||
| 174 | result.emplace_back( | ||
| 175 | Tegra::GPU::FenceAction::Build(Tegra::GPU::FenceOperation::Increment, fence.id)); | ||
| 151 | } | 176 | } |
| 152 | IoctlSubmitGpfifo params{}; | 177 | |
| 153 | std::memcpy(¶ms, input.data(), sizeof(IoctlSubmitGpfifo)); | 178 | return result; |
| 179 | } | ||
| 180 | |||
| 181 | static std::vector<Tegra::CommandHeader> BuildIncrementWithWfiCommandList(Fence fence, | ||
| 182 | u32 add_increment) { | ||
| 183 | std::vector<Tegra::CommandHeader> result{ | ||
| 184 | Tegra::BuildCommandHeader(Tegra::BufferMethods::WaitForInterrupt, 1, | ||
| 185 | Tegra::SubmissionMode::Increasing), | ||
| 186 | {}}; | ||
| 187 | const std::vector<Tegra::CommandHeader> increment{ | ||
| 188 | BuildIncrementCommandList(fence, add_increment)}; | ||
| 189 | |||
| 190 | result.insert(result.end(), increment.begin(), increment.end()); | ||
| 191 | |||
| 192 | return result; | ||
| 193 | } | ||
| 194 | |||
| 195 | u32 nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, std::vector<u8>& output, | ||
| 196 | Tegra::CommandList&& entries) { | ||
| 154 | LOG_TRACE(Service_NVDRV, "called, gpfifo={:X}, num_entries={:X}, flags={:X}", params.address, | 197 | LOG_TRACE(Service_NVDRV, "called, gpfifo={:X}, num_entries={:X}, flags={:X}", params.address, |
| 155 | params.num_entries, params.flags.raw); | 198 | params.num_entries, params.flags.raw); |
| 156 | 199 | ||
| 157 | ASSERT_MSG(input.size() == sizeof(IoctlSubmitGpfifo) + | 200 | auto& gpu = system.GPU(); |
| 158 | params.num_entries * sizeof(Tegra::CommandListHeader), | ||
| 159 | "Incorrect input size"); | ||
| 160 | 201 | ||
| 161 | Tegra::CommandList entries(params.num_entries); | 202 | params.fence_out.id = channel_fence.id; |
| 162 | std::memcpy(entries.data(), &input[sizeof(IoctlSubmitGpfifo)], | ||
| 163 | params.num_entries * sizeof(Tegra::CommandListHeader)); | ||
| 164 | 203 | ||
| 165 | UNIMPLEMENTED_IF(params.flags.add_wait.Value() != 0); | 204 | if (params.flags.add_wait.Value() && |
| 166 | UNIMPLEMENTED_IF(params.flags.add_increment.Value() != 0); | 205 | !syncpoint_manager.IsSyncpointExpired(params.fence_out.id, params.fence_out.value)) { |
| 206 | gpu.PushGPUEntries(Tegra::CommandList{BuildWaitCommandList(params.fence_out)}); | ||
| 207 | } | ||
| 167 | 208 | ||
| 168 | auto& gpu = system.GPU(); | 209 | if (params.flags.add_increment.Value() || params.flags.increment.Value()) { |
| 169 | u32 current_syncpoint_value = gpu.GetSyncpointValue(params.fence_out.id); | 210 | const u32 increment_value = params.flags.increment.Value() ? params.fence_out.value : 0; |
| 170 | if (params.flags.increment.Value()) { | 211 | params.fence_out.value = syncpoint_manager.IncreaseSyncpoint( |
| 171 | params.fence_out.value += current_syncpoint_value; | 212 | params.fence_out.id, params.AddIncrementValue() + increment_value); |
| 172 | } else { | 213 | } else { |
| 173 | params.fence_out.value = current_syncpoint_value; | 214 | params.fence_out.value = syncpoint_manager.GetSyncpointMax(params.fence_out.id); |
| 174 | } | 215 | } |
| 216 | |||
| 217 | entries.RefreshIntegrityChecks(gpu); | ||
| 175 | gpu.PushGPUEntries(std::move(entries)); | 218 | gpu.PushGPUEntries(std::move(entries)); |
| 176 | 219 | ||
| 220 | if (params.flags.add_increment.Value()) { | ||
| 221 | if (params.flags.suppress_wfi) { | ||
| 222 | gpu.PushGPUEntries(Tegra::CommandList{ | ||
| 223 | BuildIncrementCommandList(params.fence_out, params.AddIncrementValue())}); | ||
| 224 | } else { | ||
| 225 | gpu.PushGPUEntries(Tegra::CommandList{ | ||
| 226 | BuildIncrementWithWfiCommandList(params.fence_out, params.AddIncrementValue())}); | ||
| 227 | } | ||
| 228 | } | ||
| 229 | |||
| 177 | std::memcpy(output.data(), ¶ms, sizeof(IoctlSubmitGpfifo)); | 230 | std::memcpy(output.data(), ¶ms, sizeof(IoctlSubmitGpfifo)); |
| 178 | return 0; | 231 | return 0; |
| 179 | } | 232 | } |
| 180 | 233 | ||
| 234 | u32 nvhost_gpu::SubmitGPFIFO(const std::vector<u8>& input, std::vector<u8>& output) { | ||
| 235 | if (input.size() < sizeof(IoctlSubmitGpfifo)) { | ||
| 236 | UNIMPLEMENTED(); | ||
| 237 | } | ||
| 238 | IoctlSubmitGpfifo params{}; | ||
| 239 | std::memcpy(¶ms, input.data(), sizeof(IoctlSubmitGpfifo)); | ||
| 240 | |||
| 241 | Tegra::CommandList entries(params.num_entries); | ||
| 242 | std::memcpy(entries.command_lists.data(), &input[sizeof(IoctlSubmitGpfifo)], | ||
| 243 | params.num_entries * sizeof(Tegra::CommandListHeader)); | ||
| 244 | |||
| 245 | return SubmitGPFIFOImpl(params, output, std::move(entries)); | ||
| 246 | } | ||
| 247 | |||
| 181 | u32 nvhost_gpu::KickoffPB(const std::vector<u8>& input, std::vector<u8>& output, | 248 | u32 nvhost_gpu::KickoffPB(const std::vector<u8>& input, std::vector<u8>& output, |
| 182 | const std::vector<u8>& input2, IoctlVersion version) { | 249 | const std::vector<u8>& input2, IoctlVersion version) { |
| 183 | if (input.size() < sizeof(IoctlSubmitGpfifo)) { | 250 | if (input.size() < sizeof(IoctlSubmitGpfifo)) { |
| @@ -185,31 +252,17 @@ u32 nvhost_gpu::KickoffPB(const std::vector<u8>& input, std::vector<u8>& output, | |||
| 185 | } | 252 | } |
| 186 | IoctlSubmitGpfifo params{}; | 253 | IoctlSubmitGpfifo params{}; |
| 187 | std::memcpy(¶ms, input.data(), sizeof(IoctlSubmitGpfifo)); | 254 | std::memcpy(¶ms, input.data(), sizeof(IoctlSubmitGpfifo)); |
| 188 | LOG_TRACE(Service_NVDRV, "called, gpfifo={:X}, num_entries={:X}, flags={:X}", params.address, | ||
| 189 | params.num_entries, params.flags.raw); | ||
| 190 | 255 | ||
| 191 | Tegra::CommandList entries(params.num_entries); | 256 | Tegra::CommandList entries(params.num_entries); |
| 192 | if (version == IoctlVersion::Version2) { | 257 | if (version == IoctlVersion::Version2) { |
| 193 | std::memcpy(entries.data(), input2.data(), | 258 | std::memcpy(entries.command_lists.data(), input2.data(), |
| 194 | params.num_entries * sizeof(Tegra::CommandListHeader)); | 259 | params.num_entries * sizeof(Tegra::CommandListHeader)); |
| 195 | } else { | 260 | } else { |
| 196 | system.Memory().ReadBlock(params.address, entries.data(), | 261 | system.Memory().ReadBlock(params.address, entries.command_lists.data(), |
| 197 | params.num_entries * sizeof(Tegra::CommandListHeader)); | 262 | params.num_entries * sizeof(Tegra::CommandListHeader)); |
| 198 | } | 263 | } |
| 199 | UNIMPLEMENTED_IF(params.flags.add_wait.Value() != 0); | ||
| 200 | UNIMPLEMENTED_IF(params.flags.add_increment.Value() != 0); | ||
| 201 | |||
| 202 | auto& gpu = system.GPU(); | ||
| 203 | u32 current_syncpoint_value = gpu.GetSyncpointValue(params.fence_out.id); | ||
| 204 | if (params.flags.increment.Value()) { | ||
| 205 | params.fence_out.value += current_syncpoint_value; | ||
| 206 | } else { | ||
| 207 | params.fence_out.value = current_syncpoint_value; | ||
| 208 | } | ||
| 209 | gpu.PushGPUEntries(std::move(entries)); | ||
| 210 | 264 | ||
| 211 | std::memcpy(output.data(), ¶ms, output.size()); | 265 | return SubmitGPFIFOImpl(params, output, std::move(entries)); |
| 212 | return 0; | ||
| 213 | } | 266 | } |
| 214 | 267 | ||
| 215 | u32 nvhost_gpu::GetWaitbase(const std::vector<u8>& input, std::vector<u8>& output) { | 268 | u32 nvhost_gpu::GetWaitbase(const std::vector<u8>& input, std::vector<u8>& output) { |
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h index 2ac74743f..a252fc06d 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.h | |||
| @@ -11,6 +11,11 @@ | |||
| 11 | #include "common/swap.h" | 11 | #include "common/swap.h" |
| 12 | #include "core/hle/service/nvdrv/devices/nvdevice.h" | 12 | #include "core/hle/service/nvdrv/devices/nvdevice.h" |
| 13 | #include "core/hle/service/nvdrv/nvdata.h" | 13 | #include "core/hle/service/nvdrv/nvdata.h" |
| 14 | #include "video_core/dma_pusher.h" | ||
| 15 | |||
| 16 | namespace Service::Nvidia { | ||
| 17 | class SyncpointManager; | ||
| 18 | } | ||
| 14 | 19 | ||
| 15 | namespace Service::Nvidia::Devices { | 20 | namespace Service::Nvidia::Devices { |
| 16 | 21 | ||
| @@ -21,7 +26,8 @@ constexpr u32 NVGPU_IOCTL_CHANNEL_KICKOFF_PB(0x1b); | |||
| 21 | 26 | ||
| 22 | class nvhost_gpu final : public nvdevice { | 27 | class nvhost_gpu final : public nvdevice { |
| 23 | public: | 28 | public: |
| 24 | explicit nvhost_gpu(Core::System& system, std::shared_ptr<nvmap> nvmap_dev); | 29 | explicit nvhost_gpu(Core::System& system, std::shared_ptr<nvmap> nvmap_dev, |
| 30 | SyncpointManager& syncpoint_manager); | ||
| 25 | ~nvhost_gpu() override; | 31 | ~nvhost_gpu() override; |
| 26 | 32 | ||
| 27 | u32 ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, | 33 | u32 ioctl(Ioctl command, const std::vector<u8>& input, const std::vector<u8>& input2, |
| @@ -162,10 +168,15 @@ private: | |||
| 162 | u32_le raw; | 168 | u32_le raw; |
| 163 | BitField<0, 1, u32_le> add_wait; // append a wait sync_point to the list | 169 | BitField<0, 1, u32_le> add_wait; // append a wait sync_point to the list |
| 164 | BitField<1, 1, u32_le> add_increment; // append an increment to the list | 170 | BitField<1, 1, u32_le> add_increment; // append an increment to the list |
| 165 | BitField<2, 1, u32_le> new_hw_format; // Mostly ignored | 171 | BitField<2, 1, u32_le> new_hw_format; // mostly ignored |
| 172 | BitField<4, 1, u32_le> suppress_wfi; // suppress wait for interrupt | ||
| 166 | BitField<8, 1, u32_le> increment; // increment the returned fence | 173 | BitField<8, 1, u32_le> increment; // increment the returned fence |
| 167 | } flags; | 174 | } flags; |
| 168 | Fence fence_out; // returned new fence object for others to wait on | 175 | Fence fence_out; // returned new fence object for others to wait on |
| 176 | |||
| 177 | u32 AddIncrementValue() const { | ||
| 178 | return flags.add_increment.Value() << 1; | ||
| 179 | } | ||
| 169 | }; | 180 | }; |
| 170 | static_assert(sizeof(IoctlSubmitGpfifo) == 16 + sizeof(Fence), | 181 | static_assert(sizeof(IoctlSubmitGpfifo) == 16 + sizeof(Fence), |
| 171 | "IoctlSubmitGpfifo is incorrect size"); | 182 | "IoctlSubmitGpfifo is incorrect size"); |
| @@ -190,6 +201,8 @@ private: | |||
| 190 | u32 SetChannelPriority(const std::vector<u8>& input, std::vector<u8>& output); | 201 | u32 SetChannelPriority(const std::vector<u8>& input, std::vector<u8>& output); |
| 191 | u32 AllocGPFIFOEx2(const std::vector<u8>& input, std::vector<u8>& output); | 202 | u32 AllocGPFIFOEx2(const std::vector<u8>& input, std::vector<u8>& output); |
| 192 | u32 AllocateObjectContext(const std::vector<u8>& input, std::vector<u8>& output); | 203 | u32 AllocateObjectContext(const std::vector<u8>& input, std::vector<u8>& output); |
| 204 | u32 SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, std::vector<u8>& output, | ||
| 205 | Tegra::CommandList&& entries); | ||
| 193 | u32 SubmitGPFIFO(const std::vector<u8>& input, std::vector<u8>& output); | 206 | u32 SubmitGPFIFO(const std::vector<u8>& input, std::vector<u8>& output); |
| 194 | u32 KickoffPB(const std::vector<u8>& input, std::vector<u8>& output, | 207 | u32 KickoffPB(const std::vector<u8>& input, std::vector<u8>& output, |
| 195 | const std::vector<u8>& input2, IoctlVersion version); | 208 | const std::vector<u8>& input2, IoctlVersion version); |
| @@ -198,7 +211,8 @@ private: | |||
| 198 | u32 ChannelSetTimeslice(const std::vector<u8>& input, std::vector<u8>& output); | 211 | u32 ChannelSetTimeslice(const std::vector<u8>& input, std::vector<u8>& output); |
| 199 | 212 | ||
| 200 | std::shared_ptr<nvmap> nvmap_dev; | 213 | std::shared_ptr<nvmap> nvmap_dev; |
| 201 | u32 assigned_syncpoints{}; | 214 | SyncpointManager& syncpoint_manager; |
| 215 | Fence channel_fence; | ||
| 202 | }; | 216 | }; |
| 203 | 217 | ||
| 204 | } // namespace Service::Nvidia::Devices | 218 | } // namespace Service::Nvidia::Devices |
diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index 803c1a984..a46755cdc 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp | |||
| @@ -21,6 +21,7 @@ | |||
| 21 | #include "core/hle/service/nvdrv/interface.h" | 21 | #include "core/hle/service/nvdrv/interface.h" |
| 22 | #include "core/hle/service/nvdrv/nvdrv.h" | 22 | #include "core/hle/service/nvdrv/nvdrv.h" |
| 23 | #include "core/hle/service/nvdrv/nvmemp.h" | 23 | #include "core/hle/service/nvdrv/nvmemp.h" |
| 24 | #include "core/hle/service/nvdrv/syncpoint_manager.h" | ||
| 24 | #include "core/hle/service/nvflinger/nvflinger.h" | 25 | #include "core/hle/service/nvflinger/nvflinger.h" |
| 25 | 26 | ||
| 26 | namespace Service::Nvidia { | 27 | namespace Service::Nvidia { |
| @@ -36,21 +37,23 @@ void InstallInterfaces(SM::ServiceManager& service_manager, NVFlinger::NVFlinger | |||
| 36 | nvflinger.SetNVDrvInstance(module_); | 37 | nvflinger.SetNVDrvInstance(module_); |
| 37 | } | 38 | } |
| 38 | 39 | ||
| 39 | Module::Module(Core::System& system) { | 40 | Module::Module(Core::System& system) : syncpoint_manager{system.GPU()} { |
| 40 | auto& kernel = system.Kernel(); | 41 | auto& kernel = system.Kernel(); |
| 41 | for (u32 i = 0; i < MaxNvEvents; i++) { | 42 | for (u32 i = 0; i < MaxNvEvents; i++) { |
| 42 | std::string event_label = fmt::format("NVDRV::NvEvent_{}", i); | 43 | std::string event_label = fmt::format("NVDRV::NvEvent_{}", i); |
| 43 | events_interface.events[i] = Kernel::WritableEvent::CreateEventPair(kernel, event_label); | 44 | events_interface.events[i] = {Kernel::WritableEvent::CreateEventPair(kernel, event_label)}; |
| 44 | events_interface.status[i] = EventState::Free; | 45 | events_interface.status[i] = EventState::Free; |
| 45 | events_interface.registered[i] = false; | 46 | events_interface.registered[i] = false; |
| 46 | } | 47 | } |
| 47 | auto nvmap_dev = std::make_shared<Devices::nvmap>(system); | 48 | auto nvmap_dev = std::make_shared<Devices::nvmap>(system); |
| 48 | devices["/dev/nvhost-as-gpu"] = std::make_shared<Devices::nvhost_as_gpu>(system, nvmap_dev); | 49 | devices["/dev/nvhost-as-gpu"] = std::make_shared<Devices::nvhost_as_gpu>(system, nvmap_dev); |
| 49 | devices["/dev/nvhost-gpu"] = std::make_shared<Devices::nvhost_gpu>(system, nvmap_dev); | 50 | devices["/dev/nvhost-gpu"] = |
| 51 | std::make_shared<Devices::nvhost_gpu>(system, nvmap_dev, syncpoint_manager); | ||
| 50 | devices["/dev/nvhost-ctrl-gpu"] = std::make_shared<Devices::nvhost_ctrl_gpu>(system); | 52 | devices["/dev/nvhost-ctrl-gpu"] = std::make_shared<Devices::nvhost_ctrl_gpu>(system); |
| 51 | devices["/dev/nvmap"] = nvmap_dev; | 53 | devices["/dev/nvmap"] = nvmap_dev; |
| 52 | devices["/dev/nvdisp_disp0"] = std::make_shared<Devices::nvdisp_disp0>(system, nvmap_dev); | 54 | devices["/dev/nvdisp_disp0"] = std::make_shared<Devices::nvdisp_disp0>(system, nvmap_dev); |
| 53 | devices["/dev/nvhost-ctrl"] = std::make_shared<Devices::nvhost_ctrl>(system, events_interface); | 55 | devices["/dev/nvhost-ctrl"] = |
| 56 | std::make_shared<Devices::nvhost_ctrl>(system, events_interface, syncpoint_manager); | ||
| 54 | devices["/dev/nvhost-nvdec"] = std::make_shared<Devices::nvhost_nvdec>(system, nvmap_dev); | 57 | devices["/dev/nvhost-nvdec"] = std::make_shared<Devices::nvhost_nvdec>(system, nvmap_dev); |
| 55 | devices["/dev/nvhost-nvjpg"] = std::make_shared<Devices::nvhost_nvjpg>(system); | 58 | devices["/dev/nvhost-nvjpg"] = std::make_shared<Devices::nvhost_nvjpg>(system); |
| 56 | devices["/dev/nvhost-vic"] = std::make_shared<Devices::nvhost_vic>(system, nvmap_dev); | 59 | devices["/dev/nvhost-vic"] = std::make_shared<Devices::nvhost_vic>(system, nvmap_dev); |
| @@ -95,17 +98,17 @@ void Module::SignalSyncpt(const u32 syncpoint_id, const u32 value) { | |||
| 95 | if (events_interface.assigned_syncpt[i] == syncpoint_id && | 98 | if (events_interface.assigned_syncpt[i] == syncpoint_id && |
| 96 | events_interface.assigned_value[i] == value) { | 99 | events_interface.assigned_value[i] == value) { |
| 97 | events_interface.LiberateEvent(i); | 100 | events_interface.LiberateEvent(i); |
| 98 | events_interface.events[i].writable->Signal(); | 101 | events_interface.events[i].event.writable->Signal(); |
| 99 | } | 102 | } |
| 100 | } | 103 | } |
| 101 | } | 104 | } |
| 102 | 105 | ||
| 103 | std::shared_ptr<Kernel::ReadableEvent> Module::GetEvent(const u32 event_id) const { | 106 | std::shared_ptr<Kernel::ReadableEvent> Module::GetEvent(const u32 event_id) const { |
| 104 | return events_interface.events[event_id].readable; | 107 | return events_interface.events[event_id].event.readable; |
| 105 | } | 108 | } |
| 106 | 109 | ||
| 107 | std::shared_ptr<Kernel::WritableEvent> Module::GetEventWriteable(const u32 event_id) const { | 110 | std::shared_ptr<Kernel::WritableEvent> Module::GetEventWriteable(const u32 event_id) const { |
| 108 | return events_interface.events[event_id].writable; | 111 | return events_interface.events[event_id].event.writable; |
| 109 | } | 112 | } |
| 110 | 113 | ||
| 111 | } // namespace Service::Nvidia | 114 | } // namespace Service::Nvidia |
diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h index 7706a5590..f3d863dac 100644 --- a/src/core/hle/service/nvdrv/nvdrv.h +++ b/src/core/hle/service/nvdrv/nvdrv.h | |||
| @@ -10,6 +10,7 @@ | |||
| 10 | #include "common/common_types.h" | 10 | #include "common/common_types.h" |
| 11 | #include "core/hle/kernel/writable_event.h" | 11 | #include "core/hle/kernel/writable_event.h" |
| 12 | #include "core/hle/service/nvdrv/nvdata.h" | 12 | #include "core/hle/service/nvdrv/nvdata.h" |
| 13 | #include "core/hle/service/nvdrv/syncpoint_manager.h" | ||
| 13 | #include "core/hle/service/service.h" | 14 | #include "core/hle/service/service.h" |
| 14 | 15 | ||
| 15 | namespace Core { | 16 | namespace Core { |
| @@ -22,15 +23,23 @@ class NVFlinger; | |||
| 22 | 23 | ||
| 23 | namespace Service::Nvidia { | 24 | namespace Service::Nvidia { |
| 24 | 25 | ||
| 26 | class SyncpointManager; | ||
| 27 | |||
| 25 | namespace Devices { | 28 | namespace Devices { |
| 26 | class nvdevice; | 29 | class nvdevice; |
| 27 | } | 30 | } |
| 28 | 31 | ||
| 32 | /// Represents an Nvidia event | ||
| 33 | struct NvEvent { | ||
| 34 | Kernel::EventPair event; | ||
| 35 | Fence fence{}; | ||
| 36 | }; | ||
| 37 | |||
| 29 | struct EventInterface { | 38 | struct EventInterface { |
| 30 | // Mask representing currently busy events | 39 | // Mask representing currently busy events |
| 31 | u64 events_mask{}; | 40 | u64 events_mask{}; |
| 32 | // Each kernel event associated to an NV event | 41 | // Each kernel event associated to an NV event |
| 33 | std::array<Kernel::EventPair, MaxNvEvents> events; | 42 | std::array<NvEvent, MaxNvEvents> events; |
| 34 | // The status of the current NVEvent | 43 | // The status of the current NVEvent |
| 35 | std::array<EventState, MaxNvEvents> status{}; | 44 | std::array<EventState, MaxNvEvents> status{}; |
| 36 | // Tells if an NVEvent is registered or not | 45 | // Tells if an NVEvent is registered or not |
| @@ -119,6 +128,9 @@ public: | |||
| 119 | std::shared_ptr<Kernel::WritableEvent> GetEventWriteable(u32 event_id) const; | 128 | std::shared_ptr<Kernel::WritableEvent> GetEventWriteable(u32 event_id) const; |
| 120 | 129 | ||
| 121 | private: | 130 | private: |
| 131 | /// Manages syncpoints on the host | ||
| 132 | SyncpointManager syncpoint_manager; | ||
| 133 | |||
| 122 | /// Id to use for the next open file descriptor. | 134 | /// Id to use for the next open file descriptor. |
| 123 | u32 next_fd = 1; | 135 | u32 next_fd = 1; |
| 124 | 136 | ||
diff --git a/src/core/hle/service/nvdrv/syncpoint_manager.cpp b/src/core/hle/service/nvdrv/syncpoint_manager.cpp new file mode 100644 index 000000000..0151a03b7 --- /dev/null +++ b/src/core/hle/service/nvdrv/syncpoint_manager.cpp | |||
| @@ -0,0 +1,39 @@ | |||
| 1 | // Copyright 2020 yuzu emulator team | ||
| 2 | // Licensed under GPLv2 or any later version | ||
| 3 | // Refer to the license.txt file included. | ||
| 4 | |||
| 5 | #include "common/assert.h" | ||
| 6 | #include "core/hle/service/nvdrv/syncpoint_manager.h" | ||
| 7 | #include "video_core/gpu.h" | ||
| 8 | |||
| 9 | namespace Service::Nvidia { | ||
| 10 | |||
| 11 | SyncpointManager::SyncpointManager(Tegra::GPU& gpu) : gpu{gpu} {} | ||
| 12 | |||
| 13 | SyncpointManager::~SyncpointManager() = default; | ||
| 14 | |||
| 15 | u32 SyncpointManager::RefreshSyncpoint(u32 syncpoint_id) { | ||
| 16 | syncpoints[syncpoint_id].min = gpu.GetSyncpointValue(syncpoint_id); | ||
| 17 | return GetSyncpointMin(syncpoint_id); | ||
| 18 | } | ||
| 19 | |||
| 20 | u32 SyncpointManager::AllocateSyncpoint() { | ||
| 21 | for (u32 syncpoint_id = 1; syncpoint_id < MaxSyncPoints; syncpoint_id++) { | ||
| 22 | if (!syncpoints[syncpoint_id].is_allocated) { | ||
| 23 | syncpoints[syncpoint_id].is_allocated = true; | ||
| 24 | return syncpoint_id; | ||
| 25 | } | ||
| 26 | } | ||
| 27 | UNREACHABLE_MSG("No more available syncpoints!"); | ||
| 28 | return {}; | ||
| 29 | } | ||
| 30 | |||
| 31 | u32 SyncpointManager::IncreaseSyncpoint(u32 syncpoint_id, u32 value) { | ||
| 32 | for (u32 index = 0; index < value; ++index) { | ||
| 33 | syncpoints[syncpoint_id].max.fetch_add(1, std::memory_order_relaxed); | ||
| 34 | } | ||
| 35 | |||
| 36 | return GetSyncpointMax(syncpoint_id); | ||
| 37 | } | ||
| 38 | |||
| 39 | } // namespace Service::Nvidia | ||
diff --git a/src/core/hle/service/nvdrv/syncpoint_manager.h b/src/core/hle/service/nvdrv/syncpoint_manager.h new file mode 100644 index 000000000..4168b6c7e --- /dev/null +++ b/src/core/hle/service/nvdrv/syncpoint_manager.h | |||
| @@ -0,0 +1,85 @@ | |||
| 1 | // Copyright 2020 yuzu emulator team | ||
| 2 | // Licensed under GPLv2 or any later version | ||
| 3 | // Refer to the license.txt file included. | ||
| 4 | |||
| 5 | #pragma once | ||
| 6 | |||
| 7 | #include <array> | ||
| 8 | #include <atomic> | ||
| 9 | |||
| 10 | #include "common/common_types.h" | ||
| 11 | #include "core/hle/service/nvdrv/nvdata.h" | ||
| 12 | |||
| 13 | namespace Tegra { | ||
| 14 | class GPU; | ||
| 15 | } | ||
| 16 | |||
| 17 | namespace Service::Nvidia { | ||
| 18 | |||
| 19 | class SyncpointManager final { | ||
| 20 | public: | ||
| 21 | explicit SyncpointManager(Tegra::GPU& gpu); | ||
| 22 | ~SyncpointManager(); | ||
| 23 | |||
| 24 | /** | ||
| 25 | * Returns true if the specified syncpoint is expired for the given value. | ||
| 26 | * @param syncpoint_id Syncpoint ID to check. | ||
| 27 | * @param value Value to check against the specified syncpoint. | ||
| 28 | * @returns True if the specified syncpoint is expired for the given value, otherwise False. | ||
| 29 | */ | ||
| 30 | bool IsSyncpointExpired(u32 syncpoint_id, u32 value) const { | ||
| 31 | return (GetSyncpointMax(syncpoint_id) - value) >= (GetSyncpointMin(syncpoint_id) - value); | ||
| 32 | } | ||
| 33 | |||
| 34 | /** | ||
| 35 | * Gets the lower bound for the specified syncpoint. | ||
| 36 | * @param syncpoint_id Syncpoint ID to get the lower bound for. | ||
| 37 | * @returns The lower bound for the specified syncpoint. | ||
| 38 | */ | ||
| 39 | u32 GetSyncpointMin(u32 syncpoint_id) const { | ||
| 40 | return syncpoints[syncpoint_id].min.load(std::memory_order_relaxed); | ||
| 41 | } | ||
| 42 | |||
| 43 | /** | ||
| 44 | * Gets the uper bound for the specified syncpoint. | ||
| 45 | * @param syncpoint_id Syncpoint ID to get the upper bound for. | ||
| 46 | * @returns The upper bound for the specified syncpoint. | ||
| 47 | */ | ||
| 48 | u32 GetSyncpointMax(u32 syncpoint_id) const { | ||
| 49 | return syncpoints[syncpoint_id].max.load(std::memory_order_relaxed); | ||
| 50 | } | ||
| 51 | |||
| 52 | /** | ||
| 53 | * Refreshes the minimum value for the specified syncpoint. | ||
| 54 | * @param syncpoint_id Syncpoint ID to be refreshed. | ||
| 55 | * @returns The new syncpoint minimum value. | ||
| 56 | */ | ||
| 57 | u32 RefreshSyncpoint(u32 syncpoint_id); | ||
| 58 | |||
| 59 | /** | ||
| 60 | * Allocates a new syncoint. | ||
| 61 | * @returns The syncpoint ID for the newly allocated syncpoint. | ||
| 62 | */ | ||
| 63 | u32 AllocateSyncpoint(); | ||
| 64 | |||
| 65 | /** | ||
| 66 | * Increases the maximum value for the specified syncpoint. | ||
| 67 | * @param syncpoint_id Syncpoint ID to be increased. | ||
| 68 | * @param value Value to increase the specified syncpoint by. | ||
| 69 | * @returns The new syncpoint maximum value. | ||
| 70 | */ | ||
| 71 | u32 IncreaseSyncpoint(u32 syncpoint_id, u32 value); | ||
| 72 | |||
| 73 | private: | ||
| 74 | struct Syncpoint { | ||
| 75 | std::atomic<u32> min; | ||
| 76 | std::atomic<u32> max; | ||
| 77 | std::atomic<bool> is_allocated; | ||
| 78 | }; | ||
| 79 | |||
| 80 | std::array<Syncpoint, MaxSyncPoints> syncpoints{}; | ||
| 81 | |||
| 82 | Tegra::GPU& gpu; | ||
| 83 | }; | ||
| 84 | |||
| 85 | } // namespace Service::Nvidia | ||
diff --git a/src/core/hle/service/nvflinger/nvflinger.cpp b/src/core/hle/service/nvflinger/nvflinger.cpp index c64673dba..44aa2bdae 100644 --- a/src/core/hle/service/nvflinger/nvflinger.cpp +++ b/src/core/hle/service/nvflinger/nvflinger.cpp | |||
| @@ -242,6 +242,10 @@ void NVFlinger::Compose() { | |||
| 242 | 242 | ||
| 243 | const auto& igbp_buffer = buffer->get().igbp_buffer; | 243 | const auto& igbp_buffer = buffer->get().igbp_buffer; |
| 244 | 244 | ||
| 245 | if (!system.IsPoweredOn()) { | ||
| 246 | return; // We are likely shutting down | ||
| 247 | } | ||
| 248 | |||
| 245 | auto& gpu = system.GPU(); | 249 | auto& gpu = system.GPU(); |
| 246 | const auto& multi_fence = buffer->get().multi_fence; | 250 | const auto& multi_fence = buffer->get().multi_fence; |
| 247 | guard->unlock(); | 251 | guard->unlock(); |