General: Recover Prometheus project from harddrive failure

This commit: Implements CPU Interrupts, Replaces Cycle Timing for Host 
Timing, Reworks the Kernel's Scheduler, Introduce Idle State and 
Suspended State, Recreates the bootmanager, Initializes Multicore 
system.

11 files changed, 679 insertions, 344 deletions

diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index 7655382fa..ba051a7d8 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -13,11 +13,13 @@
 
 #include "common/assert.h"
 #include "common/logging/log.h"
+#include "common/thread.h"
 #include "core/arm/arm_interface.h"
 #include "core/arm/exclusive_monitor.h"
 #include "core/core.h"
 #include "core/core_timing.h"
 #include "core/core_timing_util.h"
+#include "core/cpu_manager.h"
 #include "core/device_memory.h"
 #include "core/hardware_properties.h"
 #include "core/hle/kernel/client_port.h"
@@ -117,7 +119,9 @@ struct KernelCore::Impl {
         InitializeSystemResourceLimit(kernel);
         InitializeMemoryLayout();
         InitializeThreads();
-        InitializePreemption();
+        InitializePreemption(kernel);
+        InitializeSchedulers();
+        InitializeSuspendThreads();
     }
 
     void Shutdown() {
@@ -155,6 +159,12 @@ struct KernelCore::Impl {
         }
     }
 
+    void InitializeSchedulers() {
+        for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+            cores[i].Scheduler().Initialize();
+        }
+    }
+
     // Creates the default system resource limit
     void InitializeSystemResourceLimit(KernelCore& kernel) {
         system_resource_limit = ResourceLimit::Create(kernel);
@@ -178,10 +188,13 @@ struct KernelCore::Impl {
             Core::Timing::CreateEvent("ThreadWakeupCallback", ThreadWakeupCallback);
     }
 
-    void InitializePreemption() {
-        preemption_event =
-            Core::Timing::CreateEvent("PreemptionCallback", [this](u64 userdata, s64 cycles_late) {
-                global_scheduler.PreemptThreads();
+    void InitializePreemption(KernelCore& kernel) {
+        preemption_event = Core::Timing::CreateEvent(
+            "PreemptionCallback", [this, &kernel](u64 userdata, s64 cycles_late) {
+                {
+                    SchedulerLock lock(kernel);
+                    global_scheduler.PreemptThreads();
+                }
                 s64 time_interval = Core::Timing::msToCycles(std::chrono::milliseconds(10));
                 system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
             });
@@ -190,6 +203,20 @@ struct KernelCore::Impl {
         system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
     }
 
+    void InitializeSuspendThreads() {
+        for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+            std::string name = "Suspend Thread Id:" + std::to_string(i);
+            std::function<void(void*)> init_func =
+                system.GetCpuManager().GetSuspendThreadStartFunc();
+            void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+            ThreadType type =
+                static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_SUSPEND);
+            auto thread_res = Thread::Create(system, type, name, 0, 0, 0, static_cast<u32>(i), 0,
+                                             nullptr, std::move(init_func), init_func_parameter);
+            suspend_threads[i] = std::move(thread_res).Unwrap();
+        }
+    }
+
     void MakeCurrentProcess(Process* process) {
         current_process = process;
 
@@ -201,7 +228,10 @@ struct KernelCore::Impl {
             core.SetIs64Bit(process->Is64BitProcess());
         }
 
-        system.Memory().SetCurrentPageTable(*process);
+        u32 core_id = GetCurrentHostThreadID();
+        if (core_id < Core::Hardware::NUM_CPU_CORES) {
+            system.Memory().SetCurrentPageTable(*process, core_id);
+        }
     }
 
     void RegisterCoreThread(std::size_t core_id) {
@@ -219,7 +249,9 @@ struct KernelCore::Impl {
         std::unique_lock lock{register_thread_mutex};
         const std::thread::id this_id = std::this_thread::get_id();
         const auto it = host_thread_ids.find(this_id);
-        ASSERT(it == host_thread_ids.end());
+        if (it != host_thread_ids.end()) {
+            return;
+        }
         host_thread_ids[this_id] = registered_thread_ids++;
     }
 
@@ -343,6 +375,8 @@ struct KernelCore::Impl {
     std::shared_ptr<Kernel::SharedMemory> irs_shared_mem;
     std::shared_ptr<Kernel::SharedMemory> time_shared_mem;
 
+    std::array<std::shared_ptr<Thread>, Core::Hardware::NUM_CPU_CORES> suspend_threads{};
+
     // System context
     Core::System& system;
 };
@@ -412,6 +446,26 @@ const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const {
     return impl->cores[id];
 }
 
+Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() {
+    u32 core_id = impl->GetCurrentHostThreadID();
+    ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
+    return impl->cores[core_id];
+}
+
+const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const {
+    u32 core_id = impl->GetCurrentHostThreadID();
+    ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
+    return impl->cores[core_id];
+}
+
+Kernel::Scheduler& KernelCore::CurrentScheduler() {
+    return CurrentPhysicalCore().Scheduler();
+}
+
+const Kernel::Scheduler& KernelCore::CurrentScheduler() const {
+    return CurrentPhysicalCore().Scheduler();
+}
+
 Kernel::Synchronization& KernelCore::Synchronization() {
     return impl->synchronization;
 }
@@ -557,4 +611,20 @@ const Kernel::SharedMemory& KernelCore::GetTimeSharedMem() const {
     return *impl->time_shared_mem;
 }
 
+void KernelCore::Suspend(bool in_suspention) {
+    const bool should_suspend = exception_exited || in_suspention;
+    {
+        SchedulerLock lock(*this);
+        ThreadStatus status = should_suspend ? ThreadStatus::Ready : ThreadStatus::WaitSleep;
+        for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+            impl->suspend_threads[i]->SetStatus(status);
+        }
+    }
+}
+
+void KernelCore::ExceptionalExit() {
+    exception_exited = true;
+    Suspend(true);
+}
+
 } // namespace Kernel
diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h
index 83de1f542..5d32a8329 100644
--- a/src/core/hle/kernel/kernel.h
+++ b/src/core/hle/kernel/kernel.h
@@ -110,6 +110,18 @@ public:
     /// Gets the an instance of the respective physical CPU core.
     const Kernel::PhysicalCore& PhysicalCore(std::size_t id) const;
 
+    /// Gets the sole instance of the Scheduler at the current running core.
+    Kernel::Scheduler& CurrentScheduler();
+
+    /// Gets the sole instance of the Scheduler at the current running core.
+    const Kernel::Scheduler& CurrentScheduler() const;
+
+    /// Gets the an instance of the current physical CPU core.
+    Kernel::PhysicalCore& CurrentPhysicalCore();
+
+    /// Gets the an instance of the current physical CPU core.
+    const Kernel::PhysicalCore& CurrentPhysicalCore() const;
+
     /// Gets the an instance of the Synchronization Interface.
     Kernel::Synchronization& Synchronization();
 
@@ -191,6 +203,12 @@ public:
     /// Gets the shared memory object for Time services.
     const Kernel::SharedMemory& GetTimeSharedMem() const;
 
+    /// Suspend/unsuspend the OS.
+    void Suspend(bool in_suspention);
+
+    /// Exceptional exit the OS.
+    void ExceptionalExit();
+
 private:
     friend class Object;
     friend class Process;
@@ -219,6 +237,7 @@ private:
 
     struct Impl;
     std::unique_ptr<Impl> impl;
+    bool exception_exited{};
 };
 
 } // namespace Kernel
diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp
index a15011076..69202540b 100644
--- a/src/core/hle/kernel/physical_core.cpp
+++ b/src/core/hle/kernel/physical_core.cpp
@@ -2,12 +2,15 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include "common/assert.h"
 #include "common/logging/log.h"
+#include "common/spin_lock.h"
 #include "core/arm/arm_interface.h"
 #ifdef ARCHITECTURE_x86_64
 #include "core/arm/dynarmic/arm_dynarmic_32.h"
 #include "core/arm/dynarmic/arm_dynarmic_64.h"
 #endif
+#include "core/arm/cpu_interrupt_handler.h"
 #include "core/arm/exclusive_monitor.h"
 #include "core/arm/unicorn/arm_unicorn.h"
 #include "core/core.h"
@@ -19,21 +22,23 @@ namespace Kernel {
 
 PhysicalCore::PhysicalCore(Core::System& system, std::size_t id,
                            Core::ExclusiveMonitor& exclusive_monitor)
-    : core_index{id} {
+    : interrupt_handler{}, core_index{id} {
 #ifdef ARCHITECTURE_x86_64
-    arm_interface_32 =
-        std::make_unique<Core::ARM_Dynarmic_32>(system, exclusive_monitor, core_index);
-    arm_interface_64 =
-        std::make_unique<Core::ARM_Dynarmic_64>(system, exclusive_monitor, core_index);
-
+    arm_interface_32 = std::make_unique<Core::ARM_Dynarmic_32>(system, interrupt_handler,
+                                                               exclusive_monitor, core_index);
+    arm_interface_64 = std::make_unique<Core::ARM_Dynarmic_64>(system, interrupt_handler,
+                                                               exclusive_monitor, core_index);
 #else
     using Core::ARM_Unicorn;
-    arm_interface_32 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch32);
-    arm_interface_64 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch64);
+    arm_interface_32 =
+        std::make_unique<ARM_Unicorn>(system, interrupt_handler, ARM_Unicorn::Arch::AArch32);
+    arm_interface_64 =
+        std::make_unique<ARM_Unicorn>(system, interrupt_handler, ARM_Unicorn::Arch::AArch64);
     LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available");
 #endif
 
     scheduler = std::make_unique<Kernel::Scheduler>(system, core_index);
+    guard = std::make_unique<Common::SpinLock>();
 }
 
 PhysicalCore::~PhysicalCore() = default;
@@ -47,6 +52,10 @@ void PhysicalCore::Step() {
     arm_interface->Step();
 }
 
+void PhysicalCore::Idle() {
+    interrupt_handler.AwaitInterrupt();
+}
+
 void PhysicalCore::Stop() {
     arm_interface->PrepareReschedule();
 }
@@ -63,4 +72,16 @@ void PhysicalCore::SetIs64Bit(bool is_64_bit) {
     }
 }
 
+void PhysicalCore::Interrupt() {
+    guard->lock();
+    interrupt_handler.SetInterrupt(true);
+    guard->unlock();
+}
+
+void PhysicalCore::ClearInterrupt() {
+    guard->lock();
+    interrupt_handler.SetInterrupt(false);
+    guard->unlock();
+}
+
 } // namespace Kernel
diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h
index 3269166be..c3da30b72 100644
--- a/src/core/hle/kernel/physical_core.h
+++ b/src/core/hle/kernel/physical_core.h
@@ -7,6 +7,12 @@
 #include <cstddef>
 #include <memory>
 
+#include "core/arm/cpu_interrupt_handler.h"
+
+namespace Common {
+    class SpinLock;
+}
+
 namespace Kernel {
 class Scheduler;
 } // namespace Kernel
@@ -32,11 +38,24 @@ public:
 
     /// Execute current jit state
     void Run();
+    /// Set this core in IdleState.
+    void Idle();
     /// Execute a single instruction in current jit.
     void Step();
     /// Stop JIT execution/exit
     void Stop();
 
+    /// Interrupt this physical core.
+    void Interrupt();
+
+    /// Clear this core's interrupt
+    void ClearInterrupt();
+
+    /// Check if this core is interrupted
+    bool IsInterrupted() const {
+        return interrupt_handler.IsInterrupted();
+    }
+
     // Shutdown this physical core.
     void Shutdown();
 
@@ -71,11 +90,13 @@ public:
     void SetIs64Bit(bool is_64_bit);
 
 private:
+    Core::CPUInterruptHandler interrupt_handler;
     std::size_t core_index;
     std::unique_ptr<Core::ARM_Interface> arm_interface_32;
     std::unique_ptr<Core::ARM_Interface> arm_interface_64;
     std::unique_ptr<Kernel::Scheduler> scheduler;
     Core::ARM_Interface* arm_interface{};
+    std::unique_ptr<Common::SpinLock> guard;

 };
 
 } // namespace Kernel
diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp
index c4c5199b1..7e26a54f4 100644
--- a/src/core/hle/kernel/process.cpp
+++ b/src/core/hle/kernel/process.cpp
@@ -30,14 +30,15 @@ namespace {
 /**
  * Sets up the primary application thread
  *
+ * @param system The system instance to create the main thread under.
  * @param owner_process The parent process for the main thread
- * @param kernel The kernel instance to create the main thread under.
  * @param priority The priority to give the main thread
  */
-void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, VAddr stack_top) {
+void SetupMainThread(Core::System& system, Process& owner_process, u32 priority, VAddr stack_top) {
     const VAddr entry_point = owner_process.PageTable().GetCodeRegionStart();
-    auto thread_res = Thread::Create(kernel, "main", entry_point, priority, 0,
-                                     owner_process.GetIdealCore(), stack_top, owner_process);
+    ThreadType type = THREADTYPE_USER;
+    auto thread_res = Thread::Create(system, type, "main", entry_point, priority, 0,
+                                     owner_process.GetIdealCore(), stack_top, &owner_process);
 
     std::shared_ptr<Thread> thread = std::move(thread_res).Unwrap();
 
@@ -48,8 +49,12 @@ void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, V
     thread->GetContext32().cpu_registers[1] = thread_handle;
     thread->GetContext64().cpu_registers[1] = thread_handle;
 
+    auto& kernel = system.Kernel();
     // Threads by default are dormant, wake up the main thread so it runs when the scheduler fires
-    thread->ResumeFromWait();
+    {
+        SchedulerLock lock{kernel};
+        thread->SetStatus(ThreadStatus::Ready);
+    }
 }
 } // Anonymous namespace
 
@@ -294,7 +299,7 @@ void Process::Run(s32 main_thread_priority, u64 stack_size) {
 
     ChangeStatus(ProcessStatus::Running);
 
-    SetupMainThread(*this, kernel, main_thread_priority, main_thread_stack_top);
+    SetupMainThread(system, *this, main_thread_priority, main_thread_stack_top);
     resource_limit->Reserve(ResourceType::Threads, 1);
     resource_limit->Reserve(ResourceType::PhysicalMemory, main_thread_stack_size);
 }
diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp
index 1140c72a3..5166020a0 100644
--- a/src/core/hle/kernel/scheduler.cpp
+++ b/src/core/hle/kernel/scheduler.cpp
@@ -11,11 +11,15 @@
 #include <utility>
 
 #include "common/assert.h"
+#include "common/bit_util.h"
+#include "common/fiber.h"
 #include "common/logging/log.h"
 #include "core/arm/arm_interface.h"
 #include "core/core.h"
 #include "core/core_timing.h"
+#include "core/cpu_manager.h"
 #include "core/hle/kernel/kernel.h"
+#include "core/hle/kernel/physical_core.h"
 #include "core/hle/kernel/process.h"
 #include "core/hle/kernel/scheduler.h"
 #include "core/hle/kernel/time_manager.h"
@@ -27,78 +31,108 @@ GlobalScheduler::GlobalScheduler(KernelCore& kernel) : kernel{kernel} {}
 GlobalScheduler::~GlobalScheduler() = default;
 
 void GlobalScheduler::AddThread(std::shared_ptr<Thread> thread) {
+    global_list_guard.lock();
     thread_list.push_back(std::move(thread));
+    global_list_guard.unlock();
 }
 
 void GlobalScheduler::RemoveThread(std::shared_ptr<Thread> thread) {
+    global_list_guard.lock();
     thread_list.erase(std::remove(thread_list.begin(), thread_list.end(), thread),
                       thread_list.end());
+    global_list_guard.unlock();
 }
 
-void GlobalScheduler::UnloadThread(std::size_t core) {
-    Scheduler& sched = kernel.Scheduler(core);
-    sched.UnloadThread();
-}
-
-void GlobalScheduler::SelectThread(std::size_t core) {
+u32 GlobalScheduler::SelectThreads() {
     const auto update_thread = [](Thread* thread, Scheduler& sched) {
+        sched.guard.lock();
         if (thread != sched.selected_thread.get()) {
             if (thread == nullptr) {
                 ++sched.idle_selection_count;
             }
             sched.selected_thread = SharedFrom(thread);
         }
-        sched.is_context_switch_pending = sched.selected_thread != sched.current_thread;
+        const bool reschedule_pending = sched.selected_thread != sched.current_thread;
+        sched.is_context_switch_pending = reschedule_pending;
         std::atomic_thread_fence(std::memory_order_seq_cst);
+        sched.guard.unlock();
+        return reschedule_pending;
     };
-    Scheduler& sched = kernel.Scheduler(core);
-    Thread* current_thread = nullptr;
-    // Step 1: Get top thread in schedule queue.
-    current_thread = scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
-    if (current_thread) {
-        update_thread(current_thread, sched);
-        return;
+    if (!is_reselection_pending.load()) {
+        return 0;
     }
-    // Step 2: Try selecting a suggested thread.
-    Thread* winner = nullptr;
-    std::set<s32> sug_cores;
-    for (auto thread : suggested_queue[core]) {
-        s32 this_core = thread->GetProcessorID();
-        Thread* thread_on_core = nullptr;
-        if (this_core >= 0) {
-            thread_on_core = scheduled_queue[this_core].front();
-        }
-        if (this_core < 0 || thread != thread_on_core) {
-            winner = thread;
-            break;
+    std::array<Thread*, Core::Hardware::NUM_CPU_CORES> top_threads{};
+
+    u32 idle_cores{};
+
+    // Step 1: Get top thread in schedule queue.
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        Thread* top_thread =
+            scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
+        if (top_thread != nullptr) {
+            // TODO(Blinkhawk): Implement Thread Pinning
+        } else {
+            idle_cores |= (1ul << core);
         }
-        sug_cores.insert(this_core);
+        top_threads[core] = top_thread;
     }
-    // if we got a suggested thread, select it, else do a second pass.
-    if (winner && winner->GetPriority() > 2) {
-        if (winner->IsRunning()) {
-            UnloadThread(static_cast<u32>(winner->GetProcessorID()));
+
+    while (idle_cores != 0) {
+        u32 core_id = Common::CountTrailingZeroes32(idle_cores);
+
+        if (!suggested_queue[core_id].empty()) {
+            std::array<s32, Core::Hardware::NUM_CPU_CORES> migration_candidates{};
+            std::size_t num_candidates = 0;
+            auto iter = suggested_queue[core_id].begin();
+            Thread* suggested = nullptr;
+            // Step 2: Try selecting a suggested thread.
+            while (iter != suggested_queue[core_id].end()) {
+                suggested = *iter;
+                iter++;
+                s32 suggested_core_id = suggested->GetProcessorID();
+                Thread* top_thread =
+                    suggested_core_id > 0 ? top_threads[suggested_core_id] : nullptr;
+                if (top_thread != suggested) {
+                    if (top_thread != nullptr &&
+                        top_thread->GetPriority() < THREADPRIO_MAX_CORE_MIGRATION) {
+                        suggested = nullptr;
+                        break;
+                        // There's a too high thread to do core migration, cancel
+                    }
+                    TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), suggested);
+                    break;
+                }
+                migration_candidates[num_candidates++] = suggested_core_id;
+            }
+            // Step 3: Select a suggested thread from another core
+            if (suggested == nullptr) {
+                for (std::size_t i = 0; i < num_candidates; i++) {
+                    s32 candidate_core = migration_candidates[i];
+                    suggested = top_threads[candidate_core];
+                    auto it = scheduled_queue[candidate_core].begin();
+                    it++;
+                    Thread* next = it != scheduled_queue[candidate_core].end() ? *it : nullptr;
+                    if (next != nullptr) {
+                        TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id),
+                                       suggested);
+                        top_threads[candidate_core] = next;
+                        break;
+                    }
+                }
+            }
+            top_threads[core_id] = suggested;
         }
-        TransferToCore(winner->GetPriority(), static_cast<s32>(core), winner);
-        update_thread(winner, sched);
-        return;
+
+        idle_cores &= ~(1ul << core_id);
     }
-    // Step 3: Select a suggested thread from another core
-    for (auto& src_core : sug_cores) {
-        auto it = scheduled_queue[src_core].begin();
-        it++;
-        if (it != scheduled_queue[src_core].end()) {
-            Thread* thread_on_core = scheduled_queue[src_core].front();
-            Thread* to_change = *it;
-            if (thread_on_core->IsRunning() || to_change->IsRunning()) {
-                UnloadThread(static_cast<u32>(src_core));
-            }
-            TransferToCore(thread_on_core->GetPriority(), static_cast<s32>(core), thread_on_core);
-            current_thread = thread_on_core;
-            break;
+    u32 cores_needing_context_switch{};
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        Scheduler& sched = kernel.Scheduler(core);
+        if (update_thread(top_threads[core], sched)) {
+            cores_needing_context_switch |= (1ul << core);
         }
     }
-    update_thread(current_thread, sched);
+    return cores_needing_context_switch;
 }
 
 bool GlobalScheduler::YieldThread(Thread* yielding_thread) {
@@ -153,9 +187,6 @@ bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) {
 
     if (winner != nullptr) {
         if (winner != yielding_thread) {
-            if (winner->IsRunning()) {
-                UnloadThread(static_cast<u32>(winner->GetProcessorID()));
-            }
             TransferToCore(winner->GetPriority(), s32(core_id), winner);
         }
     } else {
@@ -195,9 +226,6 @@ bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread
         }
         if (winner != nullptr) {
             if (winner != yielding_thread) {
-                if (winner->IsRunning()) {
-                    UnloadThread(static_cast<u32>(winner->GetProcessorID()));
-                }
                 TransferToCore(winner->GetPriority(), static_cast<s32>(core_id), winner);
             }
         } else {
@@ -213,7 +241,9 @@ void GlobalScheduler::PreemptThreads() {
         const u32 priority = preemption_priorities[core_id];
 
         if (scheduled_queue[core_id].size(priority) > 0) {
-            scheduled_queue[core_id].front(priority)->IncrementYieldCount();
+            if (scheduled_queue[core_id].size(priority) > 1) {
+                scheduled_queue[core_id].front(priority)->IncrementYieldCount();
+            }
             scheduled_queue[core_id].yield(priority);
             if (scheduled_queue[core_id].size(priority) > 1) {
                 scheduled_queue[core_id].front(priority)->IncrementYieldCount();
@@ -247,9 +277,6 @@ void GlobalScheduler::PreemptThreads() {
         }
 
         if (winner != nullptr) {
-            if (winner->IsRunning()) {
-                UnloadThread(static_cast<u32>(winner->GetProcessorID()));
-            }
             TransferToCore(winner->GetPriority(), s32(core_id), winner);
             current_thread =
                 winner->GetPriority() <= current_thread->GetPriority() ? winner : current_thread;
@@ -280,9 +307,6 @@ void GlobalScheduler::PreemptThreads() {
             }
 
             if (winner != nullptr) {
-                if (winner->IsRunning()) {
-                    UnloadThread(static_cast<u32>(winner->GetProcessorID()));
-                }
                 TransferToCore(winner->GetPriority(), s32(core_id), winner);
                 current_thread = winner;
             }
@@ -292,6 +316,28 @@ void GlobalScheduler::PreemptThreads() {
     }
 }
 
+void GlobalScheduler::EnableInterruptAndSchedule(u32 cores_pending_reschedule,
+                                                 Core::EmuThreadHandle global_thread) {
+    u32 current_core = global_thread.host_handle;
+    bool must_context_switch = global_thread.guest_handle != InvalidHandle &&
+                               (current_core < Core::Hardware::NUM_CPU_CORES);
+    while (cores_pending_reschedule != 0) {
+        u32 core = Common::CountTrailingZeroes32(cores_pending_reschedule);
+        ASSERT(core < Core::Hardware::NUM_CPU_CORES);
+        if (!must_context_switch || core != current_core) {
+            auto& phys_core = kernel.PhysicalCore(core);
+            phys_core.Interrupt();
+        } else {
+            must_context_switch = true;
+        }
+        cores_pending_reschedule &= ~(1ul << core);
+    }
+    if (must_context_switch) {
+        auto& core_scheduler = kernel.CurrentScheduler();
+        core_scheduler.TryDoContextSwitch();
+    }
+}
+
 void GlobalScheduler::Suggest(u32 priority, std::size_t core, Thread* thread) {
     suggested_queue[core].add(thread, priority);
 }
@@ -349,6 +395,108 @@ bool GlobalScheduler::AskForReselectionOrMarkRedundant(Thread* current_thread,
     }
 }
 
+void GlobalScheduler::AdjustSchedulingOnStatus(Thread* thread, u32 old_flags) {
+    if (old_flags == thread->scheduling_state) {
+        return;
+    }
+
+    if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) ==
+        ThreadSchedStatus::Runnable) {
+        // In this case the thread was running, now it's pausing/exitting
+        if (thread->processor_id >= 0) {
+            Unschedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+        }
+
+        for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+            if (core != static_cast<u32>(thread->processor_id) &&
+                ((thread->affinity_mask >> core) & 1) != 0) {
+                Unsuggest(thread->current_priority, core, thread);
+            }
+        }
+    } else if (thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable) {
+        // The thread is now set to running from being stopped
+        if (thread->processor_id >= 0) {
+            Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+        }
+
+        for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+            if (core != static_cast<u32>(thread->processor_id) &&
+                ((thread->affinity_mask >> core) & 1) != 0) {
+                Suggest(thread->current_priority, core, thread);
+            }
+        }
+    }
+
+    SetReselectionPending();
+}
+
+void GlobalScheduler::AdjustSchedulingOnPriority(Thread* thread, u32 old_priority) {
+    if (thread->GetSchedulingStatus() != ThreadSchedStatus::Runnable) {
+        return;
+    }
+    if (thread->processor_id >= 0) {
+        Unschedule(old_priority, static_cast<u32>(thread->processor_id), thread);
+    }
+
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        if (core != static_cast<u32>(thread->processor_id) &&
+            ((thread->affinity_mask >> core) & 1) != 0) {
+            Unsuggest(old_priority, core, thread);
+        }
+    }
+
+    if (thread->processor_id >= 0) {
+        // TODO(Blinkhawk): compare it with current thread running on current core, instead of
+        // checking running
+        if (thread->IsRunning()) {
+            SchedulePrepend(thread->current_priority, static_cast<u32>(thread->processor_id),
+                            thread);
+        } else {
+            Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+        }
+    }
+
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        if (core != static_cast<u32>(thread->processor_id) &&
+            ((thread->affinity_mask >> core) & 1) != 0) {
+            Suggest(thread->current_priority, core, thread);
+        }
+    }
+    thread->IncrementYieldCount();
+    SetReselectionPending();
+}
+
+void GlobalScheduler::AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask,
+                                                 s32 old_core) {
+    if (thread->GetSchedulingStatus() != ThreadSchedStatus::Runnable ||
+        thread->current_priority >= THREADPRIO_COUNT) {
+        return;
+    }
+
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        if (((old_affinity_mask >> core) & 1) != 0) {
+            if (core == static_cast<u32>(old_core)) {
+                Unschedule(thread->current_priority, core, thread);
+            } else {
+                Unsuggest(thread->current_priority, core, thread);
+            }
+        }
+    }
+
+    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+        if (((thread->affinity_mask >> core) & 1) != 0) {
+            if (core == static_cast<u32>(thread->processor_id)) {
+                Schedule(thread->current_priority, core, thread);
+            } else {
+                Suggest(thread->current_priority, core, thread);
+            }
+        }
+    }
+
+    thread->IncrementYieldCount();
+    SetReselectionPending();
+}
+
 void GlobalScheduler::Shutdown() {
     for (std::size_t core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
         scheduled_queue[core].clear();
@@ -374,13 +522,12 @@ void GlobalScheduler::Unlock() {
         ASSERT(scope_lock > 0);
         return;
     }
-    for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
-        SelectThread(i);
-    }
+    u32 cores_pending_reschedule = SelectThreads();
+    Core::EmuThreadHandle leaving_thread = current_owner;
     current_owner = Core::EmuThreadHandle::InvalidHandle();
     scope_lock = 1;
     inner_lock.unlock();
-    // TODO(Blinkhawk): Setup the interrupts and change context on current core.
+    EnableInterruptAndSchedule(cores_pending_reschedule, leaving_thread);
 }
 
 Scheduler::Scheduler(Core::System& system, std::size_t core_id)
@@ -393,56 +540,83 @@ bool Scheduler::HaveReadyThreads() const {
 }
 
 Thread* Scheduler::GetCurrentThread() const {
-    return current_thread.get();
+    if (current_thread) {
+        return current_thread.get();
+    }
+    return idle_thread.get();
 }
 
 Thread* Scheduler::GetSelectedThread() const {
     return selected_thread.get();
 }
 
-void Scheduler::SelectThreads() {
-    system.GlobalScheduler().SelectThread(core_id);
-}
-
 u64 Scheduler::GetLastContextSwitchTicks() const {
     return last_context_switch_time;
 }
 
 void Scheduler::TryDoContextSwitch() {
+    auto& phys_core = system.Kernel().CurrentPhysicalCore();
+    if (phys_core.IsInterrupted()) {
+        phys_core.ClearInterrupt();
+    }
+    guard.lock();
     if (is_context_switch_pending) {
         SwitchContext();
+    } else {
+        guard.unlock();
     }
 }
 
-void Scheduler::UnloadThread() {
-    Thread* const previous_thread = GetCurrentThread();
-    Process* const previous_process = system.Kernel().CurrentProcess();
+void Scheduler::OnThreadStart() {
+    SwitchContextStep2();
+}
 
-    UpdateLastContextSwitchTime(previous_thread, previous_process);
+void Scheduler::SwitchContextStep2() {
+    Thread* previous_thread = current_thread.get();
+    Thread* new_thread = selected_thread.get();
 
-    // Save context for previous thread
-    if (previous_thread) {
-        system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
-        system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
-        // Save the TPIDR_EL0 system register in case it was modified.
-        previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
+    // Load context of new thread
+    Process* const previous_process =
+        previous_thread != nullptr ? previous_thread->GetOwnerProcess() : nullptr;

 
-        if (previous_thread->GetStatus() == ThreadStatus::Running) {
-            // This is only the case when a reschedule is triggered without the current thread
-            // yielding execution (i.e. an event triggered, system core time-sliced, etc)
-            previous_thread->SetStatus(ThreadStatus::Ready);
+    if (new_thread) {
+        new_thread->context_guard.lock();
+        ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id),
+                   "Thread must be assigned to this core.");
+        ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready,
+                   "Thread must be ready to become running.");
+
+        // Cancel any outstanding wakeup events for this thread
+        current_thread = SharedFrom(new_thread);
+        new_thread->SetStatus(ThreadStatus::Running);
+        new_thread->SetIsRunning(true);
+
+        auto* const thread_owner_process = current_thread->GetOwnerProcess();
+        if (previous_process != thread_owner_process && thread_owner_process != nullptr) {
+            system.Kernel().MakeCurrentProcess(thread_owner_process);
         }
-        previous_thread->SetIsRunning(false);
+        if (!new_thread->IsHLEThread()) {
+            auto& cpu_core = system.ArmInterface(core_id);
+            cpu_core.LoadContext(new_thread->GetContext32());
+            cpu_core.LoadContext(new_thread->GetContext64());
+            cpu_core.SetTlsAddress(new_thread->GetTLSAddress());
+            cpu_core.SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
+        }
+    } else {
+        current_thread = nullptr;
+        // Note: We do not reset the current process and current page table when idling because
+        // technically we haven't changed processes, our threads are just paused.
     }
-    current_thread = nullptr;
+    guard.unlock();
 }
 
 void Scheduler::SwitchContext() {
-    Thread* const previous_thread = GetCurrentThread();
-    Thread* const new_thread = GetSelectedThread();
+    Thread* previous_thread = current_thread.get();
+    Thread* new_thread = selected_thread.get();
 
     is_context_switch_pending = false;
     if (new_thread == previous_thread) {
+        guard.unlock();
         return;
     }
 
@@ -452,51 +626,44 @@ void Scheduler::SwitchContext() {
 
     // Save context for previous thread
     if (previous_thread) {
-        system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
-        system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
-        // Save the TPIDR_EL0 system register in case it was modified.
-        previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
+        if (!previous_thread->IsHLEThread()) {
+            auto& cpu_core = system.ArmInterface(core_id);
+            cpu_core.SaveContext(previous_thread->GetContext32());
+            cpu_core.SaveContext(previous_thread->GetContext64());
+            // Save the TPIDR_EL0 system register in case it was modified.
+            previous_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());

 
+        }
         if (previous_thread->GetStatus() == ThreadStatus::Running) {
-            // This is only the case when a reschedule is triggered without the current thread
-            // yielding execution (i.e. an event triggered, system core time-sliced, etc)
             previous_thread->SetStatus(ThreadStatus::Ready);
         }
         previous_thread->SetIsRunning(false);
+        previous_thread->context_guard.unlock();
     }
 
-    // Load context of new thread
-    if (new_thread) {
-        ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id),
-                   "Thread must be assigned to this core.");
-        ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready,
-                   "Thread must be ready to become running.");
-
-        // Cancel any outstanding wakeup events for this thread
-        new_thread->CancelWakeupTimer();
-        current_thread = SharedFrom(new_thread);
-        new_thread->SetStatus(ThreadStatus::Running);
-        new_thread->SetIsRunning(true);
-
-        auto* const thread_owner_process = current_thread->GetOwnerProcess();
-        if (previous_process != thread_owner_process) {
-            system.Kernel().MakeCurrentProcess(thread_owner_process);
-        }
+    std::shared_ptr<Common::Fiber> old_context;
+    if (previous_thread != nullptr) {
+        old_context = previous_thread->GetHostContext();
+    } else {
+        old_context = idle_thread->GetHostContext();
+    }
 
-        system.ArmInterface(core_id).LoadContext(new_thread->GetContext32());
-        system.ArmInterface(core_id).LoadContext(new_thread->GetContext64());
-        system.ArmInterface(core_id).SetTlsAddress(new_thread->GetTLSAddress());
-        system.ArmInterface(core_id).SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
+    std::shared_ptr<Common::Fiber> next_context;
+    if (new_thread != nullptr) {
+        next_context = new_thread->GetHostContext();

     } else {
-        current_thread = nullptr;
-        // Note: We do not reset the current process and current page table when idling because
-        // technically we haven't changed processes, our threads are just paused.
+        next_context = idle_thread->GetHostContext();
     }
+
+    Common::Fiber::YieldTo(old_context, next_context);
+    /// When a thread wakes up, the scheduler may have changed to other in another core.
+    auto& next_scheduler = system.Kernel().CurrentScheduler();
+    next_scheduler.SwitchContextStep2();
 }
 
 void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
     const u64 prev_switch_ticks = last_context_switch_time;
-    const u64 most_recent_switch_ticks = system.CoreTiming().GetTicks();
+    const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks();
     const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks;
 
     if (thread != nullptr) {
@@ -510,6 +677,16 @@ void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
     last_context_switch_time = most_recent_switch_ticks;
 }
 
+void Scheduler::Initialize() {
+    std::string name = "Idle Thread Id:" + std::to_string(core_id);
+    std::function<void(void*)> init_func = system.GetCpuManager().GetIdleThreadStartFunc();
+    void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+    ThreadType type = static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_IDLE);
+    auto thread_res = Thread::Create(system, type, name, 0, 64, 0, static_cast<u32>(core_id), 0,
+                                     nullptr, std::move(init_func), init_func_parameter);
+    idle_thread = std::move(thread_res).Unwrap();
+}
+
 void Scheduler::Shutdown() {
     current_thread = nullptr;
     selected_thread = nullptr;
diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/scheduler.h
index 07df33f9c..16655b03f 100644
--- a/src/core/hle/kernel/scheduler.h
+++ b/src/core/hle/kernel/scheduler.h
@@ -11,6 +11,7 @@
 
 #include "common/common_types.h"
 #include "common/multi_level_queue.h"
+#include "common/spin_lock.h"
 #include "core/hardware_properties.h"
 #include "core/hle/kernel/thread.h"
 
@@ -41,41 +42,17 @@ public:
         return thread_list;
     }
 
-    /**
-     * Add a thread to the suggested queue of a cpu core. Suggested threads may be
-     * picked if no thread is scheduled to run on the core.
-     */
-    void Suggest(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Remove a thread to the suggested queue of a cpu core. Suggested threads may be
-     * picked if no thread is scheduled to run on the core.
-     */
-    void Unsuggest(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
-     * back the queue in its priority level.
-     */
-    void Schedule(u32 priority, std::size_t core, Thread* thread);
+    /// Notify the scheduler a thread's status has changed.
+    void AdjustSchedulingOnStatus(Thread* thread, u32 old_flags);
 
-    /**
-     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
-     * front the queue in its priority level.
-     */
-    void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
+    /// Notify the scheduler a thread's priority has changed.
+    void AdjustSchedulingOnPriority(Thread* thread, u32 old_priority);
 
-    /// Reschedule an already scheduled thread based on a new priority
-    void Reschedule(u32 priority, std::size_t core, Thread* thread);
-
-    /// Unschedules a thread.
-    void Unschedule(u32 priority, std::size_t core, Thread* thread);
-
-    /// Selects a core and forces it to unload its current thread's context
-    void UnloadThread(std::size_t core);
+    /// Notify the scheduler a thread's core and/or affinity mask has changed.
+    void AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, s32 old_core);
 
     /**
-     * Takes care of selecting the new scheduled thread in three steps:
+     * Takes care of selecting the new scheduled threads in three steps:
      *
      * 1. First a thread is selected from the top of the priority queue. If no thread
      *    is obtained then we move to step two, else we are done.
@@ -85,8 +62,10 @@ public:
      *
      * 3. Third is no suggested thread is found, we do a second pass and pick a running
      *    thread in another core and swap it with its current thread.
+     *
+     * returns the cores needing scheduling.
      */
-    void SelectThread(std::size_t core);
+    u32 SelectThreads();
 
     bool HaveReadyThreads(std::size_t core_id) const {
         return !scheduled_queue[core_id].empty();
@@ -149,6 +128,39 @@ private:
     /// Unlocks the scheduler, reselects threads, interrupts cores for rescheduling
     /// and reschedules current core if needed.
     void Unlock();
+
+    void EnableInterruptAndSchedule(u32 cores_pending_reschedule, Core::EmuThreadHandle global_thread);
+
+    /**
+     * Add a thread to the suggested queue of a cpu core. Suggested threads may be
+     * picked if no thread is scheduled to run on the core.
+     */
+    void Suggest(u32 priority, std::size_t core, Thread* thread);
+
+    /**
+     * Remove a thread to the suggested queue of a cpu core. Suggested threads may be
+     * picked if no thread is scheduled to run on the core.
+     */
+    void Unsuggest(u32 priority, std::size_t core, Thread* thread);
+
+    /**
+     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
+     * back the queue in its priority level.
+     */
+    void Schedule(u32 priority, std::size_t core, Thread* thread);
+
+    /**
+     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
+     * front the queue in its priority level.
+     */
+    void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
+
+    /// Reschedule an already scheduled thread based on a new priority
+    void Reschedule(u32 priority, std::size_t core, Thread* thread);
+
+    /// Unschedules a thread.
+    void Unschedule(u32 priority, std::size_t core, Thread* thread);
+
     /**
      * Transfers a thread into an specific core. If the destination_core is -1
      * it will be unscheduled from its source code and added into its suggested
@@ -174,6 +186,8 @@ private:
     std::atomic<s64> scope_lock{};
     Core::EmuThreadHandle current_owner{Core::EmuThreadHandle::InvalidHandle()};
 
+    Common::SpinLock global_list_guard{};
+
     /// Lists all thread ids that aren't deleted/etc.
     std::vector<std::shared_ptr<Thread>> thread_list;
     KernelCore& kernel;
@@ -190,12 +204,6 @@ public:
     /// Reschedules to the next available thread (call after current thread is suspended)
     void TryDoContextSwitch();
 
-    /// Unloads currently running thread
-    void UnloadThread();
-
-    /// Select the threads in top of the scheduling multilist.
-    void SelectThreads();
-
     /// Gets the current running thread
     Thread* GetCurrentThread() const;
 
@@ -209,15 +217,22 @@ public:
         return is_context_switch_pending;
     }
 
+    void Initialize();
+
     /// Shutdowns the scheduler.
     void Shutdown();
 
+    void OnThreadStart();
+
 private:
     friend class GlobalScheduler;
 
     /// Switches the CPU's active thread context to that of the specified thread
     void SwitchContext();
 
+    /// When a thread wakes up, it must run this through it's new scheduler
+    void SwitchContextStep2();
+
     /**
      * Called on every context switch to update the internal timestamp
      * This also updates the running time ticks for the given thread and
@@ -233,12 +248,15 @@ private:
 
     std::shared_ptr<Thread> current_thread = nullptr;
     std::shared_ptr<Thread> selected_thread = nullptr;
+    std::shared_ptr<Thread> idle_thread = nullptr;
 
     Core::System& system;
     u64 last_context_switch_time = 0;
     u64 idle_selection_count = 0;
     const std::size_t core_id;
 
+    Common::SpinLock guard{};
+
     bool is_context_switch_pending = false;
 };
 
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 4ae4529f5..d7f0dcabd 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -863,9 +863,9 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha
         if (same_thread && info_sub_id == 0xFFFFFFFFFFFFFFFF) {
             const u64 thread_ticks = current_thread->GetTotalCPUTimeTicks();
 
-            out_ticks = thread_ticks + (core_timing.GetTicks() - prev_ctx_ticks);
+            out_ticks = thread_ticks + (core_timing.GetCPUTicks() - prev_ctx_ticks);
         } else if (same_thread && info_sub_id == system.CurrentCoreIndex()) {
-            out_ticks = core_timing.GetTicks() - prev_ctx_ticks;
+            out_ticks = core_timing.GetCPUTicks() - prev_ctx_ticks;
         }
 
         *result = out_ticks;
@@ -1428,9 +1428,10 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e
 
     ASSERT(kernel.CurrentProcess()->GetResourceLimit()->Reserve(ResourceType::Threads, 1));
 
+    ThreadType type = THREADTYPE_USER;
     CASCADE_RESULT(std::shared_ptr<Thread> thread,
-                   Thread::Create(kernel, "", entry_point, priority, arg, processor_id, stack_top,
-                                  *current_process));
+                   Thread::Create(system, type, "", entry_point, priority, arg, processor_id, stack_top,
+                                  current_process));
 
     const auto new_thread_handle = current_process->GetHandleTable().Create(thread);
     if (new_thread_handle.Failed()) {
@@ -1513,13 +1514,6 @@ static void SleepThread(Core::System& system, s64 nanoseconds) {
     } else {
         current_thread->Sleep(nanoseconds);
     }
-
-    if (is_redundant) {
-        // If it's redundant, the core is pretty much idle. Some games keep idling
-        // a core while it's doing nothing, we advance timing to avoid costly continuous
-        // calls.
-        system.CoreTiming().AddTicks(2000);
-    }
     system.PrepareReschedule(current_thread->GetProcessorID());
 }
 
@@ -1725,10 +1719,7 @@ static u64 GetSystemTick(Core::System& system) {
     auto& core_timing = system.CoreTiming();
 
     // Returns the value of cntpct_el0 (https://switchbrew.org/wiki/SVC#svcGetSystemTick)
-    const u64 result{Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks())};
-
-    // Advance time to defeat dumb games that busy-wait for the frame to end.
-    core_timing.AddTicks(400);
+    const u64 result{system.CoreTiming().GetClockTicks()};
 
     return result;
 }
diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp
index db7f379ac..8cb3593db 100644
--- a/src/core/hle/kernel/thread.cpp
+++ b/src/core/hle/kernel/thread.cpp
@@ -9,12 +9,14 @@
 
 #include "common/assert.h"
 #include "common/common_types.h"
+#include "common/fiber.h"
 #include "common/logging/log.h"
 #include "common/thread_queue_list.h"
 #include "core/arm/arm_interface.h"
 #include "core/core.h"
 #include "core/core_timing.h"
 #include "core/core_timing_util.h"
+#include "core/cpu_manager.h"
 #include "core/hardware_properties.h"
 #include "core/hle/kernel/errors.h"
 #include "core/hle/kernel/handle_table.h"
@@ -23,6 +25,7 @@
 #include "core/hle/kernel/process.h"
 #include "core/hle/kernel/scheduler.h"
 #include "core/hle/kernel/thread.h"
+#include "core/hle/kernel/time_manager.h"
 #include "core/hle/result.h"
 #include "core/memory.h"
 
@@ -44,6 +47,7 @@ Thread::Thread(KernelCore& kernel) : SynchronizationObject{kernel} {}
 Thread::~Thread() = default;
 
 void Thread::Stop() {
+    SchedulerLock lock(kernel);
     // Cancel any outstanding wakeup events for this thread
     Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(),
                                                              global_handle);
@@ -71,9 +75,8 @@ void Thread::WakeAfterDelay(s64 nanoseconds) {
 
     // This function might be called from any thread so we have to be cautious and use the
     // thread-safe version of ScheduleEvent.
-    const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds});
     Core::System::GetInstance().CoreTiming().ScheduleEvent(
-        cycles, kernel.ThreadWakeupCallbackEventType(), global_handle);
+        nanoseconds, kernel.ThreadWakeupCallbackEventType(), global_handle);
 }
 
 void Thread::CancelWakeupTimer() {
@@ -125,6 +128,16 @@ void Thread::ResumeFromWait() {
     SetStatus(ThreadStatus::Ready);
 }
 
+void Thread::OnWakeUp() {
+    SchedulerLock lock(kernel);
+    if (activity == ThreadActivity::Paused) {
+        SetStatus(ThreadStatus::Paused);
+        return;
+    }
+
+    SetStatus(ThreadStatus::Ready);
+}
+
 void Thread::CancelWait() {
     if (GetSchedulingStatus() != ThreadSchedStatus::Paused) {
         is_sync_cancelled = true;
@@ -153,12 +166,29 @@ static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context,
     context.fpcr = 0;
 }
 
-ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::string name,
-                                                  VAddr entry_point, u32 priority, u64 arg,
-                                                  s32 processor_id, VAddr stack_top,
-                                                  Process& owner_process) {
+std::shared_ptr<Common::Fiber> Thread::GetHostContext() const {
+    return host_context;
+}
+
+ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
+                                                  std::string name, VAddr entry_point, u32 priority,
+                                                  u64 arg, s32 processor_id, VAddr stack_top,
+                                                  Process* owner_process) {
+    std::function<void(void*)> init_func = system.GetCpuManager().GetGuestThreadStartFunc();
+    void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+    return Create(system, type_flags, name, entry_point, priority, arg, processor_id, stack_top,
+                  owner_process, std::move(init_func), init_func_parameter);
+}
+
+ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
+                                                  std::string name, VAddr entry_point, u32 priority,
+                                                  u64 arg, s32 processor_id, VAddr stack_top,
+                                                  Process* owner_process,
+                                                  std::function<void(void*)>&& thread_start_func,
+                                                  void* thread_start_parameter) {
+    auto& kernel = system.Kernel();
     // Check if priority is in ranged. Lowest priority -> highest priority id.
-    if (priority > THREADPRIO_LOWEST) {
+    if (priority > THREADPRIO_LOWEST && (type_flags & THREADTYPE_IDLE == 0)) {
         LOG_ERROR(Kernel_SVC, "Invalid thread priority: {}", priority);
         return ERR_INVALID_THREAD_PRIORITY;
     }
@@ -168,11 +198,12 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
         return ERR_INVALID_PROCESSOR_ID;
     }
 
-    auto& system = Core::System::GetInstance();
-    if (!system.Memory().IsValidVirtualAddress(owner_process, entry_point)) {
-        LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
-        // TODO (bunnei): Find the correct error code to use here
-        return RESULT_UNKNOWN;
+    if (owner_process) {
+        if (!system.Memory().IsValidVirtualAddress(*owner_process, entry_point)) {
+            LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
+            // TODO (bunnei): Find the correct error code to use here
+            return RESULT_UNKNOWN;
+        }
     }
 
     std::shared_ptr<Thread> thread = std::make_shared<Thread>(kernel);
@@ -183,7 +214,7 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
     thread->stack_top = stack_top;
     thread->tpidr_el0 = 0;
     thread->nominal_priority = thread->current_priority = priority;
-    thread->last_running_ticks = system.CoreTiming().GetTicks();
+    thread->last_running_ticks = 0;
     thread->processor_id = processor_id;
     thread->ideal_core = processor_id;
     thread->affinity_mask = 1ULL << processor_id;
@@ -193,16 +224,27 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
     thread->wait_handle = 0;
     thread->name = std::move(name);
     thread->global_handle = kernel.GlobalHandleTable().Create(thread).Unwrap();
-    thread->owner_process = &owner_process;
-    auto& scheduler = kernel.GlobalScheduler();
-    scheduler.AddThread(thread);
-    thread->tls_address = thread->owner_process->CreateTLSRegion();
-
-    thread->owner_process->RegisterThread(thread.get());
-
-    ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
-                         static_cast<u32>(entry_point), static_cast<u32>(arg));
-    ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
+    thread->owner_process = owner_process;
+    thread->type = type_flags;
+    if ((type_flags & THREADTYPE_IDLE) == 0) {
+        auto& scheduler = kernel.GlobalScheduler();
+        scheduler.AddThread(thread);
+    }
+    if (owner_process) {
+        thread->tls_address = thread->owner_process->CreateTLSRegion();
+        thread->owner_process->RegisterThread(thread.get());
+    } else {
+        thread->tls_address = 0;
+    }
+    // TODO(peachum): move to ScheduleThread() when scheduler is added so selected core is used
+    // to initialize the context
+    if ((type_flags & THREADTYPE_HLE) == 0) {
+        ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
+                             static_cast<u32>(entry_point), static_cast<u32>(arg));
+        ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
+    }
+    thread->host_context =
+        std::make_shared<Common::Fiber>(std::move(thread_start_func), thread_start_parameter);

 
     return MakeResult<std::shared_ptr<Thread>>(std::move(thread));
 }
@@ -258,7 +300,7 @@ void Thread::SetStatus(ThreadStatus new_status) {
     }
 
     if (status == ThreadStatus::Running) {
-        last_running_ticks = Core::System::GetInstance().CoreTiming().GetTicks();
+        last_running_ticks = Core::System::GetInstance().CoreTiming().GetCPUTicks();
     }
 
     status = new_status;
@@ -375,38 +417,55 @@ void Thread::SetActivity(ThreadActivity value) {
 }
 
 void Thread::Sleep(s64 nanoseconds) {
-    // Sleep current thread and check for next thread to schedule
-    SetStatus(ThreadStatus::WaitSleep);
+    Handle event_handle{};
+    {
+        SchedulerLockAndSleep lock(kernel, event_handle, this, nanoseconds);
+        SetStatus(ThreadStatus::WaitSleep);
+    }
 
-    // Create an event to wake the thread up after the specified nanosecond delay has passed
-    WakeAfterDelay(nanoseconds);
+    if (event_handle != InvalidHandle) {
+        auto& time_manager = kernel.TimeManager();
+        time_manager.UnscheduleTimeEvent(event_handle);
+    }
 }
 
 bool Thread::YieldSimple() {
-    auto& scheduler = kernel.GlobalScheduler();
-    return scheduler.YieldThread(this);
+    bool result{};
+    {
+        SchedulerLock lock(kernel);
+        result = kernel.GlobalScheduler().YieldThread(this);
+    }
+    return result;
 }
 
 bool Thread::YieldAndBalanceLoad() {
-    auto& scheduler = kernel.GlobalScheduler();
-    return scheduler.YieldThreadAndBalanceLoad(this);
+    bool result{};
+    {
+        SchedulerLock lock(kernel);
+        result = kernel.GlobalScheduler().YieldThreadAndBalanceLoad(this);
+    }
+    return result;
 }
 
 bool Thread::YieldAndWaitForLoadBalancing() {
-    auto& scheduler = kernel.GlobalScheduler();
-    return scheduler.YieldThreadAndWaitForLoadBalancing(this);
+    bool result{};
+    {
+        SchedulerLock lock(kernel);
+        result = kernel.GlobalScheduler().YieldThreadAndWaitForLoadBalancing(this);
+    }
+    return result;
 }
 
 void Thread::SetSchedulingStatus(ThreadSchedStatus new_status) {
     const u32 old_flags = scheduling_state;
     scheduling_state = (scheduling_state & static_cast<u32>(ThreadSchedMasks::HighMask)) |
                        static_cast<u32>(new_status);
-    AdjustSchedulingOnStatus(old_flags);
+    kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_flags);
 }
 
 void Thread::SetCurrentPriority(u32 new_priority) {
     const u32 old_priority = std::exchange(current_priority, new_priority);
-    AdjustSchedulingOnPriority(old_priority);
+    kernel.GlobalScheduler().AdjustSchedulingOnPriority(this, old_priority);
 }
 
 ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
@@ -443,111 +502,12 @@ ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
                     processor_id = ideal_core;
                 }
             }
-            AdjustSchedulingOnAffinity(old_affinity_mask, old_core);
+            kernel.GlobalScheduler().AdjustSchedulingOnAffinity(this, old_affinity_mask, old_core);
         }
     }
     return RESULT_SUCCESS;
 }
 
-void Thread::AdjustSchedulingOnStatus(u32 old_flags) {
-    if (old_flags == scheduling_state) {
-        return;
-    }
-
-    auto& scheduler = kernel.GlobalScheduler();
-    if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) ==
-        ThreadSchedStatus::Runnable) {
-        // In this case the thread was running, now it's pausing/exitting
-        if (processor_id >= 0) {
-            scheduler.Unschedule(current_priority, static_cast<u32>(processor_id), this);
-        }
-
-        for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-            if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
-                scheduler.Unsuggest(current_priority, core, this);
-            }
-        }
-    } else if (GetSchedulingStatus() == ThreadSchedStatus::Runnable) {
-        // The thread is now set to running from being stopped
-        if (processor_id >= 0) {
-            scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
-        }
-
-        for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-            if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
-                scheduler.Suggest(current_priority, core, this);
-            }
-        }
-    }
-
-    scheduler.SetReselectionPending();
-}
-
-void Thread::AdjustSchedulingOnPriority(u32 old_priority) {
-    if (GetSchedulingStatus() != ThreadSchedStatus::Runnable) {
-        return;
-    }
-    auto& scheduler = kernel.GlobalScheduler();
-    if (processor_id >= 0) {
-        scheduler.Unschedule(old_priority, static_cast<u32>(processor_id), this);
-    }
-
-    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-        if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
-            scheduler.Unsuggest(old_priority, core, this);
-        }
-    }
-
-    // Add thread to the new priority queues.
-    Thread* current_thread = GetCurrentThread();
-
-    if (processor_id >= 0) {
-        if (current_thread == this) {
-            scheduler.SchedulePrepend(current_priority, static_cast<u32>(processor_id), this);
-        } else {
-            scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
-        }
-    }
-
-    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-        if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
-            scheduler.Suggest(current_priority, core, this);
-        }
-    }
-
-    scheduler.SetReselectionPending();
-}
-
-void Thread::AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core) {
-    auto& scheduler = kernel.GlobalScheduler();
-    if (GetSchedulingStatus() != ThreadSchedStatus::Runnable ||
-        current_priority >= THREADPRIO_COUNT) {
-        return;
-    }
-
-    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-        if (((old_affinity_mask >> core) & 1) != 0) {
-            if (core == static_cast<u32>(old_core)) {
-                scheduler.Unschedule(current_priority, core, this);
-            } else {
-                scheduler.Unsuggest(current_priority, core, this);
-            }
-        }
-    }
-
-    for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
-        if (((affinity_mask >> core) & 1) != 0) {
-            if (core == static_cast<u32>(processor_id)) {
-                scheduler.Schedule(current_priority, core, this);
-            } else {
-                scheduler.Suggest(current_priority, core, this);
-            }
-        }
-    }
-
-    scheduler.SetReselectionPending();
-}
-
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /**
diff --git a/src/core/hle/kernel/thread.h b/src/core/hle/kernel/thread.h
index 23fdef8a4..33d340b47 100644
--- a/src/core/hle/kernel/thread.h
+++ b/src/core/hle/kernel/thread.h
@@ -9,23 +9,42 @@
 #include <vector>
 
 #include "common/common_types.h"
+#include "common/spin_lock.h"
 #include "core/arm/arm_interface.h"
 #include "core/hle/kernel/object.h"
 #include "core/hle/kernel/synchronization_object.h"
 #include "core/hle/result.h"
 
+namespace Common {
+class Fiber;
+}
+
+namespace Core {
+class System;
+}
+
 namespace Kernel {
 
+class GlobalScheduler;
 class KernelCore;
 class Process;
 class Scheduler;
 
 enum ThreadPriority : u32 {
-    THREADPRIO_HIGHEST = 0,       ///< Highest thread priority
-    THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps
-    THREADPRIO_DEFAULT = 44,      ///< Default thread priority for userland apps
-    THREADPRIO_LOWEST = 63,       ///< Lowest thread priority
-    THREADPRIO_COUNT = 64,        ///< Total number of possible thread priorities.
+    THREADPRIO_HIGHEST = 0,             ///< Highest thread priority
+    THREADPRIO_MAX_CORE_MIGRATION = 2,  ///< Highest priority for a core migration
+    THREADPRIO_USERLAND_MAX = 24,       ///< Highest thread priority for userland apps
+    THREADPRIO_DEFAULT = 44,            ///< Default thread priority for userland apps
+    THREADPRIO_LOWEST = 63,             ///< Lowest thread priority
+    THREADPRIO_COUNT = 64,              ///< Total number of possible thread priorities.
+};
+
+enum ThreadType : u32 {
+    THREADTYPE_USER = 0x1,
+    THREADTYPE_KERNEL = 0x2,
+    THREADTYPE_HLE = 0x4,
+    THREADTYPE_IDLE = 0x8,
+    THREADTYPE_SUSPEND = 0x10,
 };
 
 enum ThreadProcessorId : s32 {
@@ -111,22 +130,43 @@ public:
         std::function<bool(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
                            std::shared_ptr<SynchronizationObject> object, std::size_t index)>;
 
+   /**
+    * Creates and returns a new thread. The new thread is immediately scheduled
+    * @param system The instance of the whole system
+    * @param name The friendly name desired for the thread
+    * @param entry_point The address at which the thread should start execution
+    * @param priority The thread's priority
+    * @param arg User data to pass to the thread
+    * @param processor_id The ID(s) of the processors on which the thread is desired to be run
+    * @param stack_top The address of the thread's stack top
+    * @param owner_process The parent process for the thread, if null, it's a kernel thread
+    * @return A shared pointer to the newly created thread
+    */
+   static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, std::string name,
+                                                    VAddr entry_point, u32 priority, u64 arg,
+                                                    s32 processor_id, VAddr stack_top,
+                                                    Process* owner_process);
+
     /**
      * Creates and returns a new thread. The new thread is immediately scheduled
-     * @param kernel The kernel instance this thread will be created under.
+     * @param system The instance of the whole system
      * @param name The friendly name desired for the thread
      * @param entry_point The address at which the thread should start execution
      * @param priority The thread's priority
      * @param arg User data to pass to the thread
      * @param processor_id The ID(s) of the processors on which the thread is desired to be run
      * @param stack_top The address of the thread's stack top
-     * @param owner_process The parent process for the thread
+     * @param owner_process The parent process for the thread, if null, it's a kernel thread
+     * @param thread_start_func The function where the host context will start.
+     * @param thread_start_parameter The parameter which will passed to host context on init
      * @return A shared pointer to the newly created thread
      */
-    static ResultVal<std::shared_ptr<Thread>> Create(KernelCore& kernel, std::string name,
+    static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, std::string name,
                                                      VAddr entry_point, u32 priority, u64 arg,
                                                      s32 processor_id, VAddr stack_top,
-                                                     Process& owner_process);
+                                                     Process* owner_process,
+                                                     std::function<void(void*)>&& thread_start_func,
+                                                     void* thread_start_parameter);
 
     std::string GetName() const override {
         return name;
@@ -192,7 +232,9 @@ public:
     }
 
     /// Resumes a thread from waiting
-    void ResumeFromWait();
+    void /* deprecated */ ResumeFromWait();
+
+    void OnWakeUp();
 
     /// Cancels a waiting operation that this thread may or may not be within.
     ///
@@ -206,10 +248,10 @@ public:
      * Schedules an event to wake up the specified thread after the specified delay
      * @param nanoseconds The time this thread will be allowed to sleep for
      */
-    void WakeAfterDelay(s64 nanoseconds);
+    void /* deprecated */ WakeAfterDelay(s64 nanoseconds);
 
     /// Cancel any outstanding wakeup events for this thread
-    void CancelWakeupTimer();
+    void /* deprecated */ CancelWakeupTimer();
 
     /**
      * Sets the result after the thread awakens (from svcWaitSynchronization)
@@ -290,6 +332,12 @@ public:
         return context_64;
     }
 
+    bool IsHLEThread() const {
+        return (type & THREADTYPE_HLE) != 0;
+    }
+
+    std::shared_ptr<Common::Fiber> GetHostContext() const;
+
     ThreadStatus GetStatus() const {
         return status;
     }
@@ -467,16 +515,19 @@ public:
     }
 
 private:
+    friend class GlobalScheduler;
+    friend class Scheduler;
+
     void SetSchedulingStatus(ThreadSchedStatus new_status);
     void SetCurrentPriority(u32 new_priority);
     ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
 
-    void AdjustSchedulingOnStatus(u32 old_flags);
-    void AdjustSchedulingOnPriority(u32 old_priority);
     void AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core);
 
     ThreadContext32 context_32{};
     ThreadContext64 context_64{};
+    Common::SpinLock context_guard{};
+    std::shared_ptr<Common::Fiber> host_context{};

 
     u64 thread_id = 0;
 
@@ -485,6 +536,8 @@ private:
     VAddr entry_point = 0;
     VAddr stack_top = 0;
 
+    ThreadType type;
+
     /// Nominal thread priority, as set by the emulated application.
     /// The nominal priority is the thread priority without priority
     /// inheritance taken into account.
diff --git a/src/core/hle/kernel/time_manager.cpp b/src/core/hle/kernel/time_manager.cpp
index 21b290468..0b8f0d993 100644
--- a/src/core/hle/kernel/time_manager.cpp
+++ b/src/core/hle/kernel/time_manager.cpp
@@ -19,7 +19,7 @@ TimeManager::TimeManager(Core::System& system) : system{system} {
             Handle proper_handle = static_cast<Handle>(thread_handle);
             std::shared_ptr<Thread> thread =
                 this->system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle);
-            thread->ResumeFromWait();
+            thread->OnWakeUp();
         });
 }