Skip to content

Commit

Permalink
Refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@lvs1974
lvs1974 committed Aug 3, 2021
1 parent 81f6040 commit b0c86b0
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 169 deletions.
114 changes: 86 additions & 28 deletions CpuTscSync/CpuTscSync.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,64 +2,122 @@
// CpuTscSync.cpp
// CpuTscSync
//
// Copyright © 2020 lvs1974. All rights reserved.
// Copyright © 2021 lvs1974. All rights reserved.
//

#include <Headers/kern_api.hpp>
#include <Headers/kern_cpu.hpp>
#include <i386/proc_reg.h>
#include <IOKit/IOTimerEventSource.h>

#include "CpuTscSync.hpp"
#include "VoodooTSCSync.h"

static CpuTscSyncPlugin *callbackCpuf = nullptr;
_Atomic(bool) CpuTscSyncPlugin::tsc_synced = false;
_Atomic(uint16_t) CpuTscSyncPlugin::cores_ready = 0;
_Atomic(uint64_t) CpuTscSyncPlugin::tsc_frequency = 0;



//stamp the tsc
void CpuTscSyncPlugin::stamp_tsc(void *tscp)
{
wrmsr64(MSR_IA32_TSC, *reinterpret_cast<uint64_t*>(tscp));
}

void CpuTscSyncPlugin::stamp_tsc_new(void *)
{
atomic_fetch_add_explicit(&cores_ready, 1, memory_order_relaxed);

uint32_t cpu = cpu_number();
if (cpu != 0) {
uint64_t tsc = 0;
while ((tsc = atomic_load_explicit(&tsc_frequency, memory_order_acquire)) == 0) {
}
wrmsr64(MSR_IA32_TSC, tsc);
} else {
uint16_t threadCount = rdmsr64(MSR_CORE_THREAD_COUNT);
while (atomic_load_explicit(&cores_ready, memory_order_acquire) != threadCount) {
}
atomic_store_explicit(&tsc_frequency, rdtsc64(), memory_order_relaxed);
}
}

void CpuTscSyncPlugin::reset_tsc_adjust(void *)
{
wrmsr64(MSR_IA32_TSC_ADJUST, 0);
}

void CpuTscSyncPlugin::tsc_adjust_or_reset()
{
if (getKernelVersion() >= KernelVersion::Monterey) {
DBGLOG("cputs", "reset tsc adjust");
mp_rendezvous_no_intrs(reset_tsc_adjust, NULL);
} else {
uint64_t tsc = rdtsc64();
DBGLOG("cputs", "current tsc from rdtsc64() is %lld. Rendezvouing..", tsc);
// call the kernel function that will call this "action" on all cores/processors
mp_rendezvous_no_intrs(stamp_tsc, &tsc);
}
tsc_synced = true;
}

void CpuTscSyncPlugin::init()
{
callbackCpuf = this;
lilu.onPatcherLoadForce(
[](void *user, KernelPatcher &patcher) {
static_cast<CpuTscSyncPlugin *>(user)->processKernel(patcher);
}, this);
callbackCpuf = this;

lilu.onPatcherLoadForce(
[](void *user, KernelPatcher &patcher) {
static_cast<CpuTscSyncPlugin *>(user)->processKernel(patcher);
}, this);
}

void CpuTscSyncPlugin::xcpm_urgency(int urgency, uint64_t rt_period, uint64_t rt_deadline)
{
if (!VoodooTSCSync::tsc_synced)
if (!tsc_synced)
{
SYSLOG("cputs", "xcpm_urgency is called when TSC presumably is not in sync, sync it");
DBGLOG("cputs", "xcpm_urgency is called when TSC presumably is not in sync, skip this call");
return;
}

FunctionCast(xcpm_urgency, callbackCpuf->org_xcpm_urgency)(urgency, rt_period, rt_deadline);
}

void CpuTscSyncPlugin::IOHibernateSystemWake() {
mp_rendezvous_no_intrs(reset_tsc_adjust, NULL);
VoodooTSCSync::tsc_synced = true;

FunctionCast(IOHibernateSystemWake, callbackCpuf->orgIOHibernateSystemWake)();
IOReturn CpuTscSyncPlugin::IOHibernateSystemHasSlept()
{
tsc_synced = false;
return FunctionCast(IOHibernateSystemHasSlept, callbackCpuf->orgIOHibernateSystemHasSlept)();
}

IOReturn CpuTscSyncPlugin::IOHibernateSystemWake()
{
tsc_adjust_or_reset();
return FunctionCast(IOHibernateSystemWake, callbackCpuf->orgIOHibernateSystemWake)();
}

void CpuTscSyncPlugin::processKernel(KernelPatcher &patcher)
{
if (!kernel_routed)
{
if (getKernelVersion() >= KernelVersion::Monterey) {
KernelPatcher::RouteRequest request {"_IOHibernateSystemWake", IOHibernateSystemWake, orgIOHibernateSystemWake};
if (!kernel_routed)
{
KernelPatcher::RouteRequest requests_for_long_jump[] {
{"_IOHibernateSystemWake", IOHibernateSystemWake, orgIOHibernateSystemWake},
{"_IOHibernateSystemHasSlept", IOHibernateSystemHasSlept, orgIOHibernateSystemHasSlept}
};

if (!patcher.routeMultipleLong(KernelPatcher::KernelID, &request, 1))
SYSLOG("cputs", "patcher.routeMultiple for %s is failed with error %d", request.symbol, patcher.getError());
}
if (!patcher.routeMultipleLong(KernelPatcher::KernelID, requests_for_long_jump))
SYSLOG("cputs", "patcher.routeMultiple for %s is failed with error %d", requests_for_long_jump[0].symbol, patcher.getError());

patcher.clearError();

KernelPatcher::RouteRequest requests[] {
{"_xcpm_urgency", xcpm_urgency, org_xcpm_urgency}
};

if (!patcher.routeMultipleLong(KernelPatcher::KernelID, requests))
SYSLOG("cputs", "patcher.routeMultiple for %s is failed with error %d", requests[0].symbol, patcher.getError());
kernel_routed = true;
}
if (!patcher.routeMultiple(KernelPatcher::KernelID, requests))
SYSLOG("cputs", "patcher.routeMultiple for %s is failed with error %d", requests[0].symbol, patcher.getError());
kernel_routed = true;
}

// Ignore all the errors for other processors
patcher.clearError();
// Ignore all the errors for other processors
patcher.clearError();
}
30 changes: 25 additions & 5 deletions CpuTscSync/CpuTscSync.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,53 @@
#define kern_cputs_hpp

#include <Headers/kern_patcher.hpp>
#include <IOKit/IOService.h>
#include <stdatomic.h>


//reg define
#define MSR_IA32_TSC 0x00000010
#define MSR_IA32_TSC_ADJUST 0x0000003b


class CpuTscSyncPlugin {
public:
void init();

bool kernel_routed = false;

private:
_Atomic(bool) kernel_routed = false;
static _Atomic(bool) tsc_synced;
static _Atomic(uint16_t) cores_ready;
static _Atomic(uint64_t) tsc_frequency;

private:
/**
* Trampolines for original resource load callback
*/
mach_vm_address_t org_xcpm_urgency {0};
mach_vm_address_t orgIOHibernateSystemHasSlept {0};
mach_vm_address_t orgIOHibernateSystemWake {0};

/**
* Hooked functions
*/
static void xcpm_urgency(int urgency, uint64_t rt_period, uint64_t rt_deadline);
static void IOHibernateSystemWake();

static IOReturn IOHibernateSystemHasSlept(void);
static IOReturn IOHibernateSystemWake();

/**
* Patch kernel
*
* @param patcher KernelPatcher instance
*/
void processKernel(KernelPatcher &patcher);


static void stamp_tsc(void *tscp);
static void stamp_tsc_new(void *);
static void reset_tsc_adjust(void *);

public:
static void tsc_adjust_or_reset();
};

#endif /* kern_cputs_hpp */
104 changes: 7 additions & 97 deletions CpuTscSync/VoodooTSCSync.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,44 +1,9 @@
#include <Headers/kern_util.hpp>
#include "VoodooTSCSync.h"
#include <stdatomic.h>
#include "CpuTscSync.hpp"

OSDefineMetaClassAndStructors(VoodooTSCSync, IOService)

_Atomic(bool) VoodooTSCSync::tsc_synced = false;
static _Atomic(uint16_t) cores_ready = 0;
static _Atomic(uint64_t) tsc_frequency = 0;

extern "C" int cpu_number(void);


//stamp the tsc
extern "C" void stamp_tsc(void *tscp)
{
wrmsr64(MSR_IA32_TSC, *reinterpret_cast<uint64_t*>(tscp));
}

extern "C" void stamp_tsc_new(void *)
{
atomic_fetch_add_explicit(&cores_ready, 1, memory_order_relaxed);
#include <i386/proc_reg.h>

uint32_t cpu = cpu_number();
if (cpu != 0) {
uint64_t tsc = 0;
while ((tsc = atomic_load_explicit(&tsc_frequency, memory_order_acquire)) == 0) {
}
wrmsr64(MSR_IA32_TSC, tsc);
} else {
uint16_t threadCount = rdmsr64(MSR_CORE_THREAD_COUNT);
while (atomic_load_explicit(&cores_ready, memory_order_acquire) != threadCount) {
}
atomic_store_explicit(&tsc_frequency, rdtsc64(), memory_order_relaxed);
}
}

extern "C" void reset_tsc_adjust(void *)
{
wrmsr64(MSR_IA32_TSC_ADJUST, 0);
}
OSDefineMetaClassAndStructors(VoodooTSCSync, IOService)

IOService* VoodooTSCSync::probe(IOService* provider, SInt32* score)
{
Expand All @@ -49,70 +14,15 @@ IOService* VoodooTSCSync::probe(IOService* provider, SInt32* score)
if (!cpuNumber) return NULL;

if (getKernelVersion() >= KernelVersion::Monterey) {
// only attach to last CPU
uint16_t threadCount = rdmsr64(MSR_CORE_THREAD_COUNT);
// only attach to the first CPU
if (cpuNumber->unsigned16BitValue() != 0) return NULL;
DBGLOG("cputs", "discovered cpu with %u threads, matching last, syncing", threadCount);
mp_rendezvous_no_intrs(reset_tsc_adjust, NULL);
DBGLOG("cputs", "done syncing");
tsc_synced = true;
CpuTscSyncPlugin::tsc_adjust_or_reset();
} else {
// only attach to last CPU
// only attach to the last CPU
uint16_t threadCount = rdmsr64(MSR_CORE_THREAD_COUNT);
if (cpuNumber->unsigned16BitValue() != threadCount-1) return NULL;
CpuTscSyncPlugin::tsc_adjust_or_reset();
}

return this;
}

IOReturn VoodooTSCSync::setPowerState(unsigned long state, IOService *whatDevice )
{
if (state == PowerStateOff)
tsc_synced = false;
else if (state == PowerStateOn)
doTSC();

return IOPMAckImplied;
}

void VoodooTSCSync::stop(IOService *provider)
{
PMstop();
super::stop(provider);
}

bool VoodooTSCSync::start(IOService *provider)
{
if (!super::start(provider)) { return false; }

// announce version
extern kmod_info_t kmod_info;
SYSLOG("cputs", "Version %s starting on OS X Darwin %d.%d.\n", kmod_info.version, version_major, version_minor);

// place version/build info in ioreg properties RM,Build and RM,Version
char buf[128];
snprintf(buf, sizeof(buf), "%s %s", kmod_info.name, kmod_info.version);
setProperty("RM,Version", buf);
#ifdef DEBUG
setProperty("RM,Build", "Debug");
#else
setProperty("RM,Build", "Release");
#endif

PMinit();
registerPowerDriver(this, powerStates, 2);
provider->joinPMtree(this);
return true;
}

// Update MSR on all processors.
void VoodooTSCSync::doTSC()
{
if (getKernelVersion() < KernelVersion::Monterey) {
uint64_t tsc = rdtsc64();
DBGLOG("cputs", "current tsc from rdtsc64() is %lld. Rendezvouing..\n", tsc);
// call the kernel function that will call this "action" on all cores/processors
mp_rendezvous_no_intrs(stamp_tsc, &tsc);
tsc_synced = true;
}
}
39 changes: 0 additions & 39 deletions CpuTscSync/VoodooTSCSync.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,52 +4,13 @@
*
*/

#include <stdatomic.h>

#include <IOKit/IOService.h>
#include <IOKit/IOLib.h>
#include <i386/proc_reg.h>

//reg define
#define MSR_IA32_TSC 0x00000010
#define MSR_IA32_TSC_ADJUST 0x0000003b

//extern function defined in mp.c from xnu
extern "C" void mp_rendezvous_no_intrs(void (*action_func)(void*), void *arg);
extern "C" void reset_tsc_adjust(void *);


class VoodooTSCSync : public IOService
{
typedef IOService super;
OSDeclareDefaultStructors(VoodooTSCSync)

private:
static void doTSC(void);

public:
virtual IOService* probe(IOService* provider, SInt32* score) override;
virtual bool start(IOService* provider) override;
virtual void stop(IOService* provider) override;
virtual IOReturn setPowerState(unsigned long state, IOService* whatDevice) override;

public:
static _Atomic(bool) tsc_synced;

/**
* Power state name indexes
*/
enum PowerState {
PowerStateOff,
PowerStateOn,
PowerStateMax
};

/**
* Power states we monitor
*/
IOPMPowerState powerStates[PowerStateMax] {
{kIOPMPowerStateVersion1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
{kIOPMPowerStateVersion1, kIOPMPowerOn | kIOPMDeviceUsable, kIOPMPowerOn, kIOPMPowerOn, 0, 0, 0, 0, 0, 0, 0, 0}
};
};

0 comments on commit b0c86b0

Please sign in to comment.