Skip to content

Commit

Permalink
- New "station" data added, closed #858
Browse files Browse the repository at this point in the history 
- Improved system deeplinks. EveEye.com added, closed #859
- Improved cronjob performance for "SovereigntyData" import, closed #853
- Updated static DB dump for `eve_universe.sql`
  • Loading branch information
@exodus4d
exodus4d committed Sep 30, 2019
1 parent d45fa9b commit 1de67f8
Show file tree
Hide file tree
Showing 88 changed files with 2,464 additions and 564 deletions.
2 changes: 1 addition & 1 deletion .jshintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"node": true,

// Allow ES6.
"esversion": 6,
"esversion": 7,

/*
* ENFORCING OPTIONS
Expand Down
10 changes: 7 additions & 3 deletions app/cron.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,13 @@ deleteStatisticsData = Cron\StatisticsUpdate->deleteStatisticsD
; truncate map history log files
truncateMapHistoryLogFiles = Cron\MapHistory->truncateFiles, @halfHour

; updates small amount of static system data from CCP API
; sync "sovereignty" and "faction warfare" data from CCP´s ESI API
updateSovereigntyData = Cron\Universe->updateSovereigntyData, @halfPastHour

; sync static system data from CCP´s ESI API
; -> Job is WIP!
;updateUniverseSystems = Cron\Universe->updateUniverseSystems, @instant
;updateSovereigntyData = Cron\Universe->updateSovereigntyData, @instant

; setup universe DB with static data from ESI
; bootstrap job for "eve_universe" DB from CCP´s ESI API
; -> Only for development! This job is used to build the initial export/sql/eve_universe.sql
;setup = Cron\Universe->setup, @instant
5 changes: 4 additions & 1 deletion app/main/controller/api/map.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,10 @@ public function initData(\Base $f3){
// get third party APIs -----------------------------------------------------------------------------------
$return->url = [
'ccpImageServer' => Config::getPathfinderData('api.ccp_image_server'),
'zKillboard' => Config::getPathfinderData('api.z_killboard')
'zKillboard' => Config::getPathfinderData('api.z_killboard'),
'eveeye' => Config::getPathfinderData('api.eveeye'),
'dotlan' => Config::getPathfinderData('api.dotlan'),
'anoik' => Config::getPathfinderData('api.anoik')
];

// Character default config -------------------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions app/main/controller/api/rest/system.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public function get(\Base $f3, $params){
$systemData->signatures = $system->getSignaturesData();
$systemData->sigHistory = $system->getSignaturesHistory();
$systemData->structures = $system->getStructuresData();
$systemData->stations = $system->getStationsData();
}
}

Expand Down
14 changes: 7 additions & 7 deletions app/main/controller/api/system.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public function graphData(\Base $f3){
}

/**
* set destination for specific systemIds
* set destination for system, station or structure
* @param \Base $f3
* @throws \Exception
*/
Expand All @@ -120,25 +120,25 @@ public function setDestination(\Base $f3){

$return = (object) [];
$return->error = [];
$return->systemData = [];
$return->destData = [];

if( !empty($postData['systemData'] )){
if(!empty($destData = (array)$postData['destData'])){
$activeCharacter = $this->getCharacter();

$return->clearOtherWaypoints = (bool)$postData['clearOtherWaypoints'];
$return->first = (bool)$postData['first'];

if( $accessToken = $activeCharacter->getAccessToken() ){
if($accessToken = $activeCharacter->getAccessToken()){
$options = [
'clearOtherWaypoints' => $return->clearOtherWaypoints,
'addToBeginning' => $return->first,
];

foreach($postData['systemData'] as $systemData){
$response = $f3->ccpClient()->setWaypoint($systemData['systemId'], $accessToken, $options);
foreach($destData as $data){
$response = $f3->ccpClient()->setWaypoint((int)$data['id'], $accessToken, $options);

if(empty($response)){
$return->systemData[] = $systemData;
$return->destData[] = $data;
}else{
$error = (object) [];
$error->type = 'error';
Expand Down
2 changes: 1 addition & 1 deletion app/main/controller/ccp/universe.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public function buildSystemsIndex(int $offset = 0, int $length = 10) : array {
$system = Model\Universe\AbstractUniverseModel::getNew('SystemModel');
$indexData = [];
foreach($systemIds as $systemId){
$system->getById($systemId);
$system->getById($systemId, 0);
if($hashKeyId = $system->getHashKey()){
$indexData[$hashKeyId] = $system->getData();
}
Expand Down
2 changes: 2 additions & 0 deletions app/main/controller/setup.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ class Setup extends Controller {
'Model\Universe\FactionModel',
'Model\Universe\AllianceModel',
'Model\Universe\CorporationModel',
'Model\Universe\RaceModel',
'Model\Universe\StationModel',
'Model\Universe\StructureModel',
'Model\Universe\StargateModel',
'Model\Universe\StarModel',
Expand Down
43 changes: 40 additions & 3 deletions app/main/cron/abstractcron.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,55 @@

abstract class AbstractCron {

// default max_execution_time for cronJobs
/**
* default max_execution_time for cronJobs
// -> should be less then execution period
*/
const DEFAULT_MAX_EXECUTION_TIME = 50;

/**
* set max execution time for cronjobs
* -> Default CLI execution time is "0" -> infinite!
* default threshold time in seconds before a running script (e.g. a large loop) should stop
* -> so there is some time or e.g. logging,... left
*/
const DEFAULT_EXECUTION_TIME_THRESHOLD = 3;

/**
* set max execution time for cronJbs
* -> Default CLI execution time is 0 == infinite!
* php.ini settings are ignored! http://php.net/manual/en/info.configuration.php#ini.max-execution-time
* @param int $time
*/
protected function setMaxExecutionTime(int $time = self::DEFAULT_MAX_EXECUTION_TIME){
ini_set('max_execution_time', $time);
}

/**
* get max execution time
* -> 0 means == infinite!
* @return int
*/
protected function getMaxExecutionTime() : int {
return (int)ini_get('max_execution_time');
}

/**
* checks execution time of a "long" running script
* -> returns false if execution time is close to maxExecutionTime
* @param float $timeTotalStart
* @param float|null $timeCheck
* @param int $timeThreshold
* @return bool
*/
protected function isExecutionTimeLeft(float $timeTotalStart, float $timeCheck = null, int $timeThreshold = self::DEFAULT_EXECUTION_TIME_THRESHOLD) : bool {
$timeLeft = true;
if($timeTotalMax = $this->getMaxExecutionTime()){
$timeTotalMaxThreshold = $timeTotalStart + $timeTotalMax - $timeThreshold;
$timeCheck = $timeCheck ? : microtime(true);
if($timeCheck >= $timeTotalMaxThreshold){
$timeLeft = false;
}
}
return $timeLeft;
}

}
79 changes: 70 additions & 9 deletions app/main/cron/universe.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
class Universe extends AbstractCron {

const LOG_TEXT = '%s type: %s %s/%s peak: %s total: %s msg: %s';
const LOG_TEXT_SOV_FW = '%s %4s/%-4s checked, %s peak, %s total, %4s updated [%4s sovChanges, %4s fwChanges], msg: %s';


/**
Expand Down Expand Up @@ -133,10 +134,11 @@ private function echoLoading(int $count, int $importCount, int $id){
* @param float $timeTotalStart
*/
private function echoLoaded(int $importCount, int $id, float $timeLoopStart, float $timeTotalStart){
$time = microtime(true);
echo '[' . date('H:i:s') . '] loaded ' . str_pad('', strlen($importCount), ' ') . ' id: ' . $this->formatIdValue($id) .
' memory: ' . $this->formatMemoryValue(memory_get_usage()) .
' time: ' . $this->formatSeconds(microtime(true) - $timeLoopStart) .
' total: ' . $this->formatSeconds(microtime(true) - $timeTotalStart) . PHP_EOL;
' time: ' . $this->formatSeconds($time - $timeLoopStart) .
' total: ' . $this->formatSeconds($time - $timeTotalStart) . PHP_EOL;
$this->echoFlush();
}

Expand Down Expand Up @@ -169,8 +171,8 @@ function setup(\Base $f3){
$msg = '';

$ids = [];
$importCount = 0;
$count = 0;
$importCount = [];
$modelClass = '';
$setupModel = function(Model\Universe\AbstractUniverseModel &$model, int $id){};

Expand All @@ -193,6 +195,18 @@ function setup(\Base $f3){
$model->loadStargatesData();
};
break;
case 'station':
$ids = $f3->ccpClient()->getUniverseSystems();
$modelClass = 'SystemModel';
$setupModel = function(Model\Universe\SystemModel &$model, int $id){
if($model->getById($id)){
$model->loadStationsData();
}else{
echo 'NOT VALID ' . $id . PHP_EOL;
die();
}
};
break;
case 'sovereignty':
// load sovereignty map data. Systems must be present first!
$sovData = $f3->ccpClient()->getSovereigntyMap();
Expand Down Expand Up @@ -243,6 +257,7 @@ function setup(\Base $f3){
sort($ids, SORT_NUMERIC);
$ids = array_slice($ids, $offset, $length);
$importCount = count($ids);
$count = 0;

$this->echoInfo($total, $offset, $importCount, $ids);
$this->echoStart();
Expand All @@ -264,9 +279,9 @@ function setup(\Base $f3){

// Log --------------------------------------------------------------------------------------------------------
$log = new \Log('cron_' . __FUNCTION__ . '.log');
$log->write( sprintf(self::LOG_TEXT, __FUNCTION__, $type,
$log->write(sprintf(self::LOG_TEXT, __FUNCTION__, $type,
$this->formatCounterValue($count), $importCount, $this->formatMemoryValue(memory_get_peak_usage ()),
$this->formatSeconds(microtime(true) - $timeTotalStart), $msg) );
$this->formatSeconds(microtime(true) - $timeTotalStart), $msg));
}

/**
Expand All @@ -278,23 +293,69 @@ function setup(\Base $f3){
*/
function updateSovereigntyData(\Base $f3){
$this->setMaxExecutionTime();
$timeTotalStart = microtime(true);
$msg = '';

/**
* @var $system Model\Universe\SystemModel
*/
$system = Model\Universe\AbstractUniverseModel::getNew('SystemModel');

$sovData = $f3->ccpClient()->getSovereigntyMap();
$fwSystems = $f3->ccpClient()->getFactionWarSystems();
$fwSystems = $fwSystems['systems'];
$ids = !empty($sovData = $sovData['map']) ? array_keys($sovData): [];
sort($ids, SORT_NUMERIC);
$importCount = count($ids);
$count = 0;

$changes = [];
foreach($ids as $id){
if($system->getById($id)){
$system->updateSovereigntyData($sovData[$id]);
$count++;

// skip wormhole systems -> can not have sov data
// -> even though they are returned from sovereignty/map endpoint?!
if(
$system->getById($id, 0) &&
strpos($system->security, 'C') === false
){
if($changedSovData = $system->updateSovereigntyData($sovData[$id])){
$changes['sovereignty'][] = $id;
}

$changedFwData = false;
if(is_array($fwSystems[$id])){
$system->updateFactionWarData($fwSystems[$id]);
if($changedFwData = $system->updateFactionWarData($fwSystems[$id])){
$changes['factionWarfare'][] = $id;
}
}

if($changedSovData || $changedFwData){
$system->buildIndex();
}
$system->reset();
}
$system->reset();

// stop loop if runtime gets close to "max_execution_time"
// -> we need some time for writing *.log file
if(!$this->isExecutionTimeLeft($timeTotalStart)){
$msg = 'Script execution stopped due to "max_execution_time" limit reached';
// TODO store current loop index and use it as new "offset" for next call
break;
}
}

$changedIds = array_reduce($changes, function(array $reducedIds, array $changedIds) : array {
return array_unique(array_merge($reducedIds, $changedIds));
}, []);

// Log ------------------------
$log = new \Log('cron_' . __FUNCTION__ . '.log');
$log->write(sprintf(self::LOG_TEXT_SOV_FW, __FUNCTION__,
$count, $importCount, $this->formatMemoryValue(memory_get_peak_usage ()),
$this->formatSeconds(microtime(true) - $timeTotalStart),
count($changedIds), count($changes['sovereignty'] ? : []), count($changes['factionWarfare'] ? : []),
$msg));
}

/**
Expand Down
Loading

0 comments on commit 1de67f8

Please sign in to comment.