Skip to content

Commit 2a0c446

Browse files
mwjamesmwjames
authored
Dispose entities on unmaintained namespace, add maintenance alert, refs 4468 (SemanticMediaWiki#4744)
1 parent 95da0cd commit 2a0c446

13 files changed

+340
-8
lines changed

Diff for: ‎i18n/en.json

+2
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,8 @@
611611
"smw-admin-maintenancealerts-lastoptimizationrun-alert": "The system has found that the last [https://www.semantic-mediawiki.org/wiki/Table_optimization table optimization] was run $2 days ago (record from $1) which exceeds the $3 days maintenance threshold. As mentioned in the documentation, running optimizations will allow the query planner to make better decisions about queries therefore it is suggested to run the table optimization on a regular basis.",
612612
"smw-admin-maintenancealerts-outdatedentitiesmaxcount-alert-title": "Outdated entities",
613613
"smw-admin-maintenancealerts-outdatedentitiesmaxcount-alert": "The system has counted $1 [https://www.semantic-mediawiki.org/wiki/Outdated_entities outdated entities] and reached a critical level of unattended maintenance by exceeding the threshold of $2. It is recommended to run the [https://www.semantic-mediawiki.org/wiki/disposeOutdatedEntities.php <code>disposeOutdatedEntities.php</code>] maintenance script.",
614+
"smw-admin-maintenancealerts-invalidentities-alert-title": "Invalid entities",
615+
"smw-admin-maintenancealerts-invalidentities-alert": "The system matched $1 [https://www.semantic-mediawiki.org/wiki/Invalid_entities {{PLURAL:$1|entity|entities}}] to an [https://www.semantic-mediawiki.org/wiki/Unmaintained_namespace unmaintained namespace] and it is recommended to run the [https://www.semantic-mediawiki.org/wiki/disposeOutdatedEntities.php <code>disposeOutdatedEntities.php</code>] or [https://www.semantic-mediawiki.org/wiki/rebuildData.php <code>rebuildData.php</code>] maintenance script.",
614616
"smw-admin-deprecation-notice-section": "Semantic MediaWiki",
615617
"smw-admin-configutation-tab-settings": "Settings",
616618
"smw-admin-configutation-tab-namespaces": "Namespaces",

Diff for: ‎maintenance/disposeOutdatedEntities.php

+3-2
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,9 @@ public function execute() {
9696
);
9797

9898
$text = [
99-
"This script will remove outdated entities and query link entries from",
100-
"tables that hold a reference to the ID marked as outdated",
99+
"This script will remove outdated entities and entities rendered",
100+
"invalid due to redeclared namespace settings. It will also dispose of",
101+
"query link entries from tables that no longer hold a valid entity reference",
101102
"in Semantic MediaWiki."
102103
];
103104

Diff for: ‎src/Maintenance/DataRebuilder/OutdatedDisposer.php

+21-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public function __construct( EntityIdDisposerJob $entityIdDisposerJob, IteratorF
5353
public function run() {
5454

5555
$this->messageReporter->reportMessage(
56-
"Removing outdated entities and query links ...\n"
56+
"Removing outdated and invalid entities ...\n"
5757
);
5858

5959
$this->messageReporter->reportMessage(
@@ -70,6 +70,26 @@ public function run() {
7070
);
7171
}
7272

73+
$this->messageReporter->reportMessage(
74+
$this->cliMsgFormatter->firstCol( ' ... checking invalid entities by namespace ...' )
75+
);
76+
77+
$resultIterator = $this->entityIdDisposerJob->newByNamespaceInvalidEntitiesResultIterator();
78+
79+
if ( ( $count = $resultIterator->count() ) > 0 ) {
80+
$this->disposeOutdatedEntities( $resultIterator, $count );
81+
} else {
82+
$this->messageReporter->reportMessage(
83+
$this->cliMsgFormatter->secondCol( CliMsgFormatter::OK )
84+
);
85+
}
86+
87+
$this->messageReporter->reportMessage( " ... done.\n" );
88+
89+
$this->messageReporter->reportMessage(
90+
"\nRemoving query links ...\n"
91+
);
92+
7393
$this->messageReporter->reportMessage(
7494
$this->cliMsgFormatter->firstCol( ' ... checking query links (invalid) ...' )
7595
);

Diff for: ‎src/MediaWiki/Jobs/EntityIdDisposerJob.php

+16
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,22 @@ public function newOutdatedEntitiesResultIterator( RequestOptions $requestOption
6666
return $this->propertyTableIdReferenceDisposer->newOutdatedEntitiesResultIterator( $requestOptions );
6767
}
6868

69+
/**
70+
* @since 3.2
71+
*
72+
* @param RequestOptions|null $requestOptions
73+
*
74+
* @return ResultIterator
75+
*/
76+
public function newByNamespaceInvalidEntitiesResultIterator( RequestOptions $requestOptions = null ) {
77+
78+
if ( $this->propertyTableIdReferenceDisposer === null ) {
79+
$this->propertyTableIdReferenceDisposer = $this->newPropertyTableIdReferenceDisposer();
80+
}
81+
82+
return $this->propertyTableIdReferenceDisposer->newByNamespaceInvalidEntitiesResultIterator( $requestOptions );
83+
}
84+
6985
/**
7086
* @since 3.1
7187
*

Diff for: ‎src/MediaWiki/Specials/Admin/Alerts/ByNamespaceInvalidEntitiesMaintenanceAlertTaskHandler.php

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<?php
2+
3+
namespace SMW\MediaWiki\Specials\Admin\Alerts;
4+
5+
use Html;
6+
use SMW\Store;
7+
use SMW\Message;
8+
use SMW\SQLStore\SQLStore;
9+
use SMW\MediaWiki\Specials\Admin\TaskHandler;
10+
use SMW\MediaWiki\Specials\Admin\OutputFormatter;
11+
12+
/**
13+
* @license GNU GPL v2+
14+
* @since 3.2
15+
*
16+
* @author mwjames
17+
*/
18+
class ByNamespaceInvalidEntitiesMaintenanceAlertTaskHandler extends TaskHandler {
19+
20+
/**
21+
* @var Store
22+
*/
23+
private $store;
24+
25+
/**
26+
* @var array
27+
*/
28+
private $namespacesWithSemanticLinks = [];
29+
30+
/**
31+
* @since 3.2
32+
*
33+
* @param Store $store
34+
*/
35+
public function __construct( Store $store ) {
36+
$this->store = $store;
37+
}
38+
39+
/**
40+
* @since 3.2
41+
*
42+
* @param array $namespacesWithSemanticLinks
43+
*/
44+
public function setNamespacesWithSemanticLinks( array $namespacesWithSemanticLinks ) {
45+
$this->namespacesWithSemanticLinks = $namespacesWithSemanticLinks;
46+
}
47+
48+
/**
49+
* @since 3.2
50+
*
51+
* {@inheritDoc}
52+
*/
53+
public function getHtml() {
54+
55+
$count = $this->fetchCount();
56+
57+
if ( $count == 0 ) {
58+
return '';
59+
}
60+
61+
return $this->buildHTML( $count );
62+
}
63+
64+
private function fetchCount() {
65+
66+
$connection = $this->store->getConnection( 'mw.db' );
67+
68+
$row = $connection->selectRow(
69+
SQLStore::ID_TABLE,
70+
'COUNT(smw_id) AS count',
71+
[
72+
'smw_namespace NOT IN (' . $connection->makeList( array_keys( $this->namespacesWithSemanticLinks ) ) . ')'
73+
],
74+
__METHOD__
75+
);
76+
77+
return $row !== false ? (int)$row->count : 0;
78+
}
79+
80+
private function buildHTML( $count ) {
81+
82+
$html = Html::rawElement(
83+
'fieldset',
84+
[
85+
'class' => "smw-admin-alerts-section-legend"
86+
],
87+
Html::rawElement(
88+
'legend',
89+
[
90+
'class' => "smw-admin-alerts-section-legend"
91+
],
92+
$this->msg( "smw-admin-maintenancealerts-invalidentities-alert-title" )
93+
) . Html::rawElement(
94+
'p',
95+
[],
96+
$this->msg( ['smw-admin-maintenancealerts-invalidentities-alert', $count ], Message::PARSE )
97+
)
98+
);
99+
100+
return Html::rawElement(
101+
'div',
102+
[
103+
'class' => 'smw-admin-alerts smw-admin-alerts-invalid-entities'
104+
],
105+
$html
106+
);
107+
}
108+
109+
}

Diff for: ‎src/MediaWiki/Specials/Admin/TaskHandlerFactory.php

+13-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use SMW\MediaWiki\Specials\Admin\Alerts\MaintenanceAlertsTaskHandler;
1919
use SMW\MediaWiki\Specials\Admin\Alerts\LastOptimizationRunMaintenanceAlertTaskHandler;
2020
use SMW\MediaWiki\Specials\Admin\Alerts\OutdatedEntitiesMaxCountThresholdMaintenanceAlertTaskHandler;
21+
use SMW\MediaWiki\Specials\Admin\Alerts\ByNamespaceInvalidEntitiesMaintenanceAlertTaskHandler;
2122
use SMW\MediaWiki\HookDispatcherAwareTrait;
2223
use SMW\Store;
2324
use SMW\SetupFile;
@@ -271,13 +272,24 @@ public function newFulltextSearchTableRebuildJobTaskHandler() {
271272
*/
272273
public function newAlertsTaskHandler( $adminFeatures = 0 ) {
273274

275+
$settings = ApplicationFactory::getInstance()->getSettings();
276+
277+
$byNamespaceInvalidEntitiesMaintenanceAlertTaskHandler = new ByNamespaceInvalidEntitiesMaintenanceAlertTaskHandler(
278+
$this->store
279+
);
280+
281+
$byNamespaceInvalidEntitiesMaintenanceAlertTaskHandler->setNamespacesWithSemanticLinks(
282+
$settings->get( 'smwgNamespacesWithSemanticLinks' )
283+
);
284+
274285
$maintenanceAlertsTaskHandlers = [
275286
new LastOptimizationRunMaintenanceAlertTaskHandler(
276287
new SetupFile()
277288
),
278289
new OutdatedEntitiesMaxCountThresholdMaintenanceAlertTaskHandler(
279290
$this->store
280-
)
291+
),
292+
$byNamespaceInvalidEntitiesMaintenanceAlertTaskHandler
281293
];
282294

283295
$maintenanceAlertsTaskHandler = new MaintenanceAlertsTaskHandler(

Diff for: ‎src/SQLStore/PropertyTableIdReferenceDisposer.php

+45
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ class PropertyTableIdReferenceDisposer {
4949
*/
5050
private $fulltextTableUsage = false;
5151

52+
/**
53+
* @var array
54+
*/
55+
private $namespacesWithSemanticLinks = [];
56+
5257
/**
5358
* @since 2.4
5459
*
@@ -77,6 +82,15 @@ public function setFulltextTableUsage( bool $fulltextTableUsage ) {
7782
$this->fulltextTableUsage = $fulltextTableUsage;
7883
}
7984

85+
/**
86+
* @since 3.2
87+
*
88+
* @param array $namespacesWithSemanticLinks
89+
*/
90+
public function setNamespacesWithSemanticLinks( array $namespacesWithSemanticLinks ) {
91+
$this->namespacesWithSemanticLinks = $namespacesWithSemanticLinks;
92+
}
93+
8094
/**
8195
* @since 2.5
8296
*/
@@ -147,6 +161,37 @@ public function newOutdatedEntitiesResultIterator( RequestOptions $requestOption
147161
return new ResultIterator( $res );
148162
}
149163

164+
/**
165+
* @since 3.2
166+
*
167+
* @param RequestOptions|null $requestOptions
168+
*
169+
* @return ResultIterator
170+
*/
171+
public function newByNamespaceInvalidEntitiesResultIterator( RequestOptions $requestOptions = null ) {
172+
173+
$options = [];
174+
175+
if ( $requestOptions !== null ) {
176+
$options = [
177+
'LIMIT' => $requestOptions->getLimit(),
178+
'OFFSET' => $requestOptions->getOffset()
179+
];
180+
}
181+
182+
$res = $this->connection->select(
183+
SQLStore::ID_TABLE,
184+
[ 'smw_id' ],
185+
[
186+
'smw_namespace NOT IN (' . $this->connection->makeList( array_keys( $this->namespacesWithSemanticLinks ) ) . ')'
187+
],
188+
__METHOD__,
189+
$options
190+
);
191+
192+
return new ResultIterator( $res );
193+
}
194+
150195
/**
151196
* @since 2.5
152197
*

Diff for: ‎src/SQLStore/SQLStoreFactory.php

+6-1
Original file line numberDiff line numberDiff line change
@@ -1027,6 +1027,7 @@ public function newSortLetter() {
10271027
public function newPropertyTableIdReferenceDisposer() {
10281028

10291029
$applicationFactory = ApplicationFactory::getInstance();
1030+
$settings = $applicationFactory->getSettings();
10301031

10311032
$propertyTableIdReferenceDisposer = new PropertyTableIdReferenceDisposer(
10321033
$this->store,
@@ -1038,7 +1039,11 @@ public function newPropertyTableIdReferenceDisposer() {
10381039
);
10391040

10401041
$propertyTableIdReferenceDisposer->setFulltextTableUsage(
1041-
$applicationFactory->getSettings()->get( 'smwgEnabledFulltextSearch' )
1042+
$settings->get( 'smwgEnabledFulltextSearch' )
1043+
);
1044+
1045+
$propertyTableIdReferenceDisposer->setNamespacesWithSemanticLinks(
1046+
$settings->get( 'smwgNamespacesWithSemanticLinks' )
10421047
);
10431048

10441049
return $propertyTableIdReferenceDisposer;

Diff for: ‎tests/phpunit/Integration/Maintenance/DisposeOutdatedEntitiesTest.php

+6-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,12 @@ public function testRun() {
5252
);
5353

5454
$this->assertContains(
55-
'Removing outdated entities and query links',
55+
'Removing outdated and invalid entities',
56+
$this->spyMessageReporter->getMessagesAsString()
57+
);
58+
59+
$this->assertContains(
60+
'Removing query links',
5661
$this->spyMessageReporter->getMessagesAsString()
5762
);
5863
}

Diff for: ‎tests/phpunit/Unit/Maintenance/DataRebuilder/OutdatedDisposerTest.php

+14-2
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,18 @@ public function testDispose_Entities() {
6666
->disableOriginalConstructor()
6767
->getMock();
6868

69-
$resultIterator->expects( $this->once() )
69+
$resultIterator->expects( $this->exactly( 2 ) )
7070
->method( 'count' )
7171
->will( $this->returnValue( 42 ) );
7272

7373
$this->entityIdDisposerJob->expects( $this->once() )
7474
->method( 'newOutdatedEntitiesResultIterator' )
7575
->will( $this->returnValue( $resultIterator ) );
7676

77+
$this->entityIdDisposerJob->expects( $this->once() )
78+
->method( 'newByNamespaceInvalidEntitiesResultIterator' )
79+
->will( $this->returnValue( $resultIterator ) );
80+
7781
$this->entityIdDisposerJob->expects( $this->once() )
7882
->method( 'newOutdatedQueryLinksResultIterator' )
7983
->will( $this->returnValue( $this->resultIterator ) );
@@ -85,7 +89,7 @@ public function testDispose_Entities() {
8589
$this->entityIdDisposerJob->expects( $this->once() )
8690
->method( 'dispose' );
8791

88-
$this->iteratorFactory->expects( $this->once() )
92+
$this->iteratorFactory->expects( $this->exactly( 2 ) )
8993
->method( 'newChunkedIterator' )
9094
->will( $this->returnValue( $chunkedIterator ) );
9195

@@ -137,6 +141,10 @@ public function testDispose_QueryLinks_Invalid() {
137141
->method( 'newOutdatedEntitiesResultIterator' )
138142
->will( $this->returnValue( $this->resultIterator ) );
139143

144+
$this->entityIdDisposerJob->expects( $this->once() )
145+
->method( 'newByNamespaceInvalidEntitiesResultIterator' )
146+
->will( $this->returnValue( $this->resultIterator ) );
147+
140148
$this->entityIdDisposerJob->expects( $this->once() )
141149
->method( 'newUnassignedQueryLinksResultIterator' )
142150
->will( $this->returnValue( $this->resultIterator ) );
@@ -197,6 +205,10 @@ public function testDispose_QueryLinks_Unassigned() {
197205
->method( 'newOutdatedEntitiesResultIterator' )
198206
->will( $this->returnValue( $this->resultIterator ) );
199207

208+
$this->entityIdDisposerJob->expects( $this->once() )
209+
->method( 'newByNamespaceInvalidEntitiesResultIterator' )
210+
->will( $this->returnValue( $this->resultIterator ) );
211+
200212
$this->entityIdDisposerJob->expects( $this->once() )
201213
->method( 'newOutdatedQueryLinksResultIterator' )
202214
->will( $this->returnValue( $this->resultIterator ) );

Diff for: ‎tests/phpunit/Unit/MediaWiki/Jobs/EntityIdDisposerJobTest.php

+14
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,20 @@ public function testCanConstructOutdatedEntitiesResultIterator() {
8080
);
8181
}
8282

83+
public function testCanConstructByNamespaceInvalidEntitiesResultIterator() {
84+
85+
$title = $this->getMockBuilder( 'Title' )
86+
->disableOriginalConstructor()
87+
->getMock();
88+
89+
$instance = new EntityIdDisposerJob( $title );
90+
91+
$this->assertInstanceOf(
92+
'\SMW\Iterators\ResultIterator',
93+
$instance->newByNamespaceInvalidEntitiesResultIterator()
94+
);
95+
}
96+
8397
public function testCanConstructOutdatedQueryLinksResultIterator() {
8498

8599
$title = $this->getMockBuilder( 'Title' )

0 commit comments

Comments
 (0)