Skip to content

Commit 80da42d

Browse files
geofjamggeofjamg
authored
Merge pull request #198 from itesla/cim_uncompressed
Import uncompressed CIM files
2 parents 06e184f + cdc15c7 commit 80da42d

File tree

10 files changed

+3746
-41
lines changed

10 files changed

+3746
-41
lines changed

‎cim1-import/pom.xml

+16-6
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,14 @@
2323
<name>CIM ENTSO-E V1 import</name>
2424

2525
<dependencies>
26+
<!-- Compilation dependencies -->
2627
<dependency>
2728
<groupId>com.google.auto.service</groupId>
2829
<artifactId>auto-service</artifactId>
2930
</dependency>
3031
<dependency>
31-
<groupId>org.slf4j</groupId>
32-
<artifactId>slf4j-api</artifactId>
32+
<groupId>com.google.jimfs</groupId>
33+
<artifactId>jimfs</artifactId>
3334
</dependency>
3435
<dependency>
3536
<groupId>org.apache.servicemix.bundles</groupId>
@@ -40,16 +41,20 @@
4041
<artifactId>jgrapht-core</artifactId>
4142
</dependency>
4243
<dependency>
43-
<groupId>${project.groupId}</groupId>
44+
<groupId>org.slf4j</groupId>
45+
<artifactId>slf4j-api</artifactId>
46+
</dependency>
47+
<dependency>
48+
<groupId>eu.itesla_project</groupId>
4449
<artifactId>iidm-converter-api</artifactId>
4550
</dependency>
4651
<dependency>
47-
<groupId>${project.groupId}</groupId>
52+
<groupId>eu.itesla_project</groupId>
4853
<artifactId>iidm-network-api</artifactId>
4954
</dependency>
50-
55+
5156
<dependency>
52-
<groupId>eu.itesla_project</groupId>
57+
<groupId>${project.groupId}</groupId>
5358
<artifactId>cim1-model</artifactId>
5459
<version>${project.version}</version>
5560
</dependency>
@@ -60,6 +65,11 @@
6065
</dependency>
6166

6267
<!-- Test dependencies -->
68+
<dependency>
69+
<groupId>commons-io</groupId>
70+
<artifactId>commons-io</artifactId>
71+
<scope>test</scope>
72+
</dependency>
6373
<dependency>
6474
<groupId>junit</groupId>
6575
<artifactId>junit</artifactId>

‎cim1-import/src/main/java/eu/itesla_project/iidm/import_/cim1/CIM1Importer.java

+30-10
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.google.common.base.Suppliers;
1414
import com.google.gdata.util.io.base.UnicodeReader;
1515
import eu.itesla_project.commons.config.PlatformConfig;
16+
import eu.itesla_project.iidm.datasource.DataSourceUtil;
1617
import eu.itesla_project.iidm.datasource.ReadOnlyDataSource;
1718
import eu.itesla_project.iidm.import_.Importer;
1819
import eu.itesla_project.iidm.import_.Importers;
@@ -38,7 +39,6 @@
3839
import java.util.Properties;
3940

4041
/**
41-
*
4242
* @author Olivier Bretteville <olivier.bretteville at rte-france.com>
4343
*/
4444
@AutoService(Importer.class)
@@ -99,10 +99,10 @@ public String getComment() {
9999
}
100100

101101
private Packaging detectPackaging(ReadOnlyDataSource dataSource) throws IOException {
102-
if (dataSource.exists("_ME", "xml")) {
102+
if (exists(dataSource, "_ME", "xml")) {
103103
return Packaging.MERGED;
104104
}
105-
if (dataSource.exists("_EQ", "xml") && dataSource.exists("_TP", "xml") && dataSource.exists("_SV", "xml")) {
105+
if (exists(dataSource, "_EQ", "xml") && exists(dataSource, "_TP", "xml") && exists(dataSource, "_SV", "xml")) {
106106
return Packaging.SPLIT;
107107
}
108108
return null;
@@ -158,11 +158,11 @@ public boolean exists(ReadOnlyDataSource dataSource) {
158158
if (packaging != null) {
159159
switch (packaging) {
160160
case MERGED:
161-
try (InputStream is = dataSource.newInputStream("_ME", "xml")) {
161+
try (InputStream is = newInputStream(dataSource, "_ME", "xml")) {
162162
return isCim14(is);
163163
}
164164
case SPLIT:
165-
try (InputStream eqIs = dataSource.newInputStream("_EQ", "xml")) { // just test eq file to save time
165+
try (InputStream eqIs = newInputStream(dataSource, "_EQ", "xml")) { // just test eq file to save time
166166
return isCim14(eqIs);
167167
}
168168
default:
@@ -179,7 +179,7 @@ public boolean exists(ReadOnlyDataSource dataSource) {
179179

180180
private CIMModel loadMergedModel(ReadOnlyDataSource dataSource, Reader bseqr, Reader bstpr) throws Exception {
181181
CIMModel model = new CIMModel();
182-
try (Reader mer = new UnicodeReader(dataSource.newInputStream("_ME", "xml"), null)) {
182+
try (Reader mer = new UnicodeReader(newInputStream(dataSource, "_ME", "xml"), null)) {
183183

184184
long startTime2 = System.currentTimeMillis();
185185

@@ -195,9 +195,9 @@ private CIMModel loadMergedModel(ReadOnlyDataSource dataSource, Reader bseqr, Re
195195

196196
private CIMModel loadSplitModel(ReadOnlyDataSource dataSource, Reader bseqr, Reader bstpr) throws Exception {
197197
CIMModel model = new CIMModel();
198-
try (Reader eqr = new UnicodeReader(dataSource.newInputStream("_EQ", "xml"), null);
199-
Reader tpr = new UnicodeReader(dataSource.newInputStream("_TP", "xml"), null);
200-
Reader svr = new UnicodeReader(dataSource.newInputStream("_SV", "xml"), null)) {
198+
try (Reader eqr = new UnicodeReader(newInputStream(dataSource, "_EQ", "xml"), null);
199+
Reader tpr = new UnicodeReader(newInputStream(dataSource, "_TP", "xml"), null);
200+
Reader svr = new UnicodeReader(newInputStream(dataSource, "_SV", "xml"), null)) {
201201

202202
long startTime2 = System.currentTimeMillis();
203203

@@ -278,7 +278,7 @@ public Network import_(ReadOnlyDataSource dataSource, Properties parameters) {
278278
}
279279

280280
CIM1NamingStrategyFactory namingStrategyFactory;
281-
try (InputStream eqIs = dataSource.newInputStream("_EQ", "xml")) { // just test eq file to save time
281+
try (InputStream eqIs = newInputStream(dataSource, "_EQ", "xml")) { // just test eq file to save time
282282
namingStrategyFactory = usePsseNamingStrategy && isPtiCim14(eqIs) ? new CIM1PSSENamingStrategyFactory()
283283
: new CIM1DefaultNamingStrategyFactory();
284284
}
@@ -301,4 +301,24 @@ public Network import_(ReadOnlyDataSource dataSource, Properties parameters) {
301301
return network;
302302
}
303303

304+
private String getBaseName(ReadOnlyDataSource dataSource) {
305+
String basename = dataSource.getBaseName();
306+
if (basename.endsWith("_ME") || basename.endsWith("_EQ") || basename.endsWith("_TP") || basename.endsWith("_SV")) {
307+
basename = basename.substring(0, basename.length() - 3);
308+
}
309+
310+
return basename;
311+
}
312+
313+
private InputStream newInputStream(ReadOnlyDataSource dataSource, String suffix, String ext) throws IOException {
314+
String baseName = getBaseName(dataSource);
315+
String fileName = DataSourceUtil.getFileName(baseName, suffix, ext);
316+
return dataSource.newInputStream(fileName);
317+
}
318+
319+
private boolean exists(ReadOnlyDataSource dataSource, String suffix, String ext) throws IOException {
320+
String baseName = getBaseName(dataSource);
321+
String fileName = DataSourceUtil.getFileName(baseName, suffix, ext);
322+
return dataSource.exists(fileName);
323+
}
304324
}

‎cim1-import/src/test/java/eu/itesla_project/iidm/import_/cim1/CIM1ImporterTest.java

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/**
2+
* Copyright (c) 2017, RTE (http://www.rte-france.com)
3+
* This Source Code Form is subject to the terms of the Mozilla Public
4+
* License, v. 2.0. If a copy of the MPL was not distributed with this
5+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
*/
7+
package eu.itesla_project.iidm.import_.cim1;
8+
9+
import com.google.common.jimfs.Configuration;
10+
import com.google.common.jimfs.Jimfs;
11+
import eu.itesla_project.iidm.datasource.DataSource;
12+
import eu.itesla_project.iidm.datasource.FileDataSource;
13+
import eu.itesla_project.iidm.datasource.ReadOnlyDataSource;
14+
import eu.itesla_project.iidm.datasource.ZipFileDataSource;
15+
import org.apache.commons.io.IOUtils;
16+
import org.junit.After;
17+
import org.junit.Assert;
18+
import org.junit.Before;
19+
import org.junit.Test;
20+
21+
import java.io.IOException;
22+
import java.io.OutputStream;
23+
import java.nio.file.FileSystem;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.util.Properties;
27+
28+
/**
29+
* @author Mathieu Bague <mathieu.bague at rte-france.com>
30+
*/
31+
public class CIM1ImporterTest {
32+
33+
private FileSystem fileSystem;
34+
35+
private DataSource zdsMerged;
36+
private DataSource zdsSplit;
37+
private DataSource fdsMerged;
38+
private DataSource fdsUnzippedMerged;
39+
private DataSource fdsSplit;
40+
private DataSource fdsUnzippedSplit;
41+
42+
private void copyFile(DataSource dataSource, String filename) throws IOException {
43+
try (OutputStream stream = dataSource.newOutputStream(filename, false)) {
44+
IOUtils.copy(getClass().getResourceAsStream("/" + filename), stream);
45+
}
46+
}
47+
48+
@Before
49+
public void setUp() throws IOException {
50+
fileSystem = Jimfs.newFileSystem(Configuration.unix());
51+
52+
Path test1 = Files.createDirectory(fileSystem.getPath("test1"));
53+
fdsMerged = new FileDataSource(test1, "ieee14bus");
54+
fdsUnzippedMerged = new FileDataSource(test1, "ieee14bus_ME");
55+
copyFile(fdsMerged, "ieee14bus_ME.xml");
56+
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_EQ.xml");
57+
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_TP.xml");
58+
59+
Path test2 = Files.createDirectory(fileSystem.getPath("test2"));
60+
zdsMerged = new ZipFileDataSource(test2, "ieee14bus");
61+
copyFile(zdsMerged, "ieee14bus_ME.xml");
62+
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_EQ.xml");
63+
copyFile(fdsMerged, "ENTSO-E_Boundary_Set_EU_TP.xml");
64+
65+
Path test3 = Files.createDirectory(fileSystem.getPath("test3"));
66+
fdsSplit = new FileDataSource(test3, "ieee14bus");
67+
fdsUnzippedSplit = new FileDataSource(test3, "ieee14bus_EQ");
68+
copyFile(fdsSplit, "ieee14bus_EQ.xml");
69+
copyFile(fdsSplit, "ieee14bus_TP.xml");
70+
copyFile(fdsSplit, "ieee14bus_SV.xml");
71+
copyFile(fdsSplit, "ENTSO-E_Boundary_Set_EU_EQ.xml");
72+
copyFile(fdsSplit, "ENTSO-E_Boundary_Set_EU_TP.xml");
73+
74+
Path test4 = Files.createDirectory(fileSystem.getPath("test4"));
75+
zdsSplit = new ZipFileDataSource(test4, "ieee14bus");
76+
copyFile(zdsSplit, "ieee14bus_EQ.xml");
77+
copyFile(zdsSplit, "ieee14bus_TP.xml");
78+
copyFile(zdsSplit, "ieee14bus_SV.xml");
79+
copyFile(zdsSplit, "ENTSO-E_Boundary_Set_EU_EQ.xml");
80+
copyFile(zdsSplit, "ENTSO-E_Boundary_Set_EU_TP.xml");
81+
}
82+
83+
@After
84+
public void tearDown() throws IOException {
85+
fileSystem.close();
86+
}
87+
88+
@Test
89+
public void exists() {
90+
CIM1Importer importer = new CIM1Importer();
91+
Assert.assertEquals(true, importer.exists(fdsMerged));
92+
Assert.assertEquals(true, importer.exists(fdsUnzippedMerged));
93+
Assert.assertEquals(true, importer.exists(zdsMerged));
94+
Assert.assertEquals(true, importer.exists(fdsSplit));
95+
Assert.assertEquals(true, importer.exists(fdsUnzippedSplit));
96+
Assert.assertEquals(true, importer.exists(zdsSplit));
97+
}
98+
99+
@Test
100+
public void testImport() {
101+
testImport(fdsMerged);
102+
testImport(fdsUnzippedMerged);
103+
testImport(fdsSplit);
104+
testImport(fdsUnzippedSplit);
105+
}
106+
107+
private void testImport(ReadOnlyDataSource dataSource) {
108+
CIM1Importer importer = new CIM1Importer();
109+
try {
110+
importer.import_(dataSource, new Properties());
111+
Assert.fail();
112+
} catch (RuntimeException ignored) {
113+
}
114+
}
115+
}

‎cim1-import/src/test/resources/ENTSO-E_Boundary_Set_EU_EQ.xml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<!--
3+
4+
Copyright (c) 2017, RTE (http://www.rte-france.com)
5+
This Source Code Form is subject to the terms of the Mozilla Public
6+
License, v. 2.0. If a copy of the MPL was not distributed with this
7+
file, You can obtain one at http://mozilla.org/MPL/2.0/.
8+
9+
-->
10+
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:cim="http://iec.ch/TC57/2009/CIM-schema-cim14#"
11+
xmlns:md="http://iec.ch/TC57/2009/schema/CIM_model_description#">
12+
<!-- Fake ENTSO-E Boundary file (EQ file) -->
13+
</rdf:RDF>

‎cim1-import/src/test/resources/ENTSO-E_Boundary_Set_EU_TP.xml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<!--
3+
4+
Copyright (c) 2017, RTE (http://www.rte-france.com)
5+
This Source Code Form is subject to the terms of the Mozilla Public
6+
License, v. 2.0. If a copy of the MPL was not distributed with this
7+
file, You can obtain one at http://mozilla.org/MPL/2.0/.
8+
9+
-->
10+
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:cim="http://iec.ch/TC57/2009/CIM-schema-cim14#"
11+
xmlns:md="http://iec.ch/TC57/2009/schema/CIM_model_description#">
12+
<!-- Fake ENTSO-E Boundary file (TP file) -->
13+
</rdf:RDF>

0 commit comments

Comments
 (0)