Skip to content

Commit

Permalink
Merge pull request osmlab#72 from maps-osm/dev-merge
Browse files Browse the repository at this point in the history 
Dev merge for Asterix
  • Loading branch information
@jwpgage
jwpgage authored and GitHub Enterprise committed Mar 26, 2018
2 parents 0705a84 + 5b21d6b commit ac7395a
Show file tree
Hide file tree
Showing 29 changed files with 2,724 additions and 139 deletions.
171 changes: 171 additions & 0 deletions ...t/java/org/openstreetmap/atlas/geography/atlas/raw/DynamicRawAtlasSectioningTestRule.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
package org.openstreetmap.atlas.geography.atlas.raw;

import org.openstreetmap.atlas.geography.atlas.Atlas;
import org.openstreetmap.atlas.utilities.testing.CoreTestRule;
import org.openstreetmap.atlas.utilities.testing.TestAtlas;
import org.openstreetmap.atlas.utilities.testing.TestAtlas.Line;
import org.openstreetmap.atlas.utilities.testing.TestAtlas.Loc;
import org.openstreetmap.atlas.utilities.testing.TestAtlas.Point;

/**
* {@link OsmPbfToSlicedRawAtlasTest} test data.
*
* @author mgostintsev
*/
public class DynamicRawAtlasSectioningTestRule extends CoreTestRule
{
// Fully inside 8-123-122
private static final String ONE = "7.9747091, -6.6837721";
private static final String TWO = "7.9737754, -6.6823776";

// Fully inside 8-123-123
private static final String THREE = "5.7826593, -6.5920337";
private static final String FOUR = "5.7827163, -6.5914695";

// Fully inside 7-62-61
private static final String FIVE = "6.7240498, -3.4840235";
private static final String SIX = "6.7260445, -3.4840096";

// Crossing 8-123-122 into 8-123-123
private static final String SEVEN = "7.0401813, -6.4733942";
private static final String EIGHT = "6.8870387, -6.4641103";
private static final String NINE = " 6.8832099, -6.4636300";
private static final String TEN = "6.8866444, -6.4654545";

// Crossing 8-123-122 into 7-62-61
private static final String ELEVEN = "7.6829818, -5.6379654";
private static final String TWELVE = "7.6735590, -5.5867130";
private static final String THIRTEEN = "7.6717846, -5.5812728";
private static final String FOURTEEN = "7.6681651, -5.5865950";

// Crossing 8-123-123 into 7-62-61
private static final String FIFTEEN = "5.8751274, -5.6744400";
private static final String SIXTEEN = "5.8641274, -5.5664451";
private static final String SEVENTEEN = "5.8545285, -5.5181042";
private static final String EIGHTEEN = "5.8813557, -5.5808398";

// Starting in 8-123-122, through 8-123-123, ending in 7-62-61
private static final String NINETEEN = "7.3884501, -6.4772718";
private static final String TWENTY = "7.3864604, -6.4728799";
private static final String TWENTY_ONE = "6.8787867, -6.4534416";
private static final String TWENTY_TWO = "6.8738255, -6.4553079";
private static final String TWENTY_THREE = "6.8299549, -5.2608135";
private static final String TWENTY_FOUR = "6.8298791, -5.2562899";
private static final String TWENTY_FIVE = "7.3868182, -6.4796125";
private static final String TWENTY_SIX = "6.8769819, -6.4596778";
private static final String TWENTY_SEVEN = "6.8333518, -5.2585345";

@TestAtlas(lines = {
@Line(id = "541701001000", coordinates = { @Loc(value = THREE),
@Loc(value = FOUR) }, tags = { "highway=primary" }),
@Line(id = "541702001000", coordinates = { @Loc(value = SEVEN), @Loc(value = EIGHT),
@Loc(value = NINE) }, tags = { "highway=primary" }),
@Line(id = "541703001000", coordinates = { @Loc(value = EIGHT),
@Loc(value = TEN) }, tags = { "highway=primary" }),
@Line(id = "541704001000", coordinates = { @Loc(value = TWENTY_SIX),
@Loc(value = TWENTY_ONE) }, tags = { "highway=primary" }),
@Line(id = "541705001000", coordinates = { @Loc(value = FIFTEEN), @Loc(value = SIXTEEN),
@Loc(value = SEVENTEEN) }, tags = { "highway=primary" }),
@Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY),
@Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE),
@Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" })

}, points = {

@Point(id = "511111003", coordinates = @Loc(value = THREE)),
@Point(id = "511111004", coordinates = @Loc(value = FOUR)),
@Point(id = "511111009", coordinates = @Loc(value = NINE)),
@Point(id = "511111007", coordinates = @Loc(value = SEVEN)),
@Point(id = "511111008", coordinates = @Loc(value = EIGHT)),
@Point(id = "511111010", coordinates = @Loc(value = TEN)),
@Point(id = "511111015", coordinates = @Loc(value = FIFTEEN)),
@Point(id = "511111017", coordinates = @Loc(value = SEVENTEEN)),
@Point(id = "511111019", coordinates = @Loc(value = NINETEEN)),
@Point(id = "511111021", coordinates = @Loc(value = TWENTY_ONE)),
@Point(id = "511111026", coordinates = @Loc(value = TWENTY_SIX)),
@Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)),
@Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR)) })
private Atlas atlasZ8x123y123;

@TestAtlas(lines = {
@Line(id = "541707001000", coordinates = { @Loc(value = ONE),
@Loc(value = TWO) }, tags = { "highway=primary" }),
@Line(id = "541702001000", coordinates = { @Loc(value = SEVEN), @Loc(value = EIGHT),
@Loc(value = NINE) }, tags = { "highway=primary" }),
@Line(id = "541708001000", coordinates = { @Loc(value = ELEVEN), @Loc(value = TWELVE),
@Loc(value = THIRTEEN) }, tags = { "highway=primary" }),
@Line(id = "541709001000", coordinates = { @Loc(value = TWENTY_FIVE),
@Loc(value = TWENTY) }, tags = { "highway=primary" }),
@Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY),
@Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE),
@Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" })

}, points = {

@Point(id = "511111001", coordinates = @Loc(value = ONE)),
@Point(id = "511111002", coordinates = @Loc(value = TWO)),
@Point(id = "511111011", coordinates = @Loc(value = ELEVEN)),
@Point(id = "511111013", coordinates = @Loc(value = THIRTEEN)),
@Point(id = "511111007", coordinates = @Loc(value = SEVEN)),
@Point(id = "511111009", coordinates = @Loc(value = NINE)),
@Point(id = "511111019", coordinates = @Loc(value = NINETEEN)),
@Point(id = "511111020", coordinates = @Loc(value = TWENTY)),
@Point(id = "511111025", coordinates = @Loc(value = TWENTY_FIVE)),
@Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)),
@Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR))

})
private Atlas atlasZ8x123y122;

@TestAtlas(lines = {

@Line(id = "541710001000", coordinates = { @Loc(value = FIVE),
@Loc(value = SIX) }, tags = { "highway=primary" }),
@Line(id = "541711001000", coordinates = { @Loc(value = TWELVE),
@Loc(value = FOURTEEN) }, tags = { "highway=primary" }),
@Line(id = "541708001000", coordinates = { @Loc(value = ELEVEN), @Loc(value = TWELVE),
@Loc(value = THIRTEEN) }, tags = { "highway=primary" }),
@Line(id = "541712001000", coordinates = { @Loc(value = TWENTY_SEVEN),
@Loc(value = TWENTY_THREE) }, tags = { "highway=primary" }),
@Line(id = "541713001000", coordinates = { @Loc(value = SIXTEEN),
@Loc(value = EIGHTEEN) }, tags = { "highway=primary" }),
@Line(id = "541705001000", coordinates = { @Loc(value = FIFTEEN), @Loc(value = SIXTEEN),
@Loc(value = SEVENTEEN) }, tags = { "highway=primary" }),
@Line(id = "541706001000", coordinates = { @Loc(value = NINETEEN), @Loc(value = TWENTY),
@Loc(value = TWENTY_ONE), @Loc(value = TWENTY_TWO), @Loc(value = TWENTY_THREE),
@Loc(value = TWENTY_FOUR) }, tags = { "highway=primary" }) }, points = {

@Point(id = "511111005", coordinates = @Loc(value = FIVE)),
@Point(id = "511111006", coordinates = @Loc(value = SIX)),
@Point(id = "511111011", coordinates = @Loc(value = ELEVEN)),
@Point(id = "511111012", coordinates = @Loc(value = TWELVE)),
@Point(id = "511111013", coordinates = @Loc(value = THIRTEEN)),
@Point(id = "511111014", coordinates = @Loc(value = FOURTEEN)),
@Point(id = "511111015", coordinates = @Loc(value = FIFTEEN)),
@Point(id = "511111016", coordinates = @Loc(value = SIXTEEN)),
@Point(id = "511111017", coordinates = @Loc(value = SEVENTEEN)),
@Point(id = "511111018", coordinates = @Loc(value = EIGHTEEN)),
@Point(id = "511111019", coordinates = @Loc(value = NINETEEN)),
@Point(id = "511111022", coordinates = @Loc(value = TWENTY_TWO)),
@Point(id = "511111024", coordinates = @Loc(value = TWENTY_FOUR)),
@Point(id = "511111023", coordinates = @Loc(value = TWENTY_THREE)),
@Point(id = "511111027", coordinates = @Loc(value = TWENTY_SEVEN))

})
private Atlas atlasZ7x62y61;

public Atlas getAtlasz7x62y61()
{
return this.atlasZ7x62y61;
}

public Atlas getAtlasz8x123y122()
{
return this.atlasZ8x123y122;
}

public Atlas getAtlasz8x123y123()
{
return this.atlasZ8x123y123;
}
}
173 changes: 156 additions & 17 deletions ...tionTest/java/org/openstreetmap/atlas/geography/atlas/raw/OsmPbfToSlicedRawAtlasTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,40 @@
package org.openstreetmap.atlas.geography.atlas.raw;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;

import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.openstreetmap.atlas.geography.atlas.Atlas;
import org.openstreetmap.atlas.geography.atlas.ShardFileOverlapsPolygonTest;
import org.openstreetmap.atlas.geography.atlas.items.Edge;
import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption;
import org.openstreetmap.atlas.geography.atlas.pbf.OsmPbfLoader;
import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator;
import org.openstreetmap.atlas.geography.atlas.raw.sectioning.WaySectionProcessor;
import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasCountrySlicer;
import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap;
import org.openstreetmap.atlas.geography.sharding.DynamicTileSharding;
import org.openstreetmap.atlas.geography.sharding.Shard;
import org.openstreetmap.atlas.geography.sharding.SlippyTile;
import org.openstreetmap.atlas.locale.IsoCountry;
import org.openstreetmap.atlas.streaming.compression.Decompressor;
import org.openstreetmap.atlas.streaming.resource.File;
import org.openstreetmap.atlas.streaming.resource.InputStreamResource;
import org.openstreetmap.atlas.tags.ISOCountryTag;
import org.openstreetmap.atlas.tags.annotations.validation.Validators;
import org.openstreetmap.atlas.utilities.collections.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Tests OSM PBF --> Raw Atlas --> Sliced Raw Atlas flow.
* Tests several Raw Atlas Flows: 1. OSM PBF to Sliced Raw Atlas 2. Sliced Raw Atlas to Sectioned
* Raw Atlas 3. Multiple Sliced Raw Atlases to Sectioned Raw atlas
*
* @author mgostintsev
*/
Expand All @@ -31,6 +43,10 @@ public class OsmPbfToSlicedRawAtlasTest
private static CountryBoundaryMap COUNTRY_BOUNDARY_MAP;
private static Set<IsoCountry> COUNTRIES;

private static final long LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS = 541706;

private static final Logger logger = LoggerFactory.getLogger(OsmPbfToSlicedRawAtlasTest.class);

static
{
COUNTRIES = new HashSet<>();
Expand All @@ -44,29 +60,77 @@ public class OsmPbfToSlicedRawAtlasTest
.withDecompressor(Decompressor.GZIP));
}

@Rule
public DynamicRawAtlasSectioningTestRule setup = new DynamicRawAtlasSectioningTestRule();

@Test
public void testPbfToCountrySlicedAndWaySectionedAtlas()
public void testPbfToSlicedAtlasWithExpansion()
{
final String path = OsmPbfToSlicedRawAtlasTest.class.getResource("8-122-122.osm.pbf")
.getPath();
final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path));
final Atlas rawAtlas = rawAtlasGenerator.build();
// Create a simple store, populated with 3 shards and the corresponding atlases.
final Map<Shard, Atlas> store = prepareShardStore();
final Function<Shard, Optional<Atlas>> rawAtlasFetcher = shard ->
{
if (store.containsKey(shard))
{
return Optional.of(store.get(shard));
}
else
{
return Optional.empty();
}
};

final Atlas slicedRawAtlas = new RawAtlasCountrySlicer(COUNTRIES, COUNTRY_BOUNDARY_MAP)
.slice(rawAtlas);
// Create 3 atlas files, starting from each of the different shards
final Atlas atlasFromz8x123y122 = generateSectionedAtlasStartingAtShard(
new SlippyTile(123, 122, 8), rawAtlasFetcher);
logger.info(atlasFromz8x123y122.summary());

final Atlas finalAtlas = new WaySectionProcessor(slicedRawAtlas,
AtlasLoadingOption.createOptionWithAllEnabled(COUNTRY_BOUNDARY_MAP)).run();
final Atlas atlasFromz8x123y123 = generateSectionedAtlasStartingAtShard(
new SlippyTile(123, 123, 8), rawAtlasFetcher);
logger.info(atlasFromz8x123y123.summary());

System.out.println(finalAtlas.summary());
final Atlas atlasFromz7x62y61 = generateSectionedAtlasStartingAtShard(
new SlippyTile(62, 61, 7), rawAtlasFetcher);
logger.info(atlasFromz7x62y61.summary());

// TODO - compare the old atlas to the new atlas to see differences
}
// Let's focus on the edge spanning all 3 shards and verify it got sectioned properly.
final Iterable<Edge> firstGroupOfEdges = atlasFromz8x123y122
.edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS);
final Iterable<Edge> secondGroupOfEdges = atlasFromz8x123y123
.edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS);
final Iterable<Edge> thirdGroupOfEdges = atlasFromz7x62y61
.edges(edge -> edge.getOsmIdentifier() == LINE_OSM_IDENTIFIER_CROSSING_3_SHARDS);

@Test
public void testPbfToSlicedAtlasWithExpansion()
{
// TODO
// First look at absolute counts. Each shard will have two forward and reverse edges
Assert.assertTrue(Iterables.size(firstGroupOfEdges) == 4);
Assert.assertTrue(Iterables.size(secondGroupOfEdges) == 4);
Assert.assertTrue(Iterables.size(thirdGroupOfEdges) == 4);

// Next, let's check identifier consistency
final Set<Long> uniqueIdentifiers = new HashSet<>();
Iterables.stream(firstGroupOfEdges)
.forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier()));
Iterables.stream(secondGroupOfEdges)
.forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier()));
Iterables.stream(thirdGroupOfEdges)
.forEach(edge -> uniqueIdentifiers.add(edge.getIdentifier()));

// There should be 4 pieces (each having a forward and reverse edge) total.
Assert.assertTrue(uniqueIdentifiers.size() == 8);

// Validate the same edge identifiers built from different shards to test equality
final Edge piece2from122 = atlasFromz8x123y122.edge(541706001002L);
final Edge piece2from123 = atlasFromz8x123y123.edge(541706001002L);
Assert.assertTrue(piece2from122.asPolyLine().equals(piece2from123.asPolyLine()));

final Edge piece3from123 = atlasFromz8x123y123.edge(541706001003L);
final Edge piece3from62 = atlasFromz7x62y61.edge(541706001003L);
Assert.assertTrue(piece3from123.asPolyLine().equals(piece3from62.asPolyLine()));

// Let's validate absolute number of edges in each shard
Assert.assertTrue(atlasFromz8x123y122.numberOfEdges() == 12);
Assert.assertTrue(atlasFromz8x123y123.numberOfEdges() == 16);
Assert.assertTrue(atlasFromz7x62y61.numberOfEdges() == 20);
}

@Test
Expand Down Expand Up @@ -125,4 +189,79 @@ public void testPbfToSlicedRawAtlas()
+ oldSlicedAtlas.numberOfAreas()
+ oldSlicedAtlas.numberOfLines() == slicedRawAtlas.numberOfLines() - 1);
}

@Test
public void testSectioningFromRawAtlas()
{
final String path = OsmPbfToSlicedRawAtlasTest.class.getResource("8-122-122.osm.pbf")
.getPath();
final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path));
final Atlas rawAtlas = rawAtlasGenerator.build();

final Atlas slicedRawAtlas = new RawAtlasCountrySlicer(COUNTRIES, COUNTRY_BOUNDARY_MAP)
.slice(rawAtlas);

final Atlas finalAtlas = new WaySectionProcessor(slicedRawAtlas,
AtlasLoadingOption.createOptionWithAllEnabled(COUNTRY_BOUNDARY_MAP)).run();

logger.info(finalAtlas.summary());
}

@Test
public void testSectioningFromShard()
{
final String path = OsmPbfToSlicedRawAtlasTest.class.getResource("8-122-122.osm.pbf")
.getPath();
final RawAtlasGenerator rawAtlasGenerator = new RawAtlasGenerator(new File(path));
final Atlas rawAtlas = rawAtlasGenerator.build();

final Atlas slicedRawAtlas = new RawAtlasCountrySlicer(COUNTRIES, COUNTRY_BOUNDARY_MAP)
.slice(rawAtlas);

// Simple fetcher that returns the atlas from above for the corresponding shard
final Map<Shard, Atlas> store = new HashMap<>();
store.put(new SlippyTile(122, 122, 8), slicedRawAtlas);
final Function<Shard, Optional<Atlas>> rawAtlasFetcher = shard ->
{
if (store.containsKey(shard))
{
return Optional.of(store.get(shard));
}
else
{
return Optional.empty();
}
};

final Atlas finalAtlas = new WaySectionProcessor(new SlippyTile(122, 122, 8),
AtlasLoadingOption.createOptionWithAllEnabled(COUNTRY_BOUNDARY_MAP),
new DynamicTileSharding(new File(ShardFileOverlapsPolygonTest.class
.getResource(
"/org/openstreetmap/atlas/geography/boundary/tree-6-14-100000.txt.gz")
.getFile())),
rawAtlasFetcher).run();

logger.info(finalAtlas.summary());
}

private Atlas generateSectionedAtlasStartingAtShard(final Shard shard,
final Function<Shard, Optional<Atlas>> rawAtlasFetcher)
{
return new WaySectionProcessor(shard,
AtlasLoadingOption.createOptionWithAllEnabled(COUNTRY_BOUNDARY_MAP),
new DynamicTileSharding(new File(ShardFileOverlapsPolygonTest.class
.getResource(
"/org/openstreetmap/atlas/geography/boundary/tree-6-14-100000.txt.gz")
.getFile())),
rawAtlasFetcher).run();
}

private Map<Shard, Atlas> prepareShardStore()
{
final Map<Shard, Atlas> store = new HashMap<>();
store.put(new SlippyTile(62, 61, 7), this.setup.getAtlasz7x62y61());
store.put(new SlippyTile(123, 123, 8), this.setup.getAtlasz8x123y123());
store.put(new SlippyTile(123, 122, 8), this.setup.getAtlasz8x123y122());
return store;
}
}
Loading

0 comments on commit ac7395a

Please sign in to comment.