Skip to content

Commit

Permalink
Merge pull request #808 from CanalTP/revert_dedup_simplification
Browse files Browse the repository at this point in the history 
[fix] Revert dedup simplification
  • Loading branch information
@datanel
datanel authored Aug 18, 2021
2 parents e2b460d + 1d711c8 commit 7bb454b
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 58 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
authors = ["Kisio Digital <[email protected]>", "Guillaume Pinot <[email protected]>"]
name = "transit_model"
version = "0.40.0"
version = "0.40.1"
license = "AGPL-3.0-only"
description = "Transit data management"
repository = "https://github.com/CanalTP/transit_model"
Expand Down
46 changes: 26 additions & 20 deletions src/model.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ use serde::{Deserialize, Serialize};
use skip_error::skip_error_and_log;
use std::{
cmp::{self, Ordering, Reverse},
collections::{BTreeMap, HashMap, HashSet},
collections::{hash_map::DefaultHasher, BTreeMap, HashMap, HashSet},
convert::TryFrom,
hash::{Hash, Hasher},
ops,
};
use typed_index_collection::{Collection, CollectionWithId, Id, Idx};
Expand Down Expand Up @@ -159,14 +160,19 @@ impl Collections {
}
}

fn dedup_collection<T: PartialEq>(source: &mut Collection<T>) {
let mut dedup = Vec::default();
for object in source.take() {
if !dedup.contains(&object) {
dedup.push(object);
}
fn dedup_collection<T: Clone + Eq + Hash>(source: &mut Collection<T>) -> Collection<T> {
let calculate_hash = |t: &T| -> u64 {
let mut s = DefaultHasher::new();
t.hash(&mut s);
s.finish()
};
let mut set: BTreeMap<u64, T> = BTreeMap::new();
let items = source.take();
for item in items {
set.insert(calculate_hash(&item), item);
}
*source = Collection::new(dedup);
let collection: Vec<T> = set.values().cloned().collect();
Collection::new(collection)
}

self.calendars
Expand Down Expand Up @@ -480,18 +486,18 @@ impl Collections {
.retain(|level| level_id_used.contains(&level.id));
self.calendars.retain(|c| calendars_used.contains(&c.id));

dedup_collection(&mut self.frequencies);
dedup_collection(&mut self.transfers);
dedup_collection(&mut self.admin_stations);
dedup_collection(&mut self.prices_v1);
dedup_collection(&mut self.od_fares_v1);
dedup_collection(&mut self.fares_v1);
dedup_collection(&mut self.ticket_prices);
dedup_collection(&mut self.ticket_use_perimeters);
dedup_collection(&mut self.ticket_use_restrictions);
dedup_collection(&mut self.grid_exception_dates);
dedup_collection(&mut self.grid_periods);
dedup_collection(&mut self.grid_rel_calendar_line);
self.frequencies = dedup_collection(&mut self.frequencies);
self.transfers = dedup_collection(&mut self.transfers);
self.admin_stations = dedup_collection(&mut self.admin_stations);
self.prices_v1 = dedup_collection(&mut self.prices_v1);
self.od_fares_v1 = dedup_collection(&mut self.od_fares_v1);
self.fares_v1 = dedup_collection(&mut self.fares_v1);
self.ticket_prices = dedup_collection(&mut self.ticket_prices);
self.ticket_use_perimeters = dedup_collection(&mut self.ticket_use_perimeters);
self.ticket_use_restrictions = dedup_collection(&mut self.ticket_use_restrictions);
self.grid_exception_dates = dedup_collection(&mut self.grid_exception_dates);
self.grid_periods = dedup_collection(&mut self.grid_periods);
self.grid_rel_calendar_line = dedup_collection(&mut self.grid_rel_calendar_line);

Ok(())
}
Expand Down
38 changes: 23 additions & 15 deletions src/objects.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use std::str::FromStr;
use thiserror::Error;
use typed_index_collection::{impl_id, impl_with_id, Idx, WithId};

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum ObjectType {
StopArea,
Expand Down Expand Up @@ -727,7 +727,7 @@ impl VehicleJourney {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
pub struct Frequency {
#[serde(rename = "trip_id")]
pub vehicle_journey_id: String,
Expand Down Expand Up @@ -755,7 +755,7 @@ impl From<std::num::ParseIntError> for TimeError {
}
}

#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Time(u32);
impl Time {
pub fn new(h: u32, m: u32, s: u32) -> Time {
Expand Down Expand Up @@ -1514,6 +1514,14 @@ impl AddPrefix for Transfer {
}
}

impl Hash for Transfer {
fn hash<H: Hasher>(&self, state: &mut H) {
(&self.from_stop_id, &self.to_stop_id).hash(state);
}
}

impl Eq for Transfer {}

#[derive(Serialize, Deserialize, Debug, Derivative, PartialEq, Clone)]
#[derivative(Default)]
pub enum TransportType {
Expand Down Expand Up @@ -1589,7 +1597,7 @@ impl AddPrefix for Geometry {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct AdminStation {
pub admin_id: String,
pub admin_name: String,
Expand All @@ -1603,7 +1611,7 @@ impl AddPrefix for AdminStation {
}
}

#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd)]
#[derive(Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct PriceV1 {
pub id: String,
#[serde(
Expand All @@ -1629,7 +1637,7 @@ impl AddPrefix for PriceV1 {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct OdFareV1 {
#[serde(rename = "Origin ID")]
pub origin_stop_area_id: String,
Expand All @@ -1656,7 +1664,7 @@ impl AddPrefix for OdFareV1 {
}
}

#[derive(Default, Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd)]
#[derive(Clone, Default, Serialize, Deserialize, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct FareV1 {
#[serde(rename = "avant changement")]
pub before_change: String,
Expand Down Expand Up @@ -1701,7 +1709,7 @@ impl AddPrefix for Ticket {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct TicketPrice {
pub ticket_id: String,
#[serde(rename = "ticket_price", deserialize_with = "de_positive_decimal")]
Expand Down Expand Up @@ -1748,15 +1756,15 @@ impl AddPrefix for TicketUse {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub enum PerimeterAction {
#[serde(rename = "1")]
Included,
#[serde(rename = "2")]
Excluded,
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct TicketUsePerimeter {
pub ticket_use_id: String,
pub object_type: ObjectType,
Expand All @@ -1771,15 +1779,15 @@ impl AddPrefix for TicketUsePerimeter {
}
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub enum RestrictionType {
#[serde(rename = "zone")]
Zone,
#[serde(rename = "OD")]
OriginDestination,
}

#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct TicketUseRestriction {
pub ticket_use_id: String,
pub restriction_type: RestrictionType,
Expand Down Expand Up @@ -1823,7 +1831,7 @@ impl AddPrefix for GridCalendar {
}
}

#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct GridExceptionDate {
pub grid_calendar_id: String,
#[serde(
Expand All @@ -1842,7 +1850,7 @@ impl AddPrefix for GridExceptionDate {
}
}

#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct GridPeriod {
pub grid_calendar_id: String,
#[serde(
Expand All @@ -1864,7 +1872,7 @@ impl AddPrefix for GridPeriod {
}
}

#[derive(Debug, Serialize, Deserialize, PartialEq)]
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub struct GridRelCalendarLine {
pub grid_calendar_id: String,
pub line_id: String,
Expand Down
44 changes: 22 additions & 22 deletions tests/fixtures/netex_france/output/correspondances.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
<dataObjects>
<GeneralFrame id="FR:GeneralFrame:NETEX_RESEAU:" version="any">
<members>
<SiteConnection id="FR:SiteConnection:GDLR_GDLM:" version="any">
<SiteConnection id="FR:SiteConnection:GDLB_GDLM:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT180S</DefaultDuration>
<DefaultDuration>PT120S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLR:">
<QuayRef ref="FR:Quay:GDLB:">
</QuayRef>
</From>
<To>
Expand All @@ -22,9 +22,9 @@
</QuayRef>
</To>
</SiteConnection>
<SiteConnection id="FR:SiteConnection:GDLM_GDLR:" version="any">
<SiteConnection id="FR:SiteConnection:GDLM_GDLB:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT200S</DefaultDuration>
<DefaultDuration>PT120S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
Expand All @@ -35,69 +35,69 @@
<To>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLR:">
<QuayRef ref="FR:Quay:GDLB:">
</QuayRef>
</To>
</SiteConnection>
<SiteConnection id="FR:SiteConnection:GDLR_GDLB:" version="any">
<SiteConnection id="FR:SiteConnection:GDLM_GDLR:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT600S</DefaultDuration>
<DefaultDuration>PT200S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLR:">
<QuayRef ref="FR:Quay:GDLM:">
</QuayRef>
</From>
<To>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLB:">
<QuayRef ref="FR:Quay:GDLR:">
</QuayRef>
</To>
</SiteConnection>
<SiteConnection id="FR:SiteConnection:GDLB_GDLR:" version="any">
<SiteConnection id="FR:SiteConnection:GDLR_GDLB:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT0S</DefaultDuration>
<DefaultDuration>PT600S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLB:">
<QuayRef ref="FR:Quay:GDLR:">
</QuayRef>
</From>
<To>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLR:">
<QuayRef ref="FR:Quay:GDLB:">
</QuayRef>
</To>
</SiteConnection>
<SiteConnection id="FR:SiteConnection:GDLM_GDLB:" version="any">
<SiteConnection id="FR:SiteConnection:GDLB_GDLR:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT120S</DefaultDuration>
<DefaultDuration>PT0S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLM:">
<QuayRef ref="FR:Quay:GDLB:">
</QuayRef>
</From>
<To>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLB:">
<QuayRef ref="FR:Quay:GDLR:">
</QuayRef>
</To>
</SiteConnection>
<SiteConnection id="FR:SiteConnection:GDLB_GDLM:" version="any">
<SiteConnection id="FR:SiteConnection:GDLR_GDLM:" version="any">
<WalkTransferDuration>
<DefaultDuration>PT120S</DefaultDuration>
<DefaultDuration>PT180S</DefaultDuration>
</WalkTransferDuration>
<From>
<StopPlaceRef ref="FR:StopPlace:GDL:">
</StopPlaceRef>
<QuayRef ref="FR:Quay:GDLB:">
<QuayRef ref="FR:Quay:GDLR:">
</QuayRef>
</From>
<To>
Expand All @@ -110,4 +110,4 @@
</members>
</GeneralFrame>
</dataObjects>
</PublicationDelivery>
</PublicationDelivery>

0 comments on commit 7bb454b

Please sign in to comment.