Merge pull request #841 from CanalTP/DATENG-120_gtfs_extended_route_type

[feature] ntfs2gtfs: possibility to write gtfs extended route type

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Kisio Digital <[email protected]>", "Guillaume Pinot <[email protected]>"]
 name = "transit_model"
-version = "0.46.0"
+version = "0.47.0"
 license = "AGPL-3.0-only"
 description = "Transit data management"
 repository = "https://github.com/CanalTP/transit_model"

documentation/ntfs_to_gtfs_specs.md

@@ -52,26 +52,29 @@ Each line of this file corresponds to a transit line modeled in the NTFS feed. I
 
 **Mapping of `route_type` with physical modes**
 
-| physical_mode_id in the NTFS | route_type in the GTFS | Priority w.r.t. NeTex | Absolute order |
-| ---------------------------- | ---------------------- | --------------------- | -------------- |
-| Tramway                      | 0                      | 5                     | 1              |
-| RailShuttle                  | 0                      | 3                     | 2              |
-| Metro                        | 1                      | 4                     | 3              |
-| LocalTrain                   | 2                      | 3                     | 4              |
-| LongDistanceTrain            | 2                      | 3                     | 5              |
-| RapidTransit                 | 2                      | 3                     | 6              |
-| Train                        | 2                      | 3                     | 7              |
-| BusRapidTransit              | 3                      | 7                     | 8              |
-| Bus                          | 3                      | 7                     | 9              |
-| Coach                        | 3                      | 7                     | 10             |
-| Boat                         | 4                      | 2                     | 11             |
-| Ferry                        | 4                      | 2                     | 12             |
-| Funicular                    | 7                      | 6                     | 13             |
-| Shuttle                      | 7                      | 6                     | 14             |
-| SuspendedCableCar            | 6                      | 7                     | 15             |
-
-The physical_modes Air and Taxi are not available in standard GTFS `route_type`s and should be considered as unknown for the GTFS (see below).
-If the physical_mode is unknown, trips should be considered as Bus (route_type = 3) and with a priority of 16 .
+| physical_mode_id in the NTFS | route_type in the GTFS | extended GTFS route_type | Priority w.r.t. NeTex | Absolute order |
+| ---------------------------- | ---------------------- | ------------------------ | --------------------- | -------------- |
+| Tramway                      | 0                      | 900                      | 5                     | 1              |
+| RailShuttle                  | 0                      | 900                      | 3                     | 2              |
+| Metro                        | 1                      | 400                      | 4                     | 3              |
+| LocalTrain                   | 2                      | 100                      | 3                     | 4              |
+| LongDistanceTrain            | 2                      | 100                      | 3                     | 5              |
+| RapidTransit                 | 2                      | 100                      | 3                     | 6              |
+| Train                        | 2                      | 100                      | 3                     | 7              |
+| BusRapidTransit              | 3                      | 700                      | 7                     | 8              |
+| Bus                          | 3                      | 700                      | 7                     | 9              |
+| Coach                        | 3                      | 200                      | 7                     | 10             |
+| Boat                         | 4                      | 1200                     | 2                     | 11             |
+| Ferry                        | 4                      | 1200                     | 2                     | 12             |
+| Funicular                    | 7                      | 1400                     | 6                     | 13             |
+| Shuttle                      | 7                      | 1400                     | 6                     | 14             |
+| SuspendedCableCar            | 6                      | 1300                     | 7                     | 15             |
+| Air                          | 3                      | 1100                     | 1                     | 16             |
+| Taxi                         | 3                      | 1500                     | 7                     | 17             |
+
+The `physical_modes` Air and Taxi are not available in standard GTFS `route_type`s and should be considered as unknown for the GTFS (see below).
+However, these `physical_modes` are available in extended mode.
+If the physical_mode is unknown, trips should be considered as Bus (route_type = 3) and with a priority of 18 .
 
 **Export of NTFS lines containing trips with different modes**
 A GTFS `route` can only contains trips with one mode (ie. `route_type`).

ntfs2gtfs/src/main.rs

@@ -47,6 +47,14 @@ struct Opt {
     /// Add the commercial mode at the beginning of the route short name.
     #[structopt(short, long)]
     mode_in_route_short_name: bool,
+
+    #[structopt(
+        long,
+        help = "Support a more rich set of route types. \
+                For more information, see \
+                https://developers.google.com/transit/gtfs/reference/extended-route-types"
+    )]
+    extend_route_type: bool,
 }
 
 fn init_logger() {
@@ -81,10 +89,10 @@ fn run(opt: Opt) -> Result<()> {
 
     match opt.output.extension() {
         Some(ext) if ext == "zip" => {
-            transit_model::gtfs::write_to_zip(model, opt.output)?;
+            transit_model::gtfs::write_to_zip(model, opt.output, opt.extend_route_type)?;
         }
         _ => {
-            transit_model::gtfs::write(model, opt.output)?;
+            transit_model::gtfs::write(model, opt.output, opt.extend_route_type)?;
         }
     };
     Ok(())

ntfs2gtfs/tests/fixtures/output_extended_route/routes.txt

@@ -0,0 +1,2 @@
+route_id,agency_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_sort_order
+line:1,network:kept,1,Metro 1,,400,,,,

ntfs2gtfs/tests/fixtures/output_split_route_by_mode_extended/routes.txt

@@ -0,0 +1,26 @@
+route_id,agency_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_sort_order
+MIPL:Bus,MIPL:PadamMobility,1,Line 1,,700,,,,
+MIPL:Air,MIPL:PadamMobility,2,Line 2,,1100,,,,
+MIPL:Bike,MIPL:PadamMobility,3,Line 3,,700,,,,
+MIPL:BikeSharingService,MIPL:PadamMobility,4,Line 4,,700,,,,
+MIPL:Car,MIPL:PadamMobility,5,Line 5,,700,,,,
+MIPL:Tramway,MIPL:PadamMobility,6,Line 6,,900,,,,
+MIPL:RailShuttle,MIPL:PadamMobility,7,Line 7,,900,,,,
+MIPL:Metro,MIPL:PadamMobility,8,Line 8,,400,,,,
+MIPL:LocalTrain,MIPL:PadamMobility,9,Line 9,,100,,,,
+MIPL:LongDistanceTrain,MIPL:PadamMobility,10,Line 10,,100,,,,
+MIPL:RapidTransit,MIPL:PadamMobility,11,Line 11,,100,,,,
+MIPL:Train,MIPL:PadamMobility,12,Line 12,,100,,,,
+MIPL:BusRapidTransit,MIPL:PadamMobility,13,Line 13,,700,,,,
+MIPL:Coach,MIPL:PadamMobility,14,Line 14,,200,,,,
+MIPL:Boat,MIPL:PadamMobility,15,Line 15,,1200,,,,
+MIPL:Ferry,MIPL:PadamMobility,16,Line 16,,1200,,,,
+MIPL:Funicular,MIPL:PadamMobility,17,Line 17,,1400,,,,
+MIPL:Shuttle,MIPL:PadamMobility,18,Line 18,,1400,,,,
+MIPL:SuspendedCableCar,MIPL:PadamMobility,19,Line 19,,1300,,,,
+MIPL:Taxi,MIPL:PadamMobility,20,Line 20,,1500,,,,
+MIPL:Bob,MIPL:PadamMobility,21,Line 21,,700,,,,
+MIPL:Bus-Coach,MIPL:PadamMobility,22,Line 22,,700,,,,
+MIPL:Bus-Coach:Coach,MIPL:PadamMobility,22,Line 22,,200,,,,
+MIPL:Bus-Taxi,MIPL:PadamMobility,22,Line 22,,700,,,,
+MIPL:Bus-Taxi:Taxi,MIPL:PadamMobility,22,Line 22,,1500,,,,

ntfs2gtfs/tests/ntfs2gtfs.rs

@@ -26,7 +26,7 @@ fn test_stop_zones_not_exported_and_cleaned() {
         collections.remove_stop_zones();
         collections.remove_route_points();
         let model = Model::new(collections).unwrap();
-        transit_model::gtfs::write(model, path).unwrap();
+        transit_model::gtfs::write(model, path, false).unwrap();
         compare_output_dir_with_expected(&path, None, "./tests/fixtures/output");
     });
 }
@@ -37,7 +37,7 @@ fn test_mode_in_route_shortname() {
         let input = "./tests/fixtures/input";
         let model = transit_model::ntfs::read(input).unwrap();
         let model = add_mode_to_line_code(model).unwrap();
-        transit_model::gtfs::write(model, path).unwrap();
+        transit_model::gtfs::write(model, path, false).unwrap();
         compare_output_dir_with_expected(
             &path,
             Some(vec!["routes.txt"]),
@@ -51,7 +51,7 @@ fn test_platforms_preserving() {
     test_in_tmp_dir(|path| {
         let input = "./tests/fixtures/platforms/input";
         let model = transit_model::ntfs::read(input).unwrap();
-        transit_model::gtfs::write(model, path).unwrap();
+        transit_model::gtfs::write(model, path, false).unwrap();
         compare_output_dir_with_expected(
             &path,
             Some(vec!["stops.txt"]),
@@ -121,6 +121,25 @@ fn test_ntfs2gtfs_create_foobar() {
     assert!(ntfs_foobar.join("agency.txt").is_file());
 }
 
+#[test]
+fn test_ntfs2gtfs_extended() {
+    let output_dir = TempDir::new().expect("create temp dir failed");
+    Command::cargo_bin("ntfs2gtfs")
+        .expect("Failed to find binary 'ntfs2gtfs'")
+        .arg("--input")
+        .arg("tests/fixtures/input/")
+        .arg("--output")
+        .arg(output_dir.path().to_str().unwrap())
+        .arg("--extend-route-type")
+        .assert()
+        .success();
+    compare_output_dir_with_expected(
+        &output_dir,
+        Some(vec!["routes.txt"]),
+        "./tests/fixtures/output_extended_route",
+    );
+}
+
 #[test]
 fn test_ntfs2gtfs_split_route_by_mode() {
     let output_dir = TempDir::new().expect("create temp dir failed");
@@ -138,3 +157,22 @@ fn test_ntfs2gtfs_split_route_by_mode() {
         "./tests/fixtures/output_split_route_by_mode",
     );
 }
+
+#[test]
+fn test_ntfs2gtfs_split_route_by_mode_extended() {
+    let output_dir = TempDir::new().expect("create temp dir failed");
+    Command::cargo_bin("ntfs2gtfs")
+        .expect("Failed to find binary 'ntfs2gtfs'")
+        .arg("--input")
+        .arg("tests/fixtures/input_split_route_by_mode")
+        .arg("--output")
+        .arg(output_dir.path().to_str().unwrap())
+        .arg("--extend-route-type")
+        .assert()
+        .success();
+    compare_output_dir_with_expected(
+        &output_dir,
+        Some(vec!["routes.txt"]),
+        "./tests/fixtures/output_split_route_by_mode_extended",
+    );
+}

src/gtfs/mod.rs

@@ -559,10 +559,83 @@ struct Route {
     sort_order: Option<u32>,
 }
 
+/// Use to serialize extended route type
+/// For more information, see \
+/// https://developers.google.com/transit/gtfs/reference/extended-route-types"
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+struct ExtendedRoute {
+    #[serde(rename = "route_id")]
+    id: String,
+    agency_id: Option<String>,
+    #[serde(rename = "route_short_name")]
+    short_name: String,
+    #[serde(rename = "route_long_name")]
+    long_name: String,
+    #[serde(rename = "route_desc")]
+    desc: Option<String>,
+    #[serde(serialize_with = "ser_from_route_type_extended")]
+    route_type: RouteType,
+    #[serde(rename = "route_url")]
+    url: Option<String>,
+    #[serde(
+        rename = "route_color",
+        default,
+        deserialize_with = "de_with_empty_or_invalid_default"
+    )]
+    color: Option<objects::Rgb>,
+    #[serde(
+        rename = "route_text_color",
+        default,
+        deserialize_with = "de_with_empty_or_invalid_default"
+    )]
+    text_color: Option<objects::Rgb>,
+    #[serde(rename = "route_sort_order")]
+    sort_order: Option<u32>,
+}
+
+impl From<Route> for ExtendedRoute {
+    fn from(route: Route) -> Self {
+        Self {
+            id: route.id,
+            agency_id: route.agency_id,
+            short_name: route.short_name,
+            long_name: route.long_name,
+            desc: route.desc,
+            route_type: route.route_type,
+            url: route.url,
+            color: route.color,
+            text_color: route.text_color,
+            sort_order: route.sort_order,
+        }
+    }
+}
+
+fn to_gtfs_extended_value(route_type: &RouteType) -> String {
+    match *route_type {
+        RouteType::Tramway => "900".to_string(),
+        RouteType::Metro => "400".to_string(),
+        RouteType::Train => "100".to_string(),
+        RouteType::Bus | RouteType::UnknownMode => "700".to_string(),
+        RouteType::Ferry => "1200".to_string(),
+        RouteType::Funicular => "1400".to_string(),
+        RouteType::CableCar | RouteType::SuspendedCableCar => "1300".to_string(),
+        RouteType::Coach => "200".to_string(),
+        RouteType::Air => "1100".to_string(),
+        RouteType::Taxi => "1500".to_string(),
+    }
+}
+
+fn ser_from_route_type_extended<S>(r: &RouteType, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    serializer.serialize_str(&to_gtfs_extended_value(r))
+}
+
 /// Exports a `Model` to [GTFS](https://gtfs.org/reference/static) files
 /// in the given directory.
 /// see [NTFS to GTFS conversion](https://github.com/CanalTP/transit_model/blob/master/src/documentation/ntfs2gtfs.md)
-pub fn write<P: AsRef<Path>>(model: Model, path: P) -> Result<()> {
+pub fn write<P: AsRef<Path>>(model: Model, path: P, extend_route_type: bool) -> Result<()> {
     let path = path.as_ref();
     std::fs::create_dir_all(path)?;
     info!("Writing GTFS to {:?}", path);
@@ -579,7 +652,7 @@ pub fn write<P: AsRef<Path>>(model: Model, path: P) -> Result<()> {
         &model.equipments,
     )?;
     write::write_trips(path, &model)?;
-    write::write_routes(path, &model)?;
+    write::write_routes(path, &model, extend_route_type)?;
     write::write_stop_extensions(path, &model.stop_points, &model.stop_areas)?;
     write::write_stop_times(
         path,
@@ -597,11 +670,15 @@ pub fn write<P: AsRef<Path>>(model: Model, path: P) -> Result<()> {
 /// Exports a `Model` to [GTFS](https://gtfs.org/reference/static) files
 /// in the given ZIP archive.
 /// see [NTFS to GTFS conversion](https://github.com/CanalTP/transit_model/blob/master/src/documentation/ntfs2gtfs.md)
-pub fn write_to_zip<P: AsRef<std::path::Path>>(model: Model, path: P) -> Result<()> {
+pub fn write_to_zip<P: AsRef<std::path::Path>>(
+    model: Model,
+    path: P,
+    extend_route_type: bool,
+) -> Result<()> {
     let path = path.as_ref();
     info!("Writing GTFS to ZIP File {:?}", path);
     let input_tmp_dir = tempfile::tempdir()?;
-    write(model, input_tmp_dir.path())?;
+    write(model, input_tmp_dir.path(), extend_route_type)?;
     zip_to(input_tmp_dir.path(), path)?;
     input_tmp_dir.close()?;
     Ok(())

src/gtfs/write.rs

@@ -16,6 +16,7 @@ use super::{
     Agency, DirectionType, Route, RouteType, Shape, Stop, StopLocationType, StopTime, Transfer,
     Trip,
 };
+use crate::gtfs::ExtendedRoute;
 use crate::model::{GetCorresponding, Model};
 use crate::objects;
 use crate::objects::Transfer as NtfsTransfer;
@@ -344,10 +345,13 @@ impl<'a> From<&'a objects::PhysicalMode> for RouteType {
             "RailShuttle" | "Tramway" => RouteType::Tramway,
             "Metro" => RouteType::Metro,
             "LocalTrain" | "LongDistanceTrain" | "RapidTransit" | "Train" => RouteType::Train,
-            "Bus" | "BusRapidTransit" | "Coach" => RouteType::Bus,
+            "Bus" | "BusRapidTransit" => RouteType::Bus,
+            "Coach" => RouteType::Coach,
             "Boat" | "Ferry" => RouteType::Ferry,
             "Funicular" | "Shuttle" => RouteType::Funicular,
             "SuspendedCableCar" => RouteType::SuspendedCableCar,
+            "Air" => RouteType::Air,
+            "Taxi" => RouteType::Taxi,
             _ => RouteType::UnknownMode,
         }
     }
@@ -378,7 +382,9 @@ fn get_physical_mode_order(pm: &objects::PhysicalMode) -> u8 {
         "Funicular" => 13,
         "Shuttle" => 14,
         "SuspendedCableCar" => 15,
-        _ => 16,
+        "Air" => 16,
+        "Taxi" => 17,
+        _ => 18,
     }
 }
 
@@ -397,15 +403,21 @@ fn make_gtfs_route_from_ntfs_line(line: &objects::Line, pm: &PhysicalModeWithOrd
     }
 }
 
-pub fn write_routes(path: &path::Path, model: &Model) -> Result<()> {
+pub fn write_routes(path: &path::Path, model: &Model, extend_route_type: bool) -> Result<()> {
     info!("Writing routes.txt");
     let path = path.join("routes.txt");
     let mut wtr =
         csv::Writer::from_path(&path).with_context(|| format!("Error reading {:?}", path))?;
     for (from, l) in &model.lines {
         for pm in &get_line_physical_modes(from, &model.physical_modes, model) {
-            wtr.serialize(make_gtfs_route_from_ntfs_line(l, pm))
-                .with_context(|| format!("Error reading {:?}", path))?;
+            let route = make_gtfs_route_from_ntfs_line(l, pm);
+            if extend_route_type {
+                wtr.serialize(ExtendedRoute::from(route))
+                    .with_context(|| format!("Error reading {:?}", path))?;
+            } else {
+                wtr.serialize(route)
+                    .with_context(|| format!("Error reading {:?}", path))?;
+            }
         }
     }