[feature] Don't export dummy stop time

ntfs2gtfs/src/main.rs

@@ -23,7 +23,7 @@ use tracing_subscriber::{
     layer::SubscriberExt as _,
     util::SubscriberInitExt as _,
 };
-use transit_model::Result;
+use transit_model::{Model, Result};
 
 lazy_static::lazy_static! {
     pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION"));
@@ -70,8 +70,10 @@ fn init_logger() {
 
 fn run(opt: Opt) -> Result<()> {
     info!("Launching ntfs2gtfs...");
-    let mut model;
-    model = transit_model::ntfs::read(opt.input)?;
+    let mut collections = transit_model::ntfs::read_collections(opt.input)?;
+    collections.remove_stop_zones();
+    collections.remove_route_points();
+    let mut model = Model::new(collections)?;
 
     if opt.mode_in_route_short_name {
         model = add_mode_to_line_code(model)?;

ntfs2gtfs/tests/fixtures/input/stop_times.txt

@@ -1,8 +1,9 @@
-trip_id,stop_sequence,stop_id,arrival_time,departure_time,datetime_estimated,stop_time_id,stop_headsign
-trip:1,0,stop:point:1,9:00:00,9:00:00,,stoptime:1,
-trip:1,1,stop:point:2,09:10:00,09:10:00,,stoptime:2,
-trip:2,0,stop:zone:1,9:00:00,9:00:00,,stoptime:3,
-trip:2,1,stop:zone:2,09:10:00,09:10:00,,stoptime:4,
-trip:3,0,stop:point:1,9:00:00,9:00:00,,stoptime:5,
-trip:3,1,stop:zone:2,09:10:00,09:10:00,,stoptime:5,
-trip:3,3,stop:point:2,9:20:00,9:20:00,,stoptime:6,
+trip_id,stop_sequence,stop_id,arrival_time,departure_time,datetime_estimated,stop_time_id,stop_headsign,pickup_type,drop_off_type
+trip:1,0,stop:point:1,9:00:00,9:00:00,,stoptime:1,,0,0
+trip:1,1,stop:point:3,09:05:00,09:05:00,,stoptime:3,,0,3
+trip:1,2,stop:point:2,09:10:00,09:10:00,,stoptime:2,,0,0
+trip:2,0,stop:zone:1,9:00:00,9:00:00,,stoptime:3,,0,0
+trip:2,1,stop:zone:2,09:10:00,09:10:00,,stoptime:4,,0,0
+trip:3,0,stop:point:1,9:00:00,9:00:00,,stoptime:5,,0,0
+trip:3,1,stop:zone:2,09:10:00,09:10:00,,stoptime:5,,0,0
+trip:3,3,stop:point:2,9:20:00,9:20:00,,stoptime:6,,0,0

ntfs2gtfs/tests/fixtures/input/stops.txt

@@ -3,6 +3,7 @@ stop:area:1,whatever,48.8,2.3,1,,,
 stop:area:2,whatever,48.8,2.3,1,,,
 stop:point:1,whatever,48.8,2.3,0,stop:area:1,,point1
 stop:point:2,whatever,48.8,2.3,0,stop:area:1,,
+stop:point:3,whatever,48.8,2.3,0,stop:area:1,,
 stop:zone:1,whatever,48.8,2.3,2,stop:area:2,,
 stop:zone:2,whatever,48.8,2.3,2,stop:area:2,,
 stop:entrance:1,whatever,48.8,2.3,3,stop:area:1,,entrance1

ntfs2gtfs/tests/fixtures/output/stop_times.txt

@@ -1,3 +1,3 @@
 trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type,local_zone_id,stop_headsign,timepoint
 trip:1,09:00:00,09:00:00,stop:point:1,0,0,1,,,1
-trip:1,09:10:00,09:10:00,stop:point:2,1,1,0,,,1
+trip:1,09:10:00,09:10:00,stop:point:2,2,1,0,,,1

ntfs2gtfs/tests/ntfs2gtfs.rs

@@ -16,13 +16,16 @@ use assert_cmd::prelude::*;
 use ntfs2gtfs::add_mode_to_line_code;
 use std::process::Command;
 use tempfile::TempDir;
-use transit_model::test_utils::*;
+use transit_model::{test_utils::*, Model};
 
 #[test]
 fn test_stop_zones_not_exported_and_cleaned() {
     test_in_tmp_dir(|path| {
         let input = "./tests/fixtures/input";
-        let model = transit_model::ntfs::read(input).unwrap();
+        let mut collections = transit_model::ntfs::read_collections(input).unwrap();
+        collections.remove_stop_zones();
+        collections.remove_route_points();
+        let model = Model::new(collections).unwrap();
         transit_model::gtfs::write(model, path).unwrap();
         compare_output_dir_with_expected(&path, None, "./tests/fixtures/output");
     });

ntfs2netexfr/src/main.rs

@@ -23,7 +23,7 @@ use tracing_subscriber::{
     layer::SubscriberExt as _,
     util::SubscriberInitExt as _,
 };
-use transit_model::Result;
+use transit_model::{Model, Result};
 
 lazy_static::lazy_static! {
     pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION"));
@@ -90,7 +90,9 @@ fn init_logger() {
 fn run(opt: Opt) -> Result<()> {
     info!("Launching ntfs2netexfr...");
 
-    let model = transit_model::ntfs::read(opt.input)?;
+    let mut collections = transit_model::ntfs::read_collections(opt.input)?;
+    collections.remove_route_points();
+    let model = Model::new(collections)?;
 
     let mut config = transit_model::netex_france::WriteConfiguration::new(opt.participant)
         .current_datetime(opt.current_datetime);

src/gtfs/mod.rs

@@ -21,7 +21,7 @@ use crate::{
     calendars::{manage_calendars, write_calendar_dates},
     gtfs::read::EquipmentList,
     model::{Collections, Model},
-    objects::{self, Availability, Contributor, Dataset, StopPoint, StopType, Time},
+    objects::{self, Availability, Contributor, Dataset, StopType, Time},
     read_utils,
     utils::*,
     validity_period, AddPrefix, PrefixConfiguration, Result,
@@ -32,7 +32,7 @@ use failure::ResultExt;
 use log::info;
 use serde::{Deserialize, Serialize};
 use std::{collections::BTreeMap, fmt, path::Path};
-use typed_index_collection::{CollectionWithId, Idx};
+use typed_index_collection::CollectionWithId;
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct Agency {
@@ -504,34 +504,10 @@ struct Route {
     sort_order: Option<u32>,
 }
 
-fn remove_stop_zones(model: Model) -> Collections {
-    let mut collections = model.into_collections();
-    collections
-        .stop_points
-        .retain(|sp| sp.stop_type != StopType::Zone);
-
-    let stop_point_ids: Vec<Idx<StopPoint>> = collections
-        .stop_points
-        .get_id_to_idx()
-        .values()
-        .copied()
-        .collect();
-
-    collections.vehicle_journeys.retain(|vj| {
-        vj.stop_times
-            .iter()
-            .all(|st| stop_point_ids.contains(&st.stop_point_idx))
-    });
-
-    collections
-}
-
 /// Exports a `Model` to [GTFS](https://gtfs.org/reference/static) files
 /// in the given directory.
 /// see [NTFS to GTFS conversion](https://github.com/CanalTP/transit_model/blob/master/src/documentation/ntfs2gtfs.md)
 pub fn write<P: AsRef<Path>>(model: Model, path: P) -> Result<()> {
-    let collections = remove_stop_zones(model);
-    let model = Model::new(collections)?;
     let path = path.as_ref();
     std::fs::create_dir_all(path)?;
     info!("Writing GTFS to {:?}", path);

src/model.rs

@@ -119,6 +119,33 @@ pub struct Collections {
 }
 
 impl Collections {
+    /// Remove associated schedules with route points
+    pub fn remove_route_points(&mut self) {
+        let vj_idxs: Vec<Idx<VehicleJourney>> =
+            self.vehicle_journeys.iter().map(|(idx, _)| idx).collect();
+        let is_route_point = |stop_time: &StopTime| -> bool {
+            stop_time.pickup_type == 3 || stop_time.drop_off_type == 3
+        };
+        for vj_idx in vj_idxs {
+            let mut vj = self.vehicle_journeys.index_mut(vj_idx);
+            vj.stop_times.retain(|stop_time| !is_route_point(stop_time));
+        }
+    }
+
+    /// Remove stop zone
+    pub fn remove_stop_zones(&mut self) {
+        self.stop_points.retain(|sp| sp.stop_type != StopType::Zone);
+
+        let stop_point_ids: Vec<Idx<StopPoint>> =
+            self.stop_points.get_id_to_idx().values().copied().collect();
+
+        self.vehicle_journeys.retain(|vj| {
+            vj.stop_times
+                .iter()
+                .all(|st| stop_point_ids.contains(&st.stop_point_idx))
+        });
+    }
+
     /// Restrict the validity period of the current `Collections` with the start_date and end_date
     pub fn restrict_period(&mut self, start_date: NaiveDate, end_date: NaiveDate) -> Result<()> {
         let mut calendars = self.calendars.take();
@@ -2840,5 +2867,37 @@ mod tests {
             assert_eq!(3, stop_time.pickup_type);
             assert_eq!(3, stop_time.drop_off_type);
         }
+
+        #[test]
+        fn remove_pickup_drop_off_type_3() {
+            let model = transit_model_builder::ModelBuilder::default()
+                .vj("vj1", |vj| {
+                    vj.st_mut("SP1", "10:00:00", "10:01:00", |st| {
+                        st.pickup_type = 0;
+                        st.drop_off_type = 1;
+                    })
+                    .st_mut("SP2", "11:00:00", "11:01:00", |st| {
+                        st.pickup_type = 3;
+                        st.drop_off_type = 2;
+                    })
+                    .st_mut("SP3", "12:00:00", "12:01:00", |st| {
+                        st.pickup_type = 3;
+                        st.drop_off_type = 2;
+                    })
+                    .st_mut("SP4", "13:00:00", "13:01:00", |st| {
+                        st.pickup_type = 1;
+                        st.drop_off_type = 0;
+                    });
+                })
+                .build();
+
+            let mut collections = model.into_collections();
+            collections.remove_route_points();
+            let vj = collections
+                .vehicle_journeys
+                .get("vj1")
+                .expect("Failed to find vehicle journey vj1");
+            assert_eq!(2, vj.stop_times.len());
+        }
     }
 }

src/ntfs/mod.rs

@@ -181,7 +181,6 @@ pub fn from_dir<P: AsRef<path::Path>>(p: P) -> Result<Model> {
     let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf());
     read_file_handler(&mut file_handle)
 }
-
 /// Imports a `Model` from a zip file containing the
 /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md).
 pub fn from_zip<P: AsRef<path::Path>>(p: P) -> Result<Model> {
@@ -190,6 +189,22 @@ pub fn from_zip<P: AsRef<path::Path>>(p: P) -> Result<Model> {
     read_file_handler(&mut file_handler)
 }
 
+/// Imports `Collections` from a zip file containing the
+/// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md).
+pub fn collections_from_zip<P: AsRef<path::Path>>(p: P) -> Result<Collections> {
+    let reader = std::fs::File::open(p.as_ref())?;
+    let mut file_handler = read_utils::ZipHandler::new(reader, p)?;
+    read_collections_file_handler(&mut file_handler)
+}
+
+/// Imports `Collections` from the
+/// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md)
+/// files in the given directory.
+pub fn collections_from_dir<P: AsRef<path::Path>>(p: P) -> Result<Collections> {
+    let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf());
+    read_collections_file_handler(&mut file_handle)
+}
+
 /// Imports a `Model` from an object implementing `Read` and `Seek` and containing a zip file with a
 /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md).
 ///
@@ -235,7 +250,41 @@ pub fn read<P: AsRef<path::Path>>(path: P) -> Result<Model> {
     }
 }
 
+/// Imports `Collections` from the
+/// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md)
+/// files in the given directory.
+/// This method will try to detect if the input is a zipped archive or not.
+/// If the default file type mechanism is not enough, you can use
+/// [from_zip] or [from_dir].
+pub fn read_collections<P: AsRef<path::Path>>(path: P) -> Result<Collections> {
+    let p = path.as_ref();
+    if p.is_file() {
+        // if it's a file, we consider it to be a zip (and an error will be returned if it is not)
+        Ok(collections_from_zip(p)
+            .with_context(|_| format!("impossible to read zipped ntfs {:?}", p))?)
+    } else if p.is_dir() {
+        Ok(collections_from_dir(p)
+            .with_context(|_| format!("impossible to read ntfs directory from {:?}", p))?)
+    } else {
+        Err(failure::format_err!(
+            "file {:?} is neither a file nor a directory, cannot read a ntfs from it",
+            p
+        ))
+    }
+}
+
 fn read_file_handler<H>(file_handler: &mut H) -> Result<Model>
+where
+    for<'a> &'a mut H: read_utils::FileHandler,
+{
+    let collections = read_collections_file_handler(file_handler)?;
+    info!("Indexing");
+    let res = Model::new(collections)?;
+    info!("Loading NTFS done");
+    Ok(res)
+}
+
+fn read_collections_file_handler<H>(file_handler: &mut H) -> Result<Collections>
 where
     for<'a> &'a mut H: read_utils::FileHandler,
 {
@@ -279,10 +328,7 @@ where
     read::manage_object_properties(&mut collections, file_handler)?;
     read::manage_fares_v1(&mut collections, file_handler)?;
     read::manage_companies_on_vj(&mut collections)?;
-    info!("Indexing");
-    let res = Model::new(collections)?;
-    info!("Loading NTFS done");
-    Ok(res)
+    Ok(collections)
 }
 
 /// Exports a `Model` to the

tests/fixtures/minimal_ntfs/stop_times.txt

@@ -1,24 +1,24 @@
-trip_id,stop_sequence,stop_id,arrival_time,departure_time,datetime_estimated
-M1F1,0,NATM,9:00:00,9:00:00,
-M1F1,1,GDLM,09:10:00,09:10:00,
-M1F1,2,CHAM,09:20:00,09:20:00,
-M1F1,3,CDGM,09:40:00,09:40:00,
-M1B1,9,NATM,11:10:00,11:10:00,
-M1B1,8,GDLM,11:00:00,11:00:00,
-M1B1,7,CHAM,10:50:00,10:50:00,
-M1B1,6,CDGM,10:40:00,10:40:00,
-B42F1,10,GDLB,10:10:00,10:10:00,
-B42F1,20,MTPB,10:20:00,10:20:00,
-B42B1,30,GDLB,07:10:00,07:10:00,
-B42B1,20,MTPB,07:00:00,07:00:00,
-RERAF1,1,NATR,08:09:00,08:10:00,
-RERAF1,02,GDLR,08:14:00,08:15:00,
-RERAF1,3,CDGR,08:19:00,08:20:00,
-RERAF1,05,DEFR,08:24:00,08:25:00,
-RERAB1,21,NATR,09:49:00,09:50:00,
-RERAB1,13,GDLR,09:44:00,09:45:00,
-RERAB1,08,CDGR,09:39:00,09:40:00,0
-RERAB1,05,DEFR,09:24:00,09:25:00,1
-RERAB1,50,MTPZ,19:24:00,19:25:00,
-RERAB1,51,CDGZ,19:26:00,19:27:00,0
-RERAB1,52,MTPZ,19:34:00,19:35:00,1
+trip_id,stop_sequence,stop_id,arrival_time,departure_time,datetime_estimated,pickup_type,drop_off_type
+M1F1,0,NATM,9:00:00,9:00:00,,0,0
+M1F1,1,GDLM,09:10:00,09:10:00,,3,3
+M1F1,2,CHAM,09:20:00,09:20:00,,0,0
+M1F1,3,CDGM,09:40:00,09:40:00,,0,0
+M1B1,9,NATM,11:10:00,11:10:00,,0,0
+M1B1,8,GDLM,11:00:00,11:00:00,,0,0
+M1B1,7,CHAM,10:50:00,10:50:00,,0,0
+M1B1,6,CDGM,10:40:00,10:40:00,,0,0
+B42F1,10,GDLB,10:10:00,10:10:00,,0,0
+B42F1,20,MTPB,10:20:00,10:20:00,,0,0
+B42B1,30,GDLB,07:10:00,07:10:00,,0,0
+B42B1,20,MTPB,07:00:00,07:00:00,,0,0
+RERAF1,1,NATR,08:09:00,08:10:00,,0,0
+RERAF1,02,GDLR,08:14:00,08:15:00,,0,0
+RERAF1,3,CDGR,08:19:00,08:20:00,,0,0
+RERAF1,05,DEFR,08:24:00,08:25:00,,0,0
+RERAB1,21,NATR,09:49:00,09:50:00,,0,0
+RERAB1,13,GDLR,09:44:00,09:45:00,,0,0
+RERAB1,08,CDGR,09:39:00,09:40:00,0,0,0
+RERAB1,05,DEFR,09:24:00,09:25:00,1,0,0
+RERAB1,50,MTPZ,19:24:00,19:25:00,,0,0
+RERAB1,51,CDGZ,19:26:00,19:27:00,0,0,0
+RERAB1,52,MTPZ,19:34:00,19:35:00,1,0,0

tests/fixtures/netex_france/input_gtfs/stop_times.txt

@@ -1,8 +1,8 @@
 trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type,stop_headsign,timepoint
 M1F1,00:00:00,00:00:00,NATM,0,0,0,,1
 M1F1,00:10:00,00:10:00,GDLM,1,0,0,,1
-M1F1,00:20:00,00:20:00,CHAM,2,0,0,,1
-M1F1,00:40:00,00:40:00,CDGM,3,0,0,,1
+M1F1,00:20:00,00:20:00,CHAM,3,0,0,,1
+M1F1,00:40:00,00:40:00,CDGM,4,0,0,,1
 M1B1,00:00:00,00:00:00,CDGM,6,0,0,,1
 M1B1,00:10:00,00:10:00,CHAM,7,0,0,,1
 M1B1,00:20:00,00:20:00,GDLM,8,0,0,,1

tests/fixtures/netex_france/input_ntfs/stop_times.txt

@@ -1,8 +1,9 @@
 trip_id,stop_sequence,stop_id,arrival_time,departure_time,stop_time_id,pickup_type,drop_off_type
 M1F1,0,NATM,00:00:00,00:00:00,,0,0
 M1F1,1,GDLM,00:10:00,00:10:00,,0,0
-M1F1,2,CHAM,00:20:00,00:20:00,,0,0
-M1F1,3,CDGM,00:40:00,00:40:00,,0,0
+M1F1,2,MTPB,00:15:00,00:15:00,,3,3
+M1F1,3,CHAM,00:20:00,00:20:00,,0,0
+M1F1,4,CDGM,00:40:00,00:40:00,,0,0
 M1B1,9,NATM,00:30:00,00:30:00,,0,0
 M1B1,8,GDLM,00:20:00,00:20:00,,0,0
 M1B1,7,CHAM,00:10:00,00:10:00,,0,0