From 81fdfc9a8272aaa13f87aeac7bab53f1d5c59fc3 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 15 Nov 2021 14:34:55 +0000 Subject: [PATCH 001/133] refactor(Added explict GTFS-lib reference and editor controllers for booking rules and location grou --- pom.xml | 2 +- src/main/java/com/conveyal/datatools/manager/DataManager.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f5f4e35a..e360c565f 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 7.0.3 + 81ed510d04c8c53ff32bfe7e9ae18e5651a60816 diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index 6ee5fa465..8a5d7446e 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -204,11 +204,15 @@ static void registerRoutes() throws IOException { String gtfs = IOUtils.toString(DataManager.class.getResourceAsStream("/gtfs/gtfs.yml")); gtfsConfig = yamlMapper.readTree(gtfs); new EditorControllerImpl(EDITOR_API_PREFIX, Table.AGENCY, DataManager.GTFS_DATA_SOURCE); + // NOTE: booking_rules is a GTFS Flex addition. + new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ATTRIBUTIONS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.CALENDAR, DataManager.GTFS_DATA_SOURCE); // NOTE: fare_attributes controller handles updates to nested table fare_rules. new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); + // NOTE: location groups is a GTFS Flex addition. + new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUPS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE); // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies. new EditorControllerImpl(EDITOR_API_PREFIX, Table.PATTERNS, DataManager.GTFS_DATA_SOURCE); From 6a9707edd331bd13442a021b0fd6d770e6bdc82f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 17 Nov 2021 08:12:51 +0000 Subject: [PATCH 002/133] refactor(Update to define if the loaded feed is flex): Update to set the flex value depending on the --- pom.xml | 2 +- .../datatools/manager/models/FeedSource.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e360c565f..427628363 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 81ed510d04c8c53ff32bfe7e9ae18e5651a60816 + 12bb31f737adc4b395fe893a1d9418b3f039feeb diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 612e45049..bc27b10a2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -19,6 +19,7 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.JobUtils; import com.conveyal.gtfs.GTFS; +import com.conveyal.gtfs.loader.FeedLoadResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -166,6 +167,13 @@ public String organizationId () { public String editorNamespace; + /** + * If this feed source references a feed version that is GTFS Flex, this value will be true. This is updated every + * time a feed version is retrieved as different feed versions may or may not be flex. The process to determine a + * flex feed is defined in {@link FeedLoadResult#isGTFSFlex()}. + */ + public boolean flex; + /** * Create a new feed. */ @@ -323,6 +331,7 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri String message = String.format("Fetch complete for %s", this.name); LOG.info(message); status.completeSuccessfully(message); + setFlex(version); return version; } } @@ -370,6 +379,7 @@ public FeedVersion retrieveLatest() { // Is this what happens if there are none? return null; } + setFlex(newestVersion); return newestVersion; } @@ -386,6 +396,7 @@ public FeedVersion retrievePublishedVersion() { // Is this what happens if there are none? return null; } + setFlex(publishedVersion); return publishedVersion; } @@ -695,4 +706,11 @@ public List getActiveTransformations(FeedVersi .flatMap(Collection::stream) .collect(Collectors.toList()); } + + /** + * Set the flex value according to the feed version loaded. + */ + private void setFlex(FeedVersion feedVersion) { + flex = feedVersion.feedLoadResult.isGTFSFlex(); + } } From 96d34c2e965e1157992d98d6452d1563f4a5f1d4 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 17 Nov 2021 13:59:32 +0000 Subject: [PATCH 003/133] refactor(Added Bson ignore to flex param): Added Bson ignore to flex param so it is not picked up by --- .../java/com/conveyal/datatools/manager/models/FeedSource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index bc27b10a2..72159f1c7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -172,6 +172,7 @@ public String organizationId () { * time a feed version is retrieved as different feed versions may or may not be flex. The process to determine a * flex feed is defined in {@link FeedLoadResult#isGTFSFlex()}. */ + @BsonIgnore public boolean flex; /** From 0328a0be9a224094dfa180521640fcc7aeb135fc Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 18 Nov 2021 09:15:40 +0000 Subject: [PATCH 004/133] refactor(Moved the @BsonIngore from param to setter): Updated FeedSource to move @BsonIgnore from th --- pom.xml | 2 +- .../java/com/conveyal/datatools/manager/models/FeedSource.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 427628363..8f8430a60 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 12bb31f737adc4b395fe893a1d9418b3f039feeb + 503ddb2346e5654195c91ed0d0a133a9631a6345 diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 72159f1c7..aefa7ecc7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -172,7 +172,6 @@ public String organizationId () { * time a feed version is retrieved as different feed versions may or may not be flex. The process to determine a * flex feed is defined in {@link FeedLoadResult#isGTFSFlex()}. */ - @BsonIgnore public boolean flex; /** @@ -711,6 +710,7 @@ public List getActiveTransformations(FeedVersi /** * Set the flex value according to the feed version loaded. */ + @BsonIgnore private void setFlex(FeedVersion feedVersion) { flex = feedVersion.feedLoadResult.isGTFSFlex(); } From 857b4a65c1a08e9b4faf02bcfed59331ab560396 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 18 Nov 2021 10:59:07 +0000 Subject: [PATCH 005/133] refactor(FeedSource): Added getter and setter methods for flex param --- .../datatools/manager/models/FeedSource.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index aefa7ecc7..362660c00 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -331,7 +331,7 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri String message = String.format("Fetch complete for %s", this.name); LOG.info(message); status.completeSuccessfully(message); - setFlex(version); + presetFlex(version); return version; } } @@ -379,7 +379,7 @@ public FeedVersion retrieveLatest() { // Is this what happens if there are none? return null; } - setFlex(newestVersion); + presetFlex(newestVersion); return newestVersion; } @@ -396,7 +396,7 @@ public FeedVersion retrievePublishedVersion() { // Is this what happens if there are none? return null; } - setFlex(publishedVersion); + presetFlex(publishedVersion); return publishedVersion; } @@ -710,8 +710,17 @@ public List getActiveTransformations(FeedVersi /** * Set the flex value according to the feed version loaded. */ - @BsonIgnore - private void setFlex(FeedVersion feedVersion) { + public void presetFlex(FeedVersion feedVersion) { flex = feedVersion.feedLoadResult.isGTFSFlex(); } + + @BsonIgnore + public boolean isFlex() { + return flex; + } + + @BsonIgnore + public void setFlex(boolean flex) { + this.flex = flex; + } } From 55677378267b389e71dac81abbac7081f4d4fe52 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 18 Nov 2021 14:01:20 +0000 Subject: [PATCH 006/133] refactor(FeedSource.java): Removed flex getter and setter Removed flex getter and setter --- .../conveyal/datatools/manager/models/FeedSource.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 362660c00..61f5bc085 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -713,14 +713,4 @@ public List getActiveTransformations(FeedVersi public void presetFlex(FeedVersion feedVersion) { flex = feedVersion.feedLoadResult.isGTFSFlex(); } - - @BsonIgnore - public boolean isFlex() { - return flex; - } - - @BsonIgnore - public void setFlex(boolean flex) { - this.flex = flex; - } } From f919940e881bbed81ef8bf631987ce18d71a2245 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 9 Dec 2021 15:54:05 +0000 Subject: [PATCH 007/133] refactor(DataManager.java): Added editor controllers for location meta data and location shapes --- pom.xml | 2 +- src/main/java/com/conveyal/datatools/manager/DataManager.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f8430a60..ed5ef4c29 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 503ddb2346e5654195c91ed0d0a133a9631a6345 + 791e3d57fa3a23630fd54ccf65dcfdfc2dba91af diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index 8a5d7446e..ff80bbc66 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -213,6 +213,8 @@ static void registerRoutes() throws IOException { new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); // NOTE: location groups is a GTFS Flex addition. new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUPS, DataManager.GTFS_DATA_SOURCE); + new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_META_DATA, DataManager.GTFS_DATA_SOURCE); + new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE); // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies. new EditorControllerImpl(EDITOR_API_PREFIX, Table.PATTERNS, DataManager.GTFS_DATA_SOURCE); From e005c623ff689ac0f0b16d69f6b4d5f5379902f2 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 10 Dec 2021 09:08:04 +0000 Subject: [PATCH 008/133] refactor(pom.xml): Bumped gtfs-lib version Bumped gtfs-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ed5ef4c29..0a2e4474d 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 791e3d57fa3a23630fd54ccf65dcfdfc2dba91af + 12fd424c17983842cd830172d20ebfb446c9e550 From 87287019bd80aa2b84cee574682ffae74d36af38 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 14 Dec 2021 16:08:28 +0000 Subject: [PATCH 009/133] refactor(pom.xml): Bumped GTFS-lib version Bumped gtfs-lib version which include pattern stop updates --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a2e4474d..7aea4716e 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 12fd424c17983842cd830172d20ebfb446c9e550 + 5a71495d550d22cd41527b88ec4879ad777c0d43 From 837174d916af6026afbf7f7207a6c3674463cd2c Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 7 Jan 2022 17:04:42 +0100 Subject: [PATCH 010/133] refactor(manager/DataManager): update tables to reflect gtfs-lib changes --- pom.xml | 2 +- src/main/java/com/conveyal/datatools/manager/DataManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7aea4716e..e1461de3c 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 5a71495d550d22cd41527b88ec4879ad777c0d43 + 285ac702083534aebe258b79a06fbbb58810b659 diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index ff80bbc66..65ec23a0e 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -213,7 +213,7 @@ static void registerRoutes() throws IOException { new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); // NOTE: location groups is a GTFS Flex addition. new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUPS, DataManager.GTFS_DATA_SOURCE); - new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_META_DATA, DataManager.GTFS_DATA_SOURCE); + new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE); // NOTE: Patterns controller handles updates to nested tables shapes, pattern stops, and frequencies. From 7b8da966262f1c5c0e9e46dee359582eb9e07bb0 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 21 Jan 2022 12:57:39 -0500 Subject: [PATCH 011/133] chore(deps): upgrade to latest gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e1461de3c..b6ec0329d 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 285ac702083534aebe258b79a06fbbb58810b659 + ecb239daf43dac248698ecf9be8b017088c8771e From f455b7150387a1828302009d7f3fc6145fbc017c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 24 Jan 2022 15:16:19 +0000 Subject: [PATCH 012/133] refactor(pom.xml): Bumped GTFS-lib version and updated snapshot matcher dependency source --- pom.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index b6ec0329d..0def8f60f 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - ecb239daf43dac248698ecf9be8b017088c8771e + f1816642acb015846790ccf4ac217f39f39b86f5 @@ -391,10 +391,12 @@ test + - com.github.conveyal - java-snapshot-matcher - 3495b32f7b4d3f82590e0a2284029214070b6984 + com.github.ibi-group + ibi-java-snapshot-matcher + 0820e1a3a2a6c7f845038859d0c4c09bf18c7910 test From 6bbe2f8b1d080c1f2e5e23fd4e8a7624d508e14a Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 25 Jan 2022 07:44:31 -0500 Subject: [PATCH 013/133] chore(deps): update gtfs-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0def8f60f..534836dc7 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - f1816642acb015846790ccf4ac217f39f39b86f5 + 47ca9a2be86419bd3853c900e6b1d6f606e5b91f From d997b12d179637462960b9b788bd8f32a2f2867c Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 25 Jan 2022 14:10:14 -0500 Subject: [PATCH 014/133] chore(deps): update gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 534836dc7..ad78dff4d 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 47ca9a2be86419bd3853c900e6b1d6f606e5b91f + 016b38cdcc31e945df94e4a456922736701b2f34 From a9deba38ad80b32c51920b9a64b0dcfc8d944a51 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 25 Jan 2022 15:06:37 -0500 Subject: [PATCH 015/133] chore(deps): update gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad78dff4d..1709d455d 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 016b38cdcc31e945df94e4a456922736701b2f34 + 94d74d4fc871b544381527393bd4fc4b64ae30e5 From 62bd416a18e14111a19686c9d52e2312dbeaa93c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 8 Feb 2022 07:51:42 +0000 Subject: [PATCH 016/133] refactor(DataManager.java): Re-ordered editor controllers to group flex updates --- .../java/com/conveyal/datatools/manager/DataManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index 65ec23a0e..0aa53bd05 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -204,14 +204,13 @@ static void registerRoutes() throws IOException { String gtfs = IOUtils.toString(DataManager.class.getResourceAsStream("/gtfs/gtfs.yml")); gtfsConfig = yamlMapper.readTree(gtfs); new EditorControllerImpl(EDITOR_API_PREFIX, Table.AGENCY, DataManager.GTFS_DATA_SOURCE); - // NOTE: booking_rules is a GTFS Flex addition. - new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ATTRIBUTIONS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.CALENDAR, DataManager.GTFS_DATA_SOURCE); // NOTE: fare_attributes controller handles updates to nested table fare_rules. new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); - // NOTE: location groups is a GTFS Flex addition. + // NOTE: Booking rules, location groups, locations and location shapes are GTFS Flex additions. + new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUPS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE); From 6520f40d1e8a4410634413139d014800f7236500 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 10 Feb 2022 08:25:57 +0000 Subject: [PATCH 017/133] refactor(FeedSource.java): Added flex UI features param --- .../com/conveyal/datatools/manager/models/FeedSource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index f03e81cd1..259177e77 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -183,6 +183,8 @@ public String organizationId () { */ public boolean flex; + public boolean flexUIFeaturesEnabled; + /** * Create a new feed. */ @@ -764,9 +766,10 @@ public List getActiveTransformations(FeedVersi } /** - * Set the flex value according to the feed version loaded. + * Set the flex value according to the feed version loaded. If a flex feed is loaded, enable the UI flex features. */ public void presetFlex(FeedVersion feedVersion) { flex = feedVersion.feedLoadResult.isGTFSFlex(); + if (flex) flexUIFeaturesEnabled = true; } } From 235d9a2d98cfef5f1716d739d6c323fc46795043 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 10 Feb 2022 14:56:51 +0000 Subject: [PATCH 018/133] refactor(Included the latest commit of gtfs-lib and made a few changes in relation to this): Updated --- pom.xml | 2 +- .../jobs/feedmerge/MergeLineContext.java | 58 ++++++++++--------- .../datatools/manager/models/FeedSource.java | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 1709d455d..69894fbcf 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 94d74d4fc871b544381527393bd4fc4b64ae30e5 + c4447eb0be92e5f2f1007664a46abaa2bee3bb00 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 5fb7edd0f..d1a405e4a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -236,36 +236,38 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(field.referenceTable, fieldContext.getValue()); - // Check if we're performing a service period merge, this ref field is a service_id, and it - // is not found in the list of service_ids (e.g., it was removed). - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); - - // If the current foreign ref points to another record that has - // been skipped or is a ref to a non-existent service_id during a service period merge, skip - // this record and add its primary key to the list of skipped IDs (so that other references - // can be properly omitted). - if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { - // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were - // valid service_ids found in calendar_dates, do not skip that record for both the - // calendar_date and any related trips. - if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); - } else { - String skippedKey = getTableScopedValue(keyValue); - if (orderField != null) { - skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + for (Table referenceTable: field.referenceTable) { + String key = getTableScopedValue(referenceTable, fieldContext.getValue()); + // Check if we're performing a service period merge, this ref field is a service_id, and it + // is not found in the list of service_ids (e.g., it was removed). + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); + + // If the current foreign ref points to another record that has + // been skipped or is a ref to a non-existent service_id during a service period merge, skip + // this record and add its primary key to the list of skipped IDs (so that other references + // can be properly omitted). + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were + // valid service_ids found in calendar_dates, do not skip that record for both the + // calendar_date and any related trips. + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); + } else { + String skippedKey = getTableScopedValue(keyValue); + if (orderField != null) { + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); + } + mergeFeedsResult.skippedIds.add(skippedKey); + return false; } - mergeFeedsResult.skippedIds.add(skippedKey); - return false; } - } - // If the field is a foreign reference, check to see whether the reference has been - // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). - if (mergeFeedsResult.remappedIds.containsKey(key)) { - mergeFeedsResult.remappedReferences++; - // If the value has been remapped update the value to write. - fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + // If the field is a foreign reference, check to see whether the reference has been + // remapped due to a conflicting ID from another feed (e.g., calendar#service_id). + if (mergeFeedsResult.remappedIds.containsKey(key)) { + mergeFeedsResult.remappedReferences++; + // If the value has been remapped update the value to write. + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); + } } } return true; diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 259177e77..9a596f6e4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -769,7 +769,7 @@ public List getActiveTransformations(FeedVersi * Set the flex value according to the feed version loaded. If a flex feed is loaded, enable the UI flex features. */ public void presetFlex(FeedVersion feedVersion) { - flex = feedVersion.feedLoadResult.isGTFSFlex(); + if (feedVersion.feedLoadResult != null) flex = feedVersion.feedLoadResult.isGTFSFlex(); if (flex) flexUIFeaturesEnabled = true; } } From 3ec018f5bf795ad84b023d9d228ce63956a13052 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 10 Feb 2022 11:58:04 -0800 Subject: [PATCH 019/133] refactor(FeedVersion): Update feedSource flex setting on creation --- .../com/conveyal/datatools/manager/models/FeedVersion.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 456a7d3b5..f091061b7 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -274,6 +274,12 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { assignGtfsFileAttributes(gtfsFile); String gtfsFilePath = gtfsFile.getPath(); this.feedLoadResult = GTFS.load(gtfsFilePath, DataManager.GTFS_DATA_SOURCE); + + + FeedSource flexSetFeedSource = this.parentFeedSource(); + flexSetFeedSource.flex = this.feedLoadResult.isGTFSFlex(); + Persistence.feedSources.replace(this.parentFeedSource().id, flexSetFeedSource); + if (this.feedLoadResult.fatalException != null) { status.fail("Could not load feed due to " + feedLoadResult.fatalException); return; From 1b28d7490d3b21bf8658b0f003ba5422bb948bb7 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Feb 2022 10:30:36 +0000 Subject: [PATCH 020/133] refactor(Updated snapshots to include null references): Update to work with the latest version of sn --- .../api/UserControllerTest/canCreateUser.json | 24 +++++++ .../api/UserControllerTest/canDeleteUser.json | 1 + .../canListFirstTenUsers.json | 66 +++++++++++++++++++ .../canReturnMeaningfulAuth0Error.json | 6 ++ ...nUpdatePermissionsOfExistingAuth0User.json | 26 ++++++++ ...sOfExistingAuth0UserWithExistingPerms.json | 33 ++++++++++ .../api/UserControllerTest/canUpdateUser.json | 26 ++++++++ .../updateUserFailsWhenApiTokenInvalid.json | 6 ++ ...sOfExistingAuth0UserWithExistingPerms.json | 6 ++ ...aphBuildAndServeManifestAndUserData-0.json | 7 ++ ...GraphBuildAndServeManifestAndUserData.json | 32 +++++++++ ...aphBuildAndServeManifestAndUserData-0.json | 7 ++ ...GraphBuildAndServeManifestAndUserData.json | 32 +++++++++ ...anMakeServerOnlyManifestAndUserData-0.json | 6 ++ .../canMakeServerOnlyManifestAndUserData.json | 29 ++++++++ 15 files changed, 307 insertions(+) create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json create mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json new file mode 100644 index 000000000..b3f89cdf6 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json @@ -0,0 +1,24 @@ +{ + "app_metadata" : { + "datatools" : [ { + "client_id" : "testing-client-id", + "organizations" : [ ], + "permissions" : [ ], + "projects" : [ ] + } ] + }, + "created_at" : "2019-02-16T01:27:19.810Z", + "email" : "test-new-user@test.com", + "email_verified" : false, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test-new-user" + } ], + "name" : "test-new-user@test.com", + "nickname" : "test-new-user", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2019-02-16T01:27:19.810Z", + "user_id" : "auth0|test-new-user" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json new file mode 100644 index 000000000..f32a5804e --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json new file mode 100644 index 000000000..3bf6cc9ef --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json @@ -0,0 +1,66 @@ +[ { + "app_metadata" : { + "datatools" : [ { + "client_id" : "your-auth0-client-id", + "organizations" : [ ], + "permissions" : [ ], + "projects" : [ { + "defaultFeeds" : [ ], + "permissions" : [ { + "feeds" : [ "*" ], + "type" : "view-feed" + } ], + "project_id" : "test-project-id" + } ], + "subscriptions" : [ { + "target" : [ "test-feed-id" ], + "type" : "feed-updated" + } ] + } ] + }, + "created_at" : "2017-06-22T19:19:41.404Z", + "email" : "test1@test.com", + "email_verified" : true, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test1" + } ], + "last_ip" : "1.1.1.1", + "last_login" : "2017-07-12T19:09:28.400Z", + "last_password_reset" : "2017-07-12T19:08:52.507Z", + "logins_count" : 1, + "name" : "test1", + "nickname" : "test1", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2017-07-12T19:16:11.699Z", + "user_id" : "auth0|test1" +}, { + "app_metadata" : { + "datatools" : [ { + "client_id" : "your-auth0-client-id", + "permissions" : [ { + "type" : "administer-application" + } ], + "projects" : [ ] + } ] + }, + "created_at" : "2018-12-27T16:46:32.864Z", + "email" : "test2@test.com", + "email_verified" : true, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test2" + } ], + "last_ip" : "1.1.1.1", + "last_login" : "2019-02-10T18:24:03.719Z", + "logins_count" : 49, + "name" : "test2", + "nickname" : "test2", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2019-02-10T18:24:03.719Z", + "user_id" : "auth0|test2" +} ] \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json new file mode 100644 index 000000000..7914af18c --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json @@ -0,0 +1,6 @@ +{ + "code" : 409, + "detail" : null, + "message" : "An Auth0 error occurred: The user already exists.", + "result" : "ERR" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json new file mode 100644 index 000000000..4265f8044 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json @@ -0,0 +1,26 @@ +{ + "app_metadata" : { + "datatools" : [ { + "client_id" : "testing-client-id", + "organizations" : [ ], + "permissions" : [ { + "type" : "administer-application" + } ], + "projects" : [ ] + } ] + }, + "created_at" : "2019-02-16T01:27:19.810Z", + "email" : "test-existing-user@test.com", + "email_verified" : false, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test-existing-user" + } ], + "name" : "test-existing-user@test.com", + "nickname" : "test-existing-user", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2019-02-16T01:27:19.810Z", + "user_id" : "auth0|test-existing-user" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json new file mode 100644 index 000000000..e5da63412 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json @@ -0,0 +1,33 @@ +{ + "app_metadata" : { + "datatools" : [ { + "client_id" : "testing-client-id-different-instance", + "organizations" : [ ], + "permissions" : [ { + "type" : "administer-application" + } ], + "projects" : [ ] + }, { + "client_id" : "testing-client-id", + "organizations" : [ ], + "permissions" : [ { + "type" : "administer-application" + } ], + "projects" : [ ] + } ] + }, + "created_at" : "2019-02-16T01:27:19.810Z", + "email" : "test-existing-user@test.com", + "email_verified" : false, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test-existing-user" + } ], + "name" : "test-existing-user@test.com", + "nickname" : "test-existing-user", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2019-02-16T01:27:19.810Z", + "user_id" : "auth0|test-existing-user" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json new file mode 100644 index 000000000..4265f8044 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json @@ -0,0 +1,26 @@ +{ + "app_metadata" : { + "datatools" : [ { + "client_id" : "testing-client-id", + "organizations" : [ ], + "permissions" : [ { + "type" : "administer-application" + } ], + "projects" : [ ] + } ] + }, + "created_at" : "2019-02-16T01:27:19.810Z", + "email" : "test-existing-user@test.com", + "email_verified" : false, + "identities" : [ { + "connection" : "Username-Password-Authentication", + "isSocial" : false, + "provider" : "auth0", + "user_id" : "test-existing-user" + } ], + "name" : "test-existing-user@test.com", + "nickname" : "test-existing-user", + "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", + "updated_at" : "2019-02-16T01:27:19.810Z", + "user_id" : "auth0|test-existing-user" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json new file mode 100644 index 000000000..757c9c622 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json @@ -0,0 +1,6 @@ +{ + "code" : 404, + "detail" : null, + "message" : "Could not update user: User with id auth0|test-existing-user not found (or there are issues with the Auth0 configuration)", + "result" : "ERR" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json new file mode 100644 index 000000000..1ca2ecbea --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json @@ -0,0 +1,6 @@ +{ + "code" : 400, + "detail" : null, + "message" : "User test-existing-user@test.com already exists. Try updating permissions for existing user.", + "result" : "ERR" +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-0.json index e29fc5ce0..649c31d43 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData-0.json @@ -1,11 +1,15 @@ { "baseFolder" : "/var/otp/graphs", "baseFolderDownloads" : [ { + "name" : null, "uri" : "s3://datatools-dev/test-deploy/build-config.json" }, { + "name" : null, "uri" : "s3://datatools-dev/test-deploy/router-config.json" } ], + "buildConfigJSON" : null, "buildGraph" : true, + "buildLogFile" : null, "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", "jarFile" : "/opt/otp/otp-latest-trimet-dev", "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", @@ -13,8 +17,11 @@ "otpRunnerLogFile" : "/var/log/otp-runner.log", "otpVersion" : "1.x", "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, "runServer" : true, "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, "serverStartupTimeoutSeconds" : 3300, "statusFileLocation" : "/usr/share/nginx/client/status.json", "uploadGraph" : true, diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json new file mode 100644 index 000000000..649c31d43 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json @@ -0,0 +1,32 @@ +{ + "baseFolder" : "/var/otp/graphs", + "baseFolderDownloads" : [ { + "name" : null, + "uri" : "s3://datatools-dev/test-deploy/build-config.json" + }, { + "name" : null, + "uri" : "s3://datatools-dev/test-deploy/router-config.json" + } ], + "buildConfigJSON" : null, + "buildGraph" : true, + "buildLogFile" : null, + "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", + "jarFile" : "/opt/otp/otp-latest-trimet-dev", + "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", + "nonce" : "canMakeGraphBuildUserDataScript", + "otpRunnerLogFile" : "/var/log/otp-runner.log", + "otpVersion" : "1.x", + "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, + "runServer" : true, + "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, + "serverStartupTimeoutSeconds" : 3300, + "statusFileLocation" : "/usr/share/nginx/client/status.json", + "uploadGraph" : true, + "uploadGraphBuildLogs" : true, + "uploadGraphBuildReport" : true, + "uploadOtpRunnerLogs" : true, + "uploadServerStartupLogs" : true +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-0.json index 83c98e7d3..bf9c35887 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData-0.json @@ -1,11 +1,15 @@ { "baseFolder" : "/var/otp/graphs", "baseFolderDownloads" : [ { + "name" : null, "uri" : "s3://datatools-dev/test-deploy/build-config.json" }, { + "name" : null, "uri" : "s3://datatools-dev/test-deploy/router-config.json" } ], + "buildConfigJSON" : null, "buildGraph" : true, + "buildLogFile" : null, "graphObjUri" : "s3://datatools-dev/test-deploy/graph.obj", "jarFile" : "/opt/otp/otp-latest-trimet-dev", "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", @@ -13,8 +17,11 @@ "otpRunnerLogFile" : "/var/log/otp-runner.log", "otpVersion" : "2.x", "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, "runServer" : true, "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, "serverStartupTimeoutSeconds" : 3300, "statusFileLocation" : "/usr/share/nginx/client/status.json", "uploadGraph" : true, diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json new file mode 100644 index 000000000..bf9c35887 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json @@ -0,0 +1,32 @@ +{ + "baseFolder" : "/var/otp/graphs", + "baseFolderDownloads" : [ { + "name" : null, + "uri" : "s3://datatools-dev/test-deploy/build-config.json" + }, { + "name" : null, + "uri" : "s3://datatools-dev/test-deploy/router-config.json" + } ], + "buildConfigJSON" : null, + "buildGraph" : true, + "buildLogFile" : null, + "graphObjUri" : "s3://datatools-dev/test-deploy/graph.obj", + "jarFile" : "/opt/otp/otp-latest-trimet-dev", + "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", + "nonce" : "canMakeOtp2GraphBuildUserDataScript", + "otpRunnerLogFile" : "/var/log/otp-runner.log", + "otpVersion" : "2.x", + "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, + "runServer" : true, + "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, + "serverStartupTimeoutSeconds" : 3300, + "statusFileLocation" : "/usr/share/nginx/client/status.json", + "uploadGraph" : true, + "uploadGraphBuildLogs" : true, + "uploadGraphBuildReport" : true, + "uploadOtpRunnerLogs" : true, + "uploadServerStartupLogs" : true +} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-0.json index 574f2b5bd..be908c089 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData-0.json @@ -1,9 +1,12 @@ { "baseFolder" : "/var/otp/graphs", "baseFolderDownloads" : [ { + "name" : null, "uri" : "s3://datatools-dev/test-deploy/router-config.json" } ], + "buildConfigJSON" : null, "buildGraph" : false, + "buildLogFile" : null, "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", "jarFile" : "/opt/otp/otp-latest-trimet-dev", "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", @@ -11,8 +14,11 @@ "otpRunnerLogFile" : "/var/log/otp-runner.log", "otpVersion" : "1.x", "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, "runServer" : true, "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, "serverStartupTimeoutSeconds" : 3300, "statusFileLocation" : "/usr/share/nginx/client/status.json", "uploadGraph" : false, diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json new file mode 100644 index 000000000..be908c089 --- /dev/null +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json @@ -0,0 +1,29 @@ +{ + "baseFolder" : "/var/otp/graphs", + "baseFolderDownloads" : [ { + "name" : null, + "uri" : "s3://datatools-dev/test-deploy/router-config.json" + } ], + "buildConfigJSON" : null, + "buildGraph" : false, + "buildLogFile" : null, + "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", + "jarFile" : "/opt/otp/otp-latest-trimet-dev", + "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", + "nonce" : "canMakeServerOnlyUserDataScript", + "otpRunnerLogFile" : "/var/log/otp-runner.log", + "otpVersion" : "1.x", + "prefixLogUploadsWithInstanceId" : true, + "routerConfigJSON" : null, + "routerName" : null, + "runServer" : true, + "s3UploadPath" : "s3://datatools-dev/test-deploy", + "serverLogFile" : null, + "serverStartupTimeoutSeconds" : 3300, + "statusFileLocation" : "/usr/share/nginx/client/status.json", + "uploadGraph" : false, + "uploadGraphBuildLogs" : false, + "uploadGraphBuildReport" : false, + "uploadOtpRunnerLogs" : true, + "uploadServerStartupLogs" : true +} \ No newline at end of file From 330e914a1337e7dfa514f7f70136bf1a9d0bbe82 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Feb 2022 10:35:16 +0000 Subject: [PATCH 021/133] refactor(Removed snapshots that should not have been commited): Removed redundant snapshots --- .../api/UserControllerTest/canCreateUser.json | 24 ------- .../api/UserControllerTest/canDeleteUser.json | 1 - .../canListFirstTenUsers.json | 66 ------------------- .../canReturnMeaningfulAuth0Error.json | 6 -- ...nUpdatePermissionsOfExistingAuth0User.json | 26 -------- ...sOfExistingAuth0UserWithExistingPerms.json | 33 ---------- .../api/UserControllerTest/canUpdateUser.json | 26 -------- .../updateUserFailsWhenApiTokenInvalid.json | 6 -- ...sOfExistingAuth0UserWithExistingPerms.json | 6 -- ...GraphBuildAndServeManifestAndUserData.json | 32 --------- ...GraphBuildAndServeManifestAndUserData.json | 32 --------- .../canMakeServerOnlyManifestAndUserData.json | 29 -------- 12 files changed, 287 deletions(-) delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json delete mode 100644 src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json deleted file mode 100644 index b3f89cdf6..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canCreateUser.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "app_metadata" : { - "datatools" : [ { - "client_id" : "testing-client-id", - "organizations" : [ ], - "permissions" : [ ], - "projects" : [ ] - } ] - }, - "created_at" : "2019-02-16T01:27:19.810Z", - "email" : "test-new-user@test.com", - "email_verified" : false, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test-new-user" - } ], - "name" : "test-new-user@test.com", - "nickname" : "test-new-user", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2019-02-16T01:27:19.810Z", - "user_id" : "auth0|test-new-user" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json deleted file mode 100644 index f32a5804e..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canDeleteUser.json +++ /dev/null @@ -1 +0,0 @@ -true \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json deleted file mode 100644 index 3bf6cc9ef..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canListFirstTenUsers.json +++ /dev/null @@ -1,66 +0,0 @@ -[ { - "app_metadata" : { - "datatools" : [ { - "client_id" : "your-auth0-client-id", - "organizations" : [ ], - "permissions" : [ ], - "projects" : [ { - "defaultFeeds" : [ ], - "permissions" : [ { - "feeds" : [ "*" ], - "type" : "view-feed" - } ], - "project_id" : "test-project-id" - } ], - "subscriptions" : [ { - "target" : [ "test-feed-id" ], - "type" : "feed-updated" - } ] - } ] - }, - "created_at" : "2017-06-22T19:19:41.404Z", - "email" : "test1@test.com", - "email_verified" : true, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test1" - } ], - "last_ip" : "1.1.1.1", - "last_login" : "2017-07-12T19:09:28.400Z", - "last_password_reset" : "2017-07-12T19:08:52.507Z", - "logins_count" : 1, - "name" : "test1", - "nickname" : "test1", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2017-07-12T19:16:11.699Z", - "user_id" : "auth0|test1" -}, { - "app_metadata" : { - "datatools" : [ { - "client_id" : "your-auth0-client-id", - "permissions" : [ { - "type" : "administer-application" - } ], - "projects" : [ ] - } ] - }, - "created_at" : "2018-12-27T16:46:32.864Z", - "email" : "test2@test.com", - "email_verified" : true, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test2" - } ], - "last_ip" : "1.1.1.1", - "last_login" : "2019-02-10T18:24:03.719Z", - "logins_count" : 49, - "name" : "test2", - "nickname" : "test2", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2019-02-10T18:24:03.719Z", - "user_id" : "auth0|test2" -} ] \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json deleted file mode 100644 index 7914af18c..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "code" : 409, - "detail" : null, - "message" : "An Auth0 error occurred: The user already exists.", - "result" : "ERR" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json deleted file mode 100644 index 4265f8044..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermissionsOfExistingAuth0User.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "app_metadata" : { - "datatools" : [ { - "client_id" : "testing-client-id", - "organizations" : [ ], - "permissions" : [ { - "type" : "administer-application" - } ], - "projects" : [ ] - } ] - }, - "created_at" : "2019-02-16T01:27:19.810Z", - "email" : "test-existing-user@test.com", - "email_verified" : false, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test-existing-user" - } ], - "name" : "test-existing-user@test.com", - "nickname" : "test-existing-user", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2019-02-16T01:27:19.810Z", - "user_id" : "auth0|test-existing-user" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json deleted file mode 100644 index e5da63412..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdatePermsOfExistingAuth0UserWithExistingPerms.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "app_metadata" : { - "datatools" : [ { - "client_id" : "testing-client-id-different-instance", - "organizations" : [ ], - "permissions" : [ { - "type" : "administer-application" - } ], - "projects" : [ ] - }, { - "client_id" : "testing-client-id", - "organizations" : [ ], - "permissions" : [ { - "type" : "administer-application" - } ], - "projects" : [ ] - } ] - }, - "created_at" : "2019-02-16T01:27:19.810Z", - "email" : "test-existing-user@test.com", - "email_verified" : false, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test-existing-user" - } ], - "name" : "test-existing-user@test.com", - "nickname" : "test-existing-user", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2019-02-16T01:27:19.810Z", - "user_id" : "auth0|test-existing-user" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json deleted file mode 100644 index 4265f8044..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canUpdateUser.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "app_metadata" : { - "datatools" : [ { - "client_id" : "testing-client-id", - "organizations" : [ ], - "permissions" : [ { - "type" : "administer-application" - } ], - "projects" : [ ] - } ] - }, - "created_at" : "2019-02-16T01:27:19.810Z", - "email" : "test-existing-user@test.com", - "email_verified" : false, - "identities" : [ { - "connection" : "Username-Password-Authentication", - "isSocial" : false, - "provider" : "auth0", - "user_id" : "test-existing-user" - } ], - "name" : "test-existing-user@test.com", - "nickname" : "test-existing-user", - "picture" : "https://i0.wp.com/cdn.auth0.com/avatars/tu.png?ssl=1", - "updated_at" : "2019-02-16T01:27:19.810Z", - "user_id" : "auth0|test-existing-user" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json deleted file mode 100644 index 757c9c622..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "code" : 404, - "detail" : null, - "message" : "Could not update user: User with id auth0|test-existing-user not found (or there are issues with the Auth0 configuration)", - "result" : "ERR" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json deleted file mode 100644 index 1ca2ecbea..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "code" : 400, - "detail" : null, - "message" : "User test-existing-user@test.com already exists. Try updating permissions for existing user.", - "result" : "ERR" -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json deleted file mode 100644 index 649c31d43..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeGraphBuildAndServeManifestAndUserData.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "baseFolder" : "/var/otp/graphs", - "baseFolderDownloads" : [ { - "name" : null, - "uri" : "s3://datatools-dev/test-deploy/build-config.json" - }, { - "name" : null, - "uri" : "s3://datatools-dev/test-deploy/router-config.json" - } ], - "buildConfigJSON" : null, - "buildGraph" : true, - "buildLogFile" : null, - "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", - "jarFile" : "/opt/otp/otp-latest-trimet-dev", - "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", - "nonce" : "canMakeGraphBuildUserDataScript", - "otpRunnerLogFile" : "/var/log/otp-runner.log", - "otpVersion" : "1.x", - "prefixLogUploadsWithInstanceId" : true, - "routerConfigJSON" : null, - "routerName" : null, - "runServer" : true, - "s3UploadPath" : "s3://datatools-dev/test-deploy", - "serverLogFile" : null, - "serverStartupTimeoutSeconds" : 3300, - "statusFileLocation" : "/usr/share/nginx/client/status.json", - "uploadGraph" : true, - "uploadGraphBuildLogs" : true, - "uploadGraphBuildReport" : true, - "uploadOtpRunnerLogs" : true, - "uploadServerStartupLogs" : true -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json deleted file mode 100644 index bf9c35887..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeOtp2GraphBuildAndServeManifestAndUserData.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "baseFolder" : "/var/otp/graphs", - "baseFolderDownloads" : [ { - "name" : null, - "uri" : "s3://datatools-dev/test-deploy/build-config.json" - }, { - "name" : null, - "uri" : "s3://datatools-dev/test-deploy/router-config.json" - } ], - "buildConfigJSON" : null, - "buildGraph" : true, - "buildLogFile" : null, - "graphObjUri" : "s3://datatools-dev/test-deploy/graph.obj", - "jarFile" : "/opt/otp/otp-latest-trimet-dev", - "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", - "nonce" : "canMakeOtp2GraphBuildUserDataScript", - "otpRunnerLogFile" : "/var/log/otp-runner.log", - "otpVersion" : "2.x", - "prefixLogUploadsWithInstanceId" : true, - "routerConfigJSON" : null, - "routerName" : null, - "runServer" : true, - "s3UploadPath" : "s3://datatools-dev/test-deploy", - "serverLogFile" : null, - "serverStartupTimeoutSeconds" : 3300, - "statusFileLocation" : "/usr/share/nginx/client/status.json", - "uploadGraph" : true, - "uploadGraphBuildLogs" : true, - "uploadGraphBuildReport" : true, - "uploadOtpRunnerLogs" : true, - "uploadServerStartupLogs" : true -} \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json deleted file mode 100644 index be908c089..000000000 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/jobs/DeployJobTest/canMakeServerOnlyManifestAndUserData.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "baseFolder" : "/var/otp/graphs", - "baseFolderDownloads" : [ { - "name" : null, - "uri" : "s3://datatools-dev/test-deploy/router-config.json" - } ], - "buildConfigJSON" : null, - "buildGraph" : false, - "buildLogFile" : null, - "graphObjUri" : "s3://datatools-dev/test-deploy/Graph.obj", - "jarFile" : "/opt/otp/otp-latest-trimet-dev", - "jarUri" : "https://opentripplanner-builds.s3.amazonaws.com/otp-latest-trimet-dev.jar", - "nonce" : "canMakeServerOnlyUserDataScript", - "otpRunnerLogFile" : "/var/log/otp-runner.log", - "otpVersion" : "1.x", - "prefixLogUploadsWithInstanceId" : true, - "routerConfigJSON" : null, - "routerName" : null, - "runServer" : true, - "s3UploadPath" : "s3://datatools-dev/test-deploy", - "serverLogFile" : null, - "serverStartupTimeoutSeconds" : 3300, - "statusFileLocation" : "/usr/share/nginx/client/status.json", - "uploadGraph" : false, - "uploadGraphBuildLogs" : false, - "uploadGraphBuildReport" : false, - "uploadOtpRunnerLogs" : true, - "uploadServerStartupLogs" : true -} \ No newline at end of file From 17c4b7170de73b32469980fd93f89743f6f9e7c9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Feb 2022 10:45:03 +0000 Subject: [PATCH 022/133] refactor(Update to include correct snapshot matcher and update to failing unit tests): Correct issue --- pom.xml | 4 +--- .../UserControllerTest/canReturnMeaningfulAuth0Error-0.json | 1 + .../updateUserFailsWhenApiTokenInvalid-0.json | 1 + ...lNotUpdatePermsOfExistingAuth0UserWithExistingPerms-0.json | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 69894fbcf..3f2791431 100644 --- a/pom.xml +++ b/pom.xml @@ -391,12 +391,10 @@ test - com.github.ibi-group ibi-java-snapshot-matcher - 0820e1a3a2a6c7f845038859d0c4c09bf18c7910 + v1.0.0 test diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error-0.json index 8554b2a1c..7914af18c 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/canReturnMeaningfulAuth0Error-0.json @@ -1,5 +1,6 @@ { "code" : 409, + "detail" : null, "message" : "An Auth0 error occurred: The user already exists.", "result" : "ERR" } \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid-0.json index 7911c8fab..757c9c622 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/updateUserFailsWhenApiTokenInvalid-0.json @@ -1,5 +1,6 @@ { "code" : 404, + "detail" : null, "message" : "Could not update user: User with id auth0|test-existing-user not found (or there are issues with the Auth0 configuration)", "result" : "ERR" } \ No newline at end of file diff --git a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms-0.json b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms-0.json index 899866fe6..1ca2ecbea 100644 --- a/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms-0.json +++ b/src/test/resources/snapshots/com/conveyal/datatools/manager/controllers/api/UserControllerTest/willNotUpdatePermsOfExistingAuth0UserWithExistingPerms-0.json @@ -1,5 +1,6 @@ { "code" : 400, + "detail" : null, "message" : "User test-existing-user@test.com already exists. Try updating permissions for existing user.", "result" : "ERR" } \ No newline at end of file From 1b542ecdda62efb5ff5e7ed8c1adb26dd25d5de3 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 18 Feb 2022 14:31:37 +0000 Subject: [PATCH 023/133] refactor(Bumped GTFS-lib version and address table ref name change): Updated tableRerence to tableRe --- pom.xml | 2 +- .../datatools/manager/jobs/feedmerge/MergeLineContext.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3f2791431..155c39f6a 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - c4447eb0be92e5f2f1007664a46abaa2bee3bb00 + f1f8659bfd5e97e36334ad726df0a15084d45219 diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index d1a405e4a..3e69cb4ab 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -236,7 +236,7 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - for (Table referenceTable: field.referenceTable) { + for (Table referenceTable: field.referenceTables) { String key = getTableScopedValue(referenceTable, fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). From 85773a8d7af5e9565b89d892d58b90e07e02ea0e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 21 Feb 2022 17:48:43 +0000 Subject: [PATCH 024/133] refactor(Updated lib dependencies): Update snapshot and gtfs-lib versions and removed redundant null --- pom.xml | 6 +++--- .../com/conveyal/datatools/manager/models/FeedSource.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 155c39f6a..0a77058a4 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - f1f8659bfd5e97e36334ad726df0a15084d45219 + 4adfd10cd7c166e041f9d33f12f2daf4e8a4e90d @@ -393,8 +393,8 @@ com.github.ibi-group - ibi-java-snapshot-matcher - v1.0.0 + java-snapshot-matcher + master test diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 9a596f6e4..259177e77 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -769,7 +769,7 @@ public List getActiveTransformations(FeedVersi * Set the flex value according to the feed version loaded. If a flex feed is loaded, enable the UI flex features. */ public void presetFlex(FeedVersion feedVersion) { - if (feedVersion.feedLoadResult != null) flex = feedVersion.feedLoadResult.isGTFSFlex(); + flex = feedVersion.feedLoadResult.isGTFSFlex(); if (flex) flexUIFeaturesEnabled = true; } } From e5f2afcbdf80b1fbde01b0c77adfbfc5cd75facf Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 21 Feb 2022 18:05:07 +0000 Subject: [PATCH 025/133] refactor(FeedSource.java): Reinstated null check on feedLoadResult --- .../java/com/conveyal/datatools/manager/models/FeedSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 259177e77..9a596f6e4 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -769,7 +769,7 @@ public List getActiveTransformations(FeedVersi * Set the flex value according to the feed version loaded. If a flex feed is loaded, enable the UI flex features. */ public void presetFlex(FeedVersion feedVersion) { - flex = feedVersion.feedLoadResult.isGTFSFlex(); + if (feedVersion.feedLoadResult != null) flex = feedVersion.feedLoadResult.isGTFSFlex(); if (flex) flexUIFeaturesEnabled = true; } } From 820353a3e485e5818a0476b5015f2dad770c2104 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 22 Feb 2022 16:22:36 +0000 Subject: [PATCH 026/133] refactor(FeedVersion.java): Added null check Added null check on feed load result --- .../java/com/conveyal/datatools/manager/models/FeedVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index f091061b7..9d5ae6a4a 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -277,7 +277,7 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { FeedSource flexSetFeedSource = this.parentFeedSource(); - flexSetFeedSource.flex = this.feedLoadResult.isGTFSFlex(); + if (this.feedLoadResult != null) flexSetFeedSource.flex = this.feedLoadResult.isGTFSFlex(); Persistence.feedSources.replace(this.parentFeedSource().id, flexSetFeedSource); if (this.feedLoadResult.fatalException != null) { From 0984ef548db5b81b90152fcd13b322fd93bfaf57 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 25 Feb 2022 12:14:07 -0800 Subject: [PATCH 027/133] chore(deps): upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a77058a4..db0160f0a 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 4adfd10cd7c166e041f9d33f12f2daf4e8a4e90d + 81ea46c9574517f77674af636d4dfdcabbc1325a From 75ff6697645564330a2cb9b0589396a903ed7a7c Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 1 Mar 2022 12:03:16 -0500 Subject: [PATCH 028/133] chore(deps): upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index db0160f0a..b87924ffb 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 81ea46c9574517f77674af636d4dfdcabbc1325a + 1448ff2a120c9df2526b89c8ac00b47bfb058422 From b66c5bbd6c740df756e3aa9bfa6085e5ddc420b9 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 1 Mar 2022 16:44:47 -0500 Subject: [PATCH 029/133] chore(repos): add snapshot repository for opengeo --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index b87924ffb..a478101d8 100644 --- a/pom.xml +++ b/pom.xml @@ -188,6 +188,14 @@ false true + + + true + + opengeo + OpenGeo Maven Repository + http://repo.opengeo.org + sonatype sonatype snapshots From f7771dacd72852e7eda12b133446297f7802a7fc Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 1 Mar 2022 14:03:55 -0800 Subject: [PATCH 030/133] chore(deps): update opengeo repository --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a478101d8..7a5459a89 100644 --- a/pom.xml +++ b/pom.xml @@ -194,7 +194,7 @@ opengeo OpenGeo Maven Repository - http://repo.opengeo.org + https://repo.opengeo.org sonatype @@ -278,7 +278,7 @@ com.github.conveyal gtfs-lib - 1448ff2a120c9df2526b89c8ac00b47bfb058422 + 47f961ffcdbacdea1536dbc86c2283a59663be74 From db89282b1b4ca46563e06a6be04118b74a26f395 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 8 Mar 2022 16:58:58 +0000 Subject: [PATCH 031/133] refactor(pom.xml): Removed redundant repo and bumped GTFS-lib version --- pom.xml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 0a77058a4..92921812e 100644 --- a/pom.xml +++ b/pom.xml @@ -197,15 +197,6 @@ always - - boundless - Boundless Maven Repository - https://repo.boundlessgeo.com/main - - true - always - - jitpack.io @@ -270,7 +261,7 @@ com.github.conveyal gtfs-lib - 4adfd10cd7c166e041f9d33f12f2daf4e8a4e90d + 7e6550171393df667812c0ba328704a2130c472c From 6cfdbc049f88f196c0a1ebd36f4f8eb85ce2bbc2 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 17 Mar 2022 11:31:59 -0700 Subject: [PATCH 032/133] refactor(FeedSource): persist flex check results --- .../com/conveyal/datatools/manager/models/FeedSource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 9a596f6e4..b0f93212c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -771,5 +771,9 @@ public List getActiveTransformations(FeedVersi public void presetFlex(FeedVersion feedVersion) { if (feedVersion.feedLoadResult != null) flex = feedVersion.feedLoadResult.isGTFSFlex(); if (flex) flexUIFeaturesEnabled = true; + + // Save flex changes after check + Persistence.feedSources.updateField(this.id, "flex", flex); + Persistence.feedSources.updateField(this.id, "flexUIFeaturesEnabled", flexUIFeaturesEnabled); } } From 18e19c66ed01e17906c0ebe51505f2c4fa07dce0 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 19 Apr 2022 12:42:01 +0100 Subject: [PATCH 033/133] refactor(pom.xml): Reverted gtfs-lib back to previously used version which has been overwritten by m The latest commit with flex changes does not work, so using previous commit which does --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a79a82804..26c954ddc 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 7.1.0 + 7e6550171393df667812c0ba328704a2130c472c From 62d47db588a97c3d764093b264e01da8d5363aa9 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 27 Apr 2022 17:20:47 -0400 Subject: [PATCH 034/133] refactor(pom.xml): Bumped GTFS-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 26c954ddc..6f1c3fc5c 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 7e6550171393df667812c0ba328704a2130c472c + 03994beaea57352b0b8b7a84d848c60446c82cf5 From 0ada9b2f6951b5ec3b8ef15af3b590889bd6290b Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 28 Apr 2022 13:48:17 +0100 Subject: [PATCH 035/133] refactor(pom.xml): Bumped GTFS-Lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6f1c3fc5c..c2624fb42 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 03994beaea57352b0b8b7a84d848c60446c82cf5 + 1a47a3c52f42fb1f32d2564d0b5402320e6e4213 From 49db0e1689980059c9742bd25f17d641cb064099 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 6 May 2022 11:58:17 +0100 Subject: [PATCH 036/133] refactor(Defining feed source flex value): If a feed version is flex set the feed source to always b --- .../datatools/manager/models/FeedSource.java | 23 +++++++++++-------- .../datatools/manager/models/FeedVersion.java | 5 ---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 8e40e5c1c..449e070f3 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -177,9 +177,8 @@ public String organizationId () { public String editorNamespace; /** - * If this feed source references a feed version that is GTFS Flex, this value will be true. This is updated every - * time a feed version is retrieved as different feed versions may or may not be flex. The process to determine a - * flex feed is defined in {@link FeedLoadResult#isGTFSFlex()}. + * If this feed source references a feed version that is GTFS Flex, this value will permanently be set to true. + * This is evaluated every time a feed version is retrieved as different feed versions may or may not be flex. */ public boolean flex; @@ -775,14 +774,18 @@ public List getActiveTransformations(FeedVersi } /** - * Set the flex value according to the feed version loaded. If a flex feed is loaded, enable the UI flex features. + * If a flex feed version is loaded, set this feed source to permanently be a flex feed source. Subsequent feed + * versions which are not flex will have no affected. */ public void presetFlex(FeedVersion feedVersion) { - if (feedVersion.feedLoadResult != null) flex = feedVersion.feedLoadResult.isGTFSFlex(); - if (flex) flexUIFeaturesEnabled = true; - - // Save flex changes after check - Persistence.feedSources.updateField(this.id, "flex", flex); - Persistence.feedSources.updateField(this.id, "flexUIFeaturesEnabled", flexUIFeaturesEnabled); + boolean isFlexFeedVersion = false; + if (feedVersion.feedLoadResult != null) isFlexFeedVersion = feedVersion.feedLoadResult.isGTFSFlex(); + if (!flex && isFlexFeedVersion) { + // If the feed version is flex (and the feed source was not flex) permanently enable flex. + flex = flexUIFeaturesEnabled = true; + Persistence.feedSources.updateField(this.id, "flexUIFeaturesEnabled", true); + Persistence.feedSources.updateField(this.id, "flex", true); + } } + } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 78c34f362..15366636b 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -276,11 +276,6 @@ public void load(MonitorableJob.Status status, boolean isNewVersion) { String gtfsFilePath = gtfsFile.getPath(); this.feedLoadResult = GTFS.load(gtfsFilePath, DataManager.GTFS_DATA_SOURCE); - - FeedSource flexSetFeedSource = this.parentFeedSource(); - if (this.feedLoadResult != null) flexSetFeedSource.flex = this.feedLoadResult.isGTFSFlex(); - Persistence.feedSources.replace(this.parentFeedSource().id, flexSetFeedSource); - if (this.feedLoadResult.fatalException != null) { status.fail("Could not load feed due to " + feedLoadResult.fatalException); return; From 90654f93ff8fcc6da7ba8120753685c8b19ec318 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 6 May 2022 15:05:08 +0100 Subject: [PATCH 037/133] chore(pom.xml): Updated the gtfs-lib version gtfs-lib version now references the commit at the point it was merged into dev-flex --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c2624fb42..ca4942ee1 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 1a47a3c52f42fb1f32d2564d0b5402320e6e4213 + cee9d2ad3f34917d906c0507ef0dd537293470af From 9cc17b9df4a19ba813279ae0ef7513c45ec87447 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 6 May 2022 15:38:42 +0100 Subject: [PATCH 038/133] refactor(FeedSource.java): Removed removed extra blank line --- .../java/com/conveyal/datatools/manager/models/FeedSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 449e070f3..68fa3d575 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -787,5 +787,4 @@ public void presetFlex(FeedVersion feedVersion) { Persistence.feedSources.updateField(this.id, "flex", true); } } - } From 0da8d9f25dd77afde6fae59959fd2ca1b19d538c Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 22 Aug 2022 13:24:26 +0200 Subject: [PATCH 039/133] chore(deps): upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 014774820..4245f58a2 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - cee9d2ad3f34917d906c0507ef0dd537293470af + 3e727c9479555dfc9ec20a03eb025a7033bf5ece From a0d8455e833762a1581578f63e661b649836ced5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 12 Sep 2022 15:45:35 +0100 Subject: [PATCH 040/133] refactor(pom.xml): Bumped gtfs-lib to a newer version --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4245f58a2..f05411de3 100644 --- a/pom.xml +++ b/pom.xml @@ -269,8 +269,7 @@ com.github.conveyal gtfs-lib - - 3e727c9479555dfc9ec20a03eb025a7033bf5ece + ee5e98fdee55af1efdca83a34991b6ba54c429e9 From fa69dc76d94e2c4d1b6c0c1e8913d88ef61e2332 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 27 Sep 2022 15:27:54 -0400 Subject: [PATCH 041/133] chore: update gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6ca024a8..f05411de3 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 3aac816514738e3f10a80302ba1f03d8fed3f72c + ee5e98fdee55af1efdca83a34991b6ba54c429e9 From e91d578712df1bb749696312d6ede040970b3af8 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 23 Nov 2022 13:47:11 +0000 Subject: [PATCH 042/133] chore(Bumped gtfs-lib version): Update of gtfs-lib include better geo json handling --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f05411de3..c419346d5 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - ee5e98fdee55af1efdca83a34991b6ba54c429e9 + c9c6fab615715fa7ad5c8799a5859e69b1ebff82 From ee57290f80e445a9d5a267785c6eec448561e3fe Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 16 Dec 2022 16:17:21 +0000 Subject: [PATCH 043/133] refactor(pom.xml): Bumped gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2ac77db4b..56033b7fa 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - c9c6fab615715fa7ad5c8799a5859e69b1ebff82 + fa9b4d6a15a7ce0fd12df28ce1761e1b0842fce2 From 4a0e862200d8162a083a5e5d8a4ca0b69eee390d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 10 Feb 2023 10:08:51 +0000 Subject: [PATCH 044/133] chore(Bump gtfs-lib version): Updated to latest hash --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07d4f0442..ca84ce94a 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - fca4bbaa2a4fda800d22a668c236e308fbf60f5b + 50cd60b From acb367d14e5e40a6e16d03380703c3c1b0978c4b Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 14 Mar 2023 10:35:04 -0400 Subject: [PATCH 045/133] chore(flex): bump gtfs-lib version for flex --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ca84ce94a..10345de1c 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 50cd60b + ecc0ca7 From 72b9e9058743371885c1f30abe4aca9e9e6fd1b6 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 29 Mar 2023 12:36:54 -0400 Subject: [PATCH 046/133] chore(flex): bump gtfs-lib version for flex --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 10345de1c..d885cfde3 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - ecc0ca7 + 4fcf36e From f1779014ca36106f765dd807e066f83d258f820a Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 11 Apr 2023 14:03:23 -0400 Subject: [PATCH 047/133] chore(flex): fix commit hash --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d885cfde3..cfc3a0a20 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 4fcf36e + 9ad47be From 6e5d5b68109a4302382c5d7bdb91fea54e34e225 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 13 Apr 2023 07:57:27 +0100 Subject: [PATCH 048/133] refactor(GTFS-lib version update): Update to use the location group replacement --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cfc3a0a20..a05305864 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 9ad47be + c35da9a From 6c36fff120e64edf1e3919e4a4da3bc9e2c1b63a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 19 Apr 2023 08:33:56 +0100 Subject: [PATCH 049/133] refactor(pom.xml): Updated gtfs-lib reference --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cfc3a0a20..d7776f24e 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 9ad47be + 084cb3f From 47683457e02b56bdea2108c0463c0f32e10a3b1a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 20 Apr 2023 08:55:36 +0100 Subject: [PATCH 050/133] refactor(pom.xml): Bumped GTFS-lib version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d7776f24e..5b3d1be1a 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - 084cb3f + 34a75aa From 38837477cbdc7f2ded17c73eeabeca824409b845 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 2 May 2023 15:54:51 -0400 Subject: [PATCH 051/133] feat(DataManager): use stop_areas and areas rather than location_groups --- .../java/com/conveyal/datatools/manager/DataManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/DataManager.java b/src/main/java/com/conveyal/datatools/manager/DataManager.java index f1ba033eb..a201c2912 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataManager.java +++ b/src/main/java/com/conveyal/datatools/manager/DataManager.java @@ -210,9 +210,11 @@ static void registerRoutes() throws IOException { // NOTE: fare_attributes controller handles updates to nested table fare_rules. new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_ATTRIBUTES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE); - // NOTE: Booking rules, location groups, locations and location shapes are GTFS Flex additions. + // NOTE: AREA and STOP_AREAS are additions for GTFS Flex but belong to the Fares v2 specification: + new EditorControllerImpl(EDITOR_API_PREFIX, Table.STOP_AREAS, DataManager.GTFS_DATA_SOURCE); + new EditorControllerImpl(EDITOR_API_PREFIX, Table.AREA, DataManager.GTFS_DATA_SOURCE); + // NOTE: Booking rules, locations and location shapes are GTFS Flex additions. new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE); - new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUPS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE); new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE); From a26f07568649569176d6792a40546e3250e169c7 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 16 May 2023 10:26:27 -0400 Subject: [PATCH 052/133] chore(gtfs-lib): update gtfs-lib to include location group changes. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a05305864..461203aba 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - c35da9a + df937fd From dc699a600c764ac6ea8e3b2b08e70f56c37ab228 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 28 Jun 2023 16:51:59 -0400 Subject: [PATCH 053/133] refactor(stop areas): update gtfs-lib reference --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 461203aba..b58f4c66b 100644 --- a/pom.xml +++ b/pom.xml @@ -269,7 +269,7 @@ com.github.conveyal gtfs-lib - df937fd + 09eb1fd From 40c39e248a5ad7c42bbabbaf2344e9f7a0d9bb55 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 10 Jul 2023 12:13:37 +0100 Subject: [PATCH 054/133] refactor(Bumped GTFS-Lib version): Version now matches the latest GTFS-Lib commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 62b14da2b..a3ff73dba 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 7826c92 + 5c383ab From 3cff05ef819f8f5d31a959bb5d68ecf8aae14acd Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 20 Jul 2023 12:06:41 +0100 Subject: [PATCH 055/133] refactor(Update to various transform classes to accommodate flex): --- pom.xml | 2 +- .../models/transform/FeedTransformation.java | 13 ++++++- .../NormalizeFieldTransformation.java | 5 +++ .../PreserveCustomFieldsTransformation.java | 10 ++--- .../ReplaceFileFromVersionTransformation.java | 2 +- .../transform/StringTransformation.java | 2 +- .../jobs/NormalizeFieldTransformJobTest.java | 39 +++++++++++++++---- .../fake-agency-with-only-calendar/areas.txt | 2 + .../booking_rules.txt | 2 + .../stop_areas.txt | 2 + 10 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt diff --git a/pom.xml b/pom.xml index 782449668..a330a27a9 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 5c383ab + 890bb29 diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index e63001b37..b7a275cf1 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.utils.GtfsUtils; +import com.conveyal.gtfs.loader.Table; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -100,8 +101,16 @@ public void doTransform(FeedTransformTarget target, MonitorableJob.Status status */ protected void validateTableName(MonitorableJob.Status status) { // Validate fields before running transform. - if (GtfsUtils.getGtfsTable(table) == null) { - status.fail("Table must be valid GTFS spec table name (without .txt)."); + if (GtfsUtils.getGtfsTable(table) == null && !Table.LOCATION_GEO_JSON_FILE_NAME.equals(table)) { + status.fail(String.format("Table must be valid GTFS spec table name (without .txt) or %s.", Table.LOCATION_GEO_JSON_FILE_NAME)); } } + + protected String getTableName() { + return Table.LOCATION_GEO_JSON_FILE_NAME.equals(table) ? table : table + ".txt"; + } + + protected String getTableSuffix() { + return Table.LOCATION_GEO_JSON_FILE_NAME.equals(table) ? ".geojson" : ".txt"; + } } diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index e9c4d6ce7..ccf7b94d5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -179,6 +179,11 @@ public static List getInvalidSubstitutionPatterns(List sub @Override public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { + if (Table.LOCATION_GEO_JSON_FILE_NAME.equals(table)) { + // It's not possible to select the locations.geojson file from the normalize field transformation list. This + // is here in case that changes. + throw new UnsupportedOperationException("It is not possible to normalize geo json fields."); + } String tableName = table + ".txt"; try( // Hold output before writing to ZIP diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java index e247d2db3..5bf92e327 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/PreserveCustomFieldsTransformation.java @@ -60,14 +60,14 @@ private static HashMap> createCsvHashMap(CsvMapReade @Override public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) throws Exception{ - String tableName = table + ".txt"; + String tableName = getTableName(); Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); Optional streamResult = Arrays.stream(Table.tablesInOrder) .filter(t -> t.name.equals(table)) .findFirst(); - if (!streamResult.isPresent()) { - throw new Exception(String.format("could not find specTable for table %s", table)); + if (streamResult.isEmpty()) { + throw new IOException(String.format("could not find specTable for table %s", table)); } Table specTable = streamResult.get(); @@ -77,8 +77,8 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); - final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); - File output = File.createTempFile(tableName + "-output-temp", ".txt"); + final File tempFile = File.createTempFile(tableName + "-temp", getTableSuffix()); + File output = File.createTempFile(tableName + "-output-temp", getTableSuffix()); int rowsModified = 0; List customFields; diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java index 53963fb16..76234ae3e 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/ReplaceFileFromVersionTransformation.java @@ -42,7 +42,7 @@ public void validateParameters(MonitorableJob.Status status) { @Override public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { FeedVersion sourceVersion = getSourceVersion(); - String tableName = table + ".txt"; + String tableName = getTableName(); // Run the replace transformation Path sourceZipPath = Paths.get(sourceVersion.retrieveGtfsFile().getAbsolutePath()); try (FileSystem sourceZipFs = FileSystems.newFileSystem(sourceZipPath, (ClassLoader) null)) { diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java index c845d4f37..f822eb6b9 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/StringTransformation.java @@ -32,7 +32,7 @@ public void validateParameters(MonitorableJob.Status status) { @Override public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { - String tableName = table + ".txt"; + String tableName = getTableName(); Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); try ( FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 8eb99259e..99f193cc7 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -12,18 +12,25 @@ import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformationTest; import com.conveyal.datatools.manager.models.transform.Substitution; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.loader.JdbcTableWriter; +import com.conveyal.gtfs.loader.Table; import com.google.common.collect.Lists; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Date; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -71,17 +78,27 @@ public void tearDownTest() { if (targetVersion != null) targetVersion.delete(); } + private static Stream createNormalizeFieldReplacements() { + return Stream.of( + Arguments.of("route", "route_long_name", "Route", "Rte"), + Arguments.of("booking_rules", "booking_rule_id", "Booking", "Bkg"), + Arguments.of("stop_areas", "area_id", "Area", "StopArea"), + Arguments.of("areas", "area_id", "Area", "Loc") + ); + } + /** * Test that a {@link NormalizeFieldTransformation} will successfully complete. * FIXME: On certain Windows machines, this test fails. */ - @Test - public void canNormalizeField() throws IOException { + @ParameterizedTest + @MethodSource("createNormalizeFieldReplacements") + void canNormalizeField(String tableName, String fieldName, String pattern, String replacement) throws IOException { // Create transform. // In this test, as an illustration, replace "Route" with the "Rte" abbreviation in routes.txt. FeedTransformation transformation = NormalizeFieldTransformationTest.createTransformation( - TABLE_NAME, FIELD_NAME, null, Lists.newArrayList( - new Substitution("Route", "Rte") + tableName, fieldName, null, Lists.newArrayList( + new Substitution(pattern, replacement) ) ); initializeFeedSource(transformation); @@ -94,7 +111,7 @@ public void canNormalizeField() throws IOException { try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has routes table modified. - ZipEntry entry = zip.getEntry(TABLE_NAME + ".txt"); + ZipEntry entry = zip.getEntry(tableName + ".txt"); assertNotNull(entry); // Scan the first data row in routes.txt and check that the substitution @@ -105,11 +122,11 @@ public void canNormalizeField() throws IOException { BufferedReader reader = new BufferedReader(streamReader) ) { String[] columns = reader.readLine().split(","); - int fieldIndex = ArrayUtils.indexOf(columns, FIELD_NAME); + int fieldIndex = ArrayUtils.indexOf(columns, fieldName); String row1 = reader.readLine(); String[] row1Fields = row1.split(","); - assertTrue(row1Fields[fieldIndex].startsWith("Rte "), row1); + assertTrue(row1Fields[fieldIndex].startsWith(replacement), row1); } } } @@ -140,6 +157,14 @@ public void canHandleInvalidSubstitutionPatterns() throws IOException { assertTrue(targetVersion.hasCriticalErrors()); } + @Test + void canHandleUnsupportedLocationsGeoJsonFile() { + FeedTransformation transformation = NormalizeFieldTransformationTest.createTransformation( + Table.LOCATION_GEO_JSON_FILE_NAME, "", null, null + ); + Assertions.assertThrows(UnsupportedOperationException.class, () -> transformation.transform(null, null)); + } + /** * Create and persist a feed source using the given transformation. */ diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt new file mode 100644 index 000000000..6b1bbd6ec --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt @@ -0,0 +1,2 @@ +area_id,name +Area One,This is area one diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt new file mode 100644 index 000000000..32b28163a --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt @@ -0,0 +1,2 @@ +booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url +Booking rule 1,0,,,,,,,,,,,,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt new file mode 100644 index 000000000..77f143856 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt @@ -0,0 +1,2 @@ +area_id,stop_id +Area One,tc6v From 47d1636bb7e4dc366763fc9ec756cc6da31e5adf Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 20 Jul 2023 16:35:48 +0100 Subject: [PATCH 056/133] refactor(EditorControllerTest.java): Changed feed source to one that has less test data --- .../editor/controllers/api/EditorControllerTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index 45a658b3f..b6ead3b37 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -73,7 +73,7 @@ public static void setUp() throws Exception { Persistence.feedSources.create(feedSourceCascadeDelete); feedVersion = createFeedVersionFromGtfsZip(feedSource, "bart_old.zip"); - feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "bart_old.zip"); + feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "caltrain_gtfs_lite.zip"); // Create and run snapshot jobs crateAndRunSnapshotJob(feedVersion.name, feedSource.id, feedVersion.namespace); @@ -172,15 +172,15 @@ public void canPatchStopsConditionally() throws IOException { void canCascadeDeleteStop() throws IOException, SQLException { // Get a fresh feed source so that the editor namespace was updated after snapshot. FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersionCascadeDelete.feedSourceId); - String stopId = "WARM"; + String stopId = "Bayshore Caltrain"; String stopCountSql = getCountSql(freshFeedSource.editorNamespace, "stops", stopId); String stopTimesCountSql = getCountSql(freshFeedSource.editorNamespace, "stop_times", stopId); String patternStopsCountSql = getCountSql(freshFeedSource.editorNamespace, "pattern_stops", stopId); // Check for presence of stopId in stops, stop times and pattern stops. assertThatSqlCountQueryYieldsExpectedCount(stopCountSql, 1); - assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 522); - assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 4); + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 1); + assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 1); String path = String.format( "/api/editor/secure/stop/%s/cascadeDeleteStop?feedId=%s&sessionId=test", From 946461fca5c516a75fb7e3a386e60ec8121a8b03 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 24 Jul 2023 11:30:54 +0100 Subject: [PATCH 057/133] refactor(Unit test and gtfs-lib update): Update to use refactored stop time normalization (from gtfs --- pom.xml | 2 +- .../editor/controllers/api/EditorControllerTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a330a27a9..0b2183eb5 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 890bb29 + 97311fe diff --git a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java index b6ead3b37..45a658b3f 100644 --- a/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java +++ b/src/test/java/com/conveyal/datatools/editor/controllers/api/EditorControllerTest.java @@ -73,7 +73,7 @@ public static void setUp() throws Exception { Persistence.feedSources.create(feedSourceCascadeDelete); feedVersion = createFeedVersionFromGtfsZip(feedSource, "bart_old.zip"); - feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "caltrain_gtfs_lite.zip"); + feedVersionCascadeDelete = createFeedVersionFromGtfsZip(feedSourceCascadeDelete, "bart_old.zip"); // Create and run snapshot jobs crateAndRunSnapshotJob(feedVersion.name, feedSource.id, feedVersion.namespace); @@ -172,15 +172,15 @@ public void canPatchStopsConditionally() throws IOException { void canCascadeDeleteStop() throws IOException, SQLException { // Get a fresh feed source so that the editor namespace was updated after snapshot. FeedSource freshFeedSource = Persistence.feedSources.getById(feedVersionCascadeDelete.feedSourceId); - String stopId = "Bayshore Caltrain"; + String stopId = "WARM"; String stopCountSql = getCountSql(freshFeedSource.editorNamespace, "stops", stopId); String stopTimesCountSql = getCountSql(freshFeedSource.editorNamespace, "stop_times", stopId); String patternStopsCountSql = getCountSql(freshFeedSource.editorNamespace, "pattern_stops", stopId); // Check for presence of stopId in stops, stop times and pattern stops. assertThatSqlCountQueryYieldsExpectedCount(stopCountSql, 1); - assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 1); - assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 1); + assertThatSqlCountQueryYieldsExpectedCount(stopTimesCountSql, 522); + assertThatSqlCountQueryYieldsExpectedCount(patternStopsCountSql, 4); String path = String.format( "/api/editor/secure/stop/%s/cascadeDeleteStop?feedId=%s&sessionId=test", From 368b98a7cd809b7646ac26d9c9e0543aef77da3f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 24 Jul 2023 13:52:09 +0100 Subject: [PATCH 058/133] refactor(pom.xml): Reverted gtfs-lib back to latest dev-flex commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b2183eb5..90c306df2 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 97311fe + 4806767 From 70d9554332df110ef5ebb7897ffd6ad98254ba2f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 25 Jul 2023 13:30:47 +0100 Subject: [PATCH 059/133] refactor(pom.xml): Changed gtfs-lib to reference latest stop times normalization refactor --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 90c306df2..eb6694b04 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 4806767 + 3c5fd30 From e0f19d3690d84dd6339fba15241982d53e8cbcfc Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 26 Jul 2023 17:34:32 +0100 Subject: [PATCH 060/133] refactor(Test data update): Added missing commas to test data which was causing a unit test to not f --- .../gtfs/fake-agency-with-only-calendar/booking_rules.txt | 2 +- .../gtfs/fake-agency-with-only-calendar/stop_areas.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt index 32b28163a..e04ef917e 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt @@ -1,2 +1,2 @@ booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url -Booking rule 1,0,,,,,,,,,,,,, +Booking rule 1,0,,,,,,,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt index 77f143856..ff04dc91d 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt @@ -1,2 +1,2 @@ area_id,stop_id -Area One,tc6v +Area One,tc6v, \ No newline at end of file From c3a3a7182a6b2237ceefba00d6144f42fb817216 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 10 Aug 2023 14:53:52 +0100 Subject: [PATCH 061/133] refactor(Updated unit test): Created distinct feed for normalizing fields --- .../manager/jobs/NormalizeFieldTransformJobTest.java | 4 ++-- .../gtfs/fake-agency-for-field-normalizing/areas.txt | 2 ++ .../booking_rules.txt | 2 +- .../gtfs/fake-agency-for-field-normalizing/routes.txt | 3 +++ .../gtfs/fake-agency-for-field-normalizing/stop_areas.txt | 2 ++ .../datatools/gtfs/fake-agency-with-only-calendar/areas.txt | 2 -- .../gtfs/fake-agency-with-only-calendar/stop_times.txt | 5 ----- 7 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt rename src/test/resources/com/conveyal/datatools/gtfs/{fake-agency-with-only-calendar => fake-agency-for-field-normalizing}/booking_rules.txt (54%) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt delete mode 100755 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 99f193cc7..0b89bd281 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -80,7 +80,7 @@ public void tearDownTest() { private static Stream createNormalizeFieldReplacements() { return Stream.of( - Arguments.of("route", "route_long_name", "Route", "Rte"), + Arguments.of("routes", "route_long_name", "Route", "Rte"), Arguments.of("booking_rules", "booking_rule_id", "Booking", "Bkg"), Arguments.of("stop_areas", "area_id", "Area", "StopArea"), Arguments.of("areas", "area_id", "Area", "Loc") @@ -106,7 +106,7 @@ void canNormalizeField(String tableName, String fieldName, String pattern, Strin // Create target version that the transform will operate on. targetVersion = createFeedVersion( feedSource, - zipFolderFiles("fake-agency-with-only-calendar") + zipFolderFiles("fake-agency-for-field-normalizing") ); try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt new file mode 100644 index 000000000..73b9b8eb9 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt @@ -0,0 +1,2 @@ +area_id,area_name +Area One,This is area one, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/booking_rules.txt similarity index 54% rename from src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt rename to src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/booking_rules.txt index e04ef917e..398fe59e0 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/booking_rules.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/booking_rules.txt @@ -1,2 +1,2 @@ booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url -Booking rule 1,0,,,,,,,,,,,,,, \ No newline at end of file +Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://www.info_url.com","http://www.booking_url.com" diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/routes.txt new file mode 100644 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt new file mode 100644 index 000000000..ff04dc91d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt @@ -0,0 +1,2 @@ +area_id,stop_id +Area One,tc6v, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt deleted file mode 100644 index 6b1bbd6ec..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/areas.txt +++ /dev/null @@ -1,2 +0,0 @@ -area_id,name -Area One,This is area one diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt deleted file mode 100755 index 646706c5c..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt +++ /dev/null @@ -1,5 +0,0 @@ -trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint -only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, -only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, -only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file From 379809ca88d2350aad8acba784ecc534e1425bfc Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 10 Aug 2023 15:27:13 +0100 Subject: [PATCH 062/133] refactor(Corrected feed config): Deleted stop areas and re-instated stop times --- .../gtfs/fake-agency-with-only-calendar/stop_areas.txt | 2 -- .../gtfs/fake-agency-with-only-calendar/stop_times.txt | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt deleted file mode 100644 index ff04dc91d..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_areas.txt +++ /dev/null @@ -1,2 +0,0 @@ -area_id,stop_id -Area One,tc6v, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt new file mode 100644 index 000000000..646706c5c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-only-calendar/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file From dd91fb95b903c4e1cdf23779ad3080f982fdb06c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 10 Aug 2023 16:30:09 +0100 Subject: [PATCH 063/133] refactor(NormalizeFieldTransformJobTest.java): Refactored to create multiple transformation type wit --- .../jobs/NormalizeFieldTransformJobTest.java | 128 ++++++++++-------- 1 file changed, 71 insertions(+), 57 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 0b89bd281..649590b97 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -12,25 +12,19 @@ import com.conveyal.datatools.manager.models.transform.NormalizeFieldTransformationTest; import com.conveyal.datatools.manager.models.transform.Substitution; import com.conveyal.datatools.manager.persistence.Persistence; -import com.conveyal.gtfs.loader.JdbcTableWriter; -import com.conveyal.gtfs.loader.Table; import com.google.common.collect.Lists; import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Date; -import java.util.stream.Stream; +import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -78,30 +72,23 @@ public void tearDownTest() { if (targetVersion != null) targetVersion.delete(); } - private static Stream createNormalizeFieldReplacements() { - return Stream.of( - Arguments.of("routes", "route_long_name", "Route", "Rte"), - Arguments.of("booking_rules", "booking_rule_id", "Booking", "Bkg"), - Arguments.of("stop_areas", "area_id", "Area", "StopArea"), - Arguments.of("areas", "area_id", "Area", "Loc") - ); - } - /** * Test that a {@link NormalizeFieldTransformation} will successfully complete. * FIXME: On certain Windows machines, this test fails. */ - @ParameterizedTest - @MethodSource("createNormalizeFieldReplacements") - void canNormalizeField(String tableName, String fieldName, String pattern, String replacement) throws IOException { - // Create transform. - // In this test, as an illustration, replace "Route" with the "Rte" abbreviation in routes.txt. - FeedTransformation transformation = NormalizeFieldTransformationTest.createTransformation( - tableName, fieldName, null, Lists.newArrayList( - new Substitution(pattern, replacement) - ) - ); - initializeFeedSource(transformation); + @Test + void canNormalizeField() throws IOException { + TransformationCase route = new TransformationCase("routes", "route_long_name", "Route", "Rte"); + TransformationCase bookingRules = new TransformationCase("booking_rules", "booking_rule_id", "Booking", "Bkg"); + TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); + TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); + // Create transformations. + initializeFeedSource(List.of( + createTransformation(route), + createTransformation(bookingRules), + createTransformation(stopAreas), + createTransformation(areas) + )); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -110,24 +97,32 @@ void canNormalizeField(String tableName, String fieldName, String pattern, Strin ); try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { - // Check that new version has routes table modified. - ZipEntry entry = zip.getEntry(tableName + ".txt"); - assertNotNull(entry); - - // Scan the first data row in routes.txt and check that the substitution - // that was defined in setUp was done. - try ( - InputStream stream = zip.getInputStream(entry); - InputStreamReader streamReader = new InputStreamReader(stream); - BufferedReader reader = new BufferedReader(streamReader) - ) { - String[] columns = reader.readLine().split(","); - int fieldIndex = ArrayUtils.indexOf(columns, fieldName); - - String row1 = reader.readLine(); - String[] row1Fields = row1.split(","); - assertTrue(row1Fields[fieldIndex].startsWith(replacement), row1); - } + // Check that new version has expected modifications. + checkTableForModification(zip, route); + checkTableForModification(zip, bookingRules); + checkTableForModification(zip, stopAreas); + checkTableForModification(zip, areas); + } + } + + private void checkTableForModification(ZipFile zip, TransformationCase transformationCase) throws IOException { + // Check that the new version has been modified. + ZipEntry entry = zip.getEntry(transformationCase.table + ".txt"); + assertNotNull(entry); + + // Scan the first data row and check that the substitution that was defined in the set-up was done. + try ( + InputStream stream = zip.getInputStream(entry); + InputStreamReader streamReader = new InputStreamReader(stream); + BufferedReader reader = new BufferedReader(streamReader) + ) { + String[] columns = reader.readLine().split(","); + int fieldIndex = ArrayUtils.indexOf(columns, transformationCase.fieldName); + + String row1 = reader.readLine(); + assertNotNull(row1, String.format("First row in table %s is null!", transformationCase.table)); + String[] row1Fields = row1.split(","); + assertTrue(row1Fields[fieldIndex].startsWith(transformationCase.replacement), row1); } } @@ -135,7 +130,7 @@ void canNormalizeField(String tableName, String fieldName, String pattern, Strin * Test that a {@link NormalizeFieldTransformation} will fail if invalid substitution patterns are provided. */ @Test - public void canHandleInvalidSubstitutionPatterns() throws IOException { + void canHandleInvalidSubstitutionPatterns() throws IOException { // Create transform. // In this test, we provide an invalid pattern '\Cir\b' (instead of '\bCir\b'), // when trying to replace e.g. 'Piedmont Cir' with 'Piedmont Circle'. @@ -144,7 +139,7 @@ public void canHandleInvalidSubstitutionPatterns() throws IOException { new Substitution("\\Cir\\b", "Circle") ) ); - initializeFeedSource(transformation); + initializeFeedSource(List.of(transformation)); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -157,24 +152,29 @@ public void canHandleInvalidSubstitutionPatterns() throws IOException { assertTrue(targetVersion.hasCriticalErrors()); } - @Test - void canHandleUnsupportedLocationsGeoJsonFile() { - FeedTransformation transformation = NormalizeFieldTransformationTest.createTransformation( - Table.LOCATION_GEO_JSON_FILE_NAME, "", null, null - ); - Assertions.assertThrows(UnsupportedOperationException.class, () -> transformation.transform(null, null)); + private FeedTransformation createTransformation(TransformationCase transformationCase) { + return NormalizeFieldTransformationTest + .createTransformation( + transformationCase.table, + transformationCase.fieldName, + null, + Lists.newArrayList( + new Substitution(transformationCase.pattern, transformationCase.replacement) + ) + ); } /** * Create and persist a feed source using the given transformation. */ - private static void initializeFeedSource(FeedTransformation transformation) { - FeedTransformRules transformRules = new FeedTransformRules(transformation); + private static void initializeFeedSource(List> transformations) { // Create feed source with above transform. feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); feedSource.deployable = false; - feedSource.transformRules.add(transformRules); + for (FeedTransformation transformation : transformations) { + feedSource.transformRules.add(new FeedTransformRules(transformation)); + } Persistence.feedSources.create(feedSource); } @@ -182,8 +182,22 @@ private static void initializeFeedSource(FeedTransformation Date: Fri, 11 Aug 2023 11:12:09 +0100 Subject: [PATCH 064/133] refactor(Bumped gtfs lib to latest dev-flex version.): Updated test data for normalizing. --- pom.xml | 2 +- .../datatools/manager/jobs/NormalizeFieldTransformJobTest.java | 2 +- .../datatools/gtfs/fake-agency-for-field-normalizing/areas.txt | 2 +- .../gtfs/fake-agency-for-field-normalizing/stop_areas.txt | 2 +- .../datatools/gtfs/fake-agency-for-field-normalizing/stops.txt | 2 ++ 5 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stops.txt diff --git a/pom.xml b/pom.xml index eb6694b04..51333f166 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 3c5fd30 + 5b4eec6 diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 649590b97..dd839b380 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -96,7 +96,7 @@ void canNormalizeField() throws IOException { zipFolderFiles("fake-agency-for-field-normalizing") ); - try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { + try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has expected modifications. checkTableForModification(zip, route); checkTableForModification(zip, bookingRules); diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt index 73b9b8eb9..6eefe8872 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/areas.txt @@ -1,2 +1,2 @@ area_id,area_name -Area One,This is area one, +Area One,"This is area one" diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt index ff04dc91d..f83e4e19f 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt @@ -1,2 +1,2 @@ area_id,stop_id -Area One,tc6v, \ No newline at end of file +Area One,4u6g \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stops.txt new file mode 100644 index 000000000..38c4c23f7 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stops.txt @@ -0,0 +1,2 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, From 6ffa121064bfca9ec0ccc9f4b210548fbed7ff2a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Aug 2023 12:36:26 +0100 Subject: [PATCH 065/133] refactor(Added additional trips file): unit test failing validation because of missing trips file --- .../datatools/gtfs/fake-agency-for-field-normalizing/trips.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/trips.txt diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/trips.txt new file mode 100644 index 000000000..387b076cd --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip1,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file From 7a1f101e2c1bcc72a4dae40231be1534dba0c3d2 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Aug 2023 13:36:44 +0100 Subject: [PATCH 066/133] refactor(Reduce normalize fields unit test to just routes): --- .../jobs/NormalizeFieldTransformJobTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index dd839b380..c62718af7 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -79,15 +79,15 @@ public void tearDownTest() { @Test void canNormalizeField() throws IOException { TransformationCase route = new TransformationCase("routes", "route_long_name", "Route", "Rte"); - TransformationCase bookingRules = new TransformationCase("booking_rules", "booking_rule_id", "Booking", "Bkg"); - TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); - TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); +// TransformationCase bookingRules = new TransformationCase("booking_rules", "booking_rule_id", "Booking", "Bkg"); +// TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); +// TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); // Create transformations. initializeFeedSource(List.of( - createTransformation(route), - createTransformation(bookingRules), - createTransformation(stopAreas), - createTransformation(areas) + createTransformation(route) +// createTransformation(bookingRules), +// createTransformation(stopAreas), +// createTransformation(areas) )); // Create target version that the transform will operate on. @@ -99,9 +99,9 @@ void canNormalizeField() throws IOException { try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has expected modifications. checkTableForModification(zip, route); - checkTableForModification(zip, bookingRules); - checkTableForModification(zip, stopAreas); - checkTableForModification(zip, areas); +// checkTableForModification(zip, bookingRules); +// checkTableForModification(zip, stopAreas); +// checkTableForModification(zip, areas); } } From 31992839c2fc330e2e1d5cb845ac3b8168282198 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Aug 2023 14:02:55 +0100 Subject: [PATCH 067/133] refactor(Re-instated areas and stop areas normalize field transformations): --- .../jobs/NormalizeFieldTransformJobTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index c62718af7..b3d112ced 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -80,14 +80,14 @@ public void tearDownTest() { void canNormalizeField() throws IOException { TransformationCase route = new TransformationCase("routes", "route_long_name", "Route", "Rte"); // TransformationCase bookingRules = new TransformationCase("booking_rules", "booking_rule_id", "Booking", "Bkg"); -// TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); -// TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); + TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); + TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); // Create transformations. initializeFeedSource(List.of( - createTransformation(route) + createTransformation(route), // createTransformation(bookingRules), -// createTransformation(stopAreas), -// createTransformation(areas) + createTransformation(stopAreas), + createTransformation(areas) )); // Create target version that the transform will operate on. @@ -100,8 +100,8 @@ void canNormalizeField() throws IOException { // Check that new version has expected modifications. checkTableForModification(zip, route); // checkTableForModification(zip, bookingRules); -// checkTableForModification(zip, stopAreas); -// checkTableForModification(zip, areas); + checkTableForModification(zip, stopAreas); + checkTableForModification(zip, areas); } } From bfa99ebfe67ca7cba2658f97c0e2a2c473d62100 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Aug 2023 14:48:26 +0100 Subject: [PATCH 068/133] refactor(NormalizeFieldTransformJobTest): Updated fields to be normalized. Removed stop areas becaus --- .../manager/jobs/NormalizeFieldTransformJobTest.java | 11 ++++------- .../fake-agency-for-field-normalizing/stop_areas.txt | 2 -- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index b3d112ced..3aa79408f 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -79,14 +79,12 @@ public void tearDownTest() { @Test void canNormalizeField() throws IOException { TransformationCase route = new TransformationCase("routes", "route_long_name", "Route", "Rte"); -// TransformationCase bookingRules = new TransformationCase("booking_rules", "booking_rule_id", "Booking", "Bkg"); - TransformationCase stopAreas = new TransformationCase("stop_areas", "area_id", "Area", "StopArea"); - TransformationCase areas = new TransformationCase("areas", "area_id", "Area", "Location"); + TransformationCase bookingRules = new TransformationCase("booking_rules", "pickup_message", "message", "msg"); + TransformationCase areas = new TransformationCase("areas", "area_name", "area", "location"); // Create transformations. initializeFeedSource(List.of( + createTransformation(bookingRules), createTransformation(route), -// createTransformation(bookingRules), - createTransformation(stopAreas), createTransformation(areas) )); @@ -99,8 +97,7 @@ void canNormalizeField() throws IOException { try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has expected modifications. checkTableForModification(zip, route); -// checkTableForModification(zip, bookingRules); - checkTableForModification(zip, stopAreas); + checkTableForModification(zip, bookingRules); checkTableForModification(zip, areas); } } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt deleted file mode 100644 index f83e4e19f..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-for-field-normalizing/stop_areas.txt +++ /dev/null @@ -1,2 +0,0 @@ -area_id,stop_id -Area One,4u6g \ No newline at end of file From a42d8b62dfa2d53612c5d55500da6c22409df206 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 11 Aug 2023 15:39:27 +0100 Subject: [PATCH 069/133] refactor(Created separate feed source for normalizing field unit tests): --- .../jobs/NormalizeFieldTransformJobTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 3aa79408f..fee9968c1 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -38,7 +38,6 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { private static final String TABLE_NAME = "routes"; private static final String FIELD_NAME = "route_long_name"; private static Project project; - private static FeedSource feedSource; private FeedVersion targetVersion; @@ -82,19 +81,19 @@ void canNormalizeField() throws IOException { TransformationCase bookingRules = new TransformationCase("booking_rules", "pickup_message", "message", "msg"); TransformationCase areas = new TransformationCase("areas", "area_name", "area", "location"); // Create transformations. - initializeFeedSource(List.of( - createTransformation(bookingRules), + FeedSource feedSourceNormalize = initializeFeedSource(List.of( createTransformation(route), + createTransformation(bookingRules), createTransformation(areas) )); // Create target version that the transform will operate on. targetVersion = createFeedVersion( - feedSource, + feedSourceNormalize, zipFolderFiles("fake-agency-for-field-normalizing") ); - try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { + try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has expected modifications. checkTableForModification(zip, route); checkTableForModification(zip, bookingRules); @@ -136,7 +135,7 @@ void canHandleInvalidSubstitutionPatterns() throws IOException { new Substitution("\\Cir\\b", "Circle") ) ); - initializeFeedSource(List.of(transformation)); + FeedSource feedSource = initializeFeedSource(List.of(transformation)); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -164,15 +163,16 @@ private FeedTransformation createTransformation(Transfor /** * Create and persist a feed source using the given transformation. */ - private static void initializeFeedSource(List> transformations) { + private FeedSource initializeFeedSource(List> transformations) { // Create feed source with above transform. - feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); + FeedSource feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); feedSource.deployable = false; for (FeedTransformation transformation : transformations) { feedSource.transformRules.add(new FeedTransformRules(transformation)); } Persistence.feedSources.create(feedSource); + return feedSource; } // FIXME: Refactor (almost same code as AutoDeployJobTest in PR #361, From 76a63e1ca2f5a3da45bde7df159a504798ed7235 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 14 Aug 2023 15:50:37 +0100 Subject: [PATCH 070/133] refactor(Bug fixed and parameterized normalizing of field tests): --- .../manager/jobs/ProcessSingleFeedJob.java | 24 +++++++---- .../datatools/manager/models/FeedVersion.java | 4 +- .../jobs/NormalizeFieldTransformJobTest.java | 40 +++++++++---------- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index f26eba655..b344b6608 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -13,6 +13,7 @@ import com.conveyal.datatools.manager.models.transform.FeedTransformRules; import com.conveyal.datatools.manager.models.transform.FeedTransformZipTarget; import com.conveyal.datatools.manager.models.transform.ZipTransformation; +import com.conveyal.gtfs.validator.ValidationResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.slf4j.Logger; @@ -198,15 +199,22 @@ private String getErrorReasonMessage() { public String getNotificationMessage() { StringBuilder message = new StringBuilder(); if (!status.error) { - message.append(String.format("New feed version created for %s (valid from %s - %s). ", - feedSource.name, - feedVersion.validationResult.firstCalendarDate, - feedVersion.validationResult.lastCalendarDate)); - if (feedVersion.validationResult.errorCount > 0) { - message.append(String.format("During validation, we found %s issue(s)", - feedVersion.validationResult.errorCount)); + ValidationResult validationResult = feedVersion.validationResult; + if (validationResult != null) { + message.append(String.format("New feed version created for %s (valid from %s - %s). ", + feedSource.name, + feedVersion.validationResult.firstCalendarDate, + feedVersion.validationResult.lastCalendarDate + )); } else { - message.append("The validation check found no issues with this new dataset!"); + message.append(String.format("New feed version created for %s.", feedSource.name)); + } + if (validationResult != null) { + if (validationResult.errorCount > 0) { + message.append(String.format("During validation, we found %s issue(s)", validationResult.errorCount)); + } else { + message.append("The validation check found no issues with this new dataset!"); + } } } else { // Processing did not complete. Depending on which sub-task this occurred in, diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index f62b4ed27..2442634b2 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -554,7 +554,9 @@ public void delete() { feedStore.deleteFeed(id); // Delete feed version tables in GTFS database - GTFS.delete(this.namespace, DataManager.GTFS_DATA_SOURCE); + if (this.namespace != null) { + GTFS.delete(this.namespace, DataManager.GTFS_DATA_SOURCE); + } LOG.info("Dropped version's GTFS tables from Postgres."); // Remove this FeedVersion from all Deployments associated with this FeedVersion's FeedSource's Project // TODO TEST THOROUGHLY THAT THIS UPDATE EXPRESSION IS CORRECT diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index fee9968c1..99ed35952 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -18,13 +18,16 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Date; -import java.util.List; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -75,17 +78,10 @@ public void tearDownTest() { * Test that a {@link NormalizeFieldTransformation} will successfully complete. * FIXME: On certain Windows machines, this test fails. */ - @Test - void canNormalizeField() throws IOException { - TransformationCase route = new TransformationCase("routes", "route_long_name", "Route", "Rte"); - TransformationCase bookingRules = new TransformationCase("booking_rules", "pickup_message", "message", "msg"); - TransformationCase areas = new TransformationCase("areas", "area_name", "area", "location"); - // Create transformations. - FeedSource feedSourceNormalize = initializeFeedSource(List.of( - createTransformation(route), - createTransformation(bookingRules), - createTransformation(areas) - )); + @ParameterizedTest + @MethodSource("createNormalizedFieldCases") + void canNormalizeField(TransformationCase transformationCase) throws IOException { + FeedSource feedSourceNormalize = initializeFeedSource(createTransformation(transformationCase)); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -95,12 +91,18 @@ void canNormalizeField() throws IOException { try (ZipFile zip = new ZipFile(targetVersion.retrieveGtfsFile())) { // Check that new version has expected modifications. - checkTableForModification(zip, route); - checkTableForModification(zip, bookingRules); - checkTableForModification(zip, areas); + checkTableForModification(zip, transformationCase); } } + private static Stream createNormalizedFieldCases() { + return Stream.of( + Arguments.of(new TransformationCase("routes", "route_long_name", "Route", "Rte")), + Arguments.of(new TransformationCase("booking_rules", "pickup_message", "message", "msg")), + Arguments.of(new TransformationCase("areas", "area_name", "area", "location")) + ); + } + private void checkTableForModification(ZipFile zip, TransformationCase transformationCase) throws IOException { // Check that the new version has been modified. ZipEntry entry = zip.getEntry(transformationCase.table + ".txt"); @@ -135,7 +137,7 @@ void canHandleInvalidSubstitutionPatterns() throws IOException { new Substitution("\\Cir\\b", "Circle") ) ); - FeedSource feedSource = initializeFeedSource(List.of(transformation)); + FeedSource feedSource = initializeFeedSource(transformation); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -163,14 +165,12 @@ private FeedTransformation createTransformation(Transfor /** * Create and persist a feed source using the given transformation. */ - private FeedSource initializeFeedSource(List> transformations) { + private FeedSource initializeFeedSource(FeedTransformation transformation) { // Create feed source with above transform. FeedSource feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); feedSource.deployable = false; - for (FeedTransformation transformation : transformations) { - feedSource.transformRules.add(new FeedTransformRules(transformation)); - } + feedSource.transformRules.add(new FeedTransformRules(transformation)); Persistence.feedSources.create(feedSource); return feedSource; } From ada194bc4e3db14ab01a259569cd16b6fda42f9e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 15 Aug 2023 08:39:32 +0100 Subject: [PATCH 071/133] refactor(NormalizeFieldTransformJobTest): Changed back to using a single feed source. --- .../jobs/NormalizeFieldTransformJobTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 99ed35952..ad0715b76 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -42,7 +42,7 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { private static final String FIELD_NAME = "route_long_name"; private static Project project; private FeedVersion targetVersion; - + FeedSource feedSource; /** * Initialize Data Tools and set up a simple feed source and project. @@ -81,11 +81,11 @@ public void tearDownTest() { @ParameterizedTest @MethodSource("createNormalizedFieldCases") void canNormalizeField(TransformationCase transformationCase) throws IOException { - FeedSource feedSourceNormalize = initializeFeedSource(createTransformation(transformationCase)); + initializeFeedSource(createTransformation(transformationCase)); // Create target version that the transform will operate on. targetVersion = createFeedVersion( - feedSourceNormalize, + feedSource, zipFolderFiles("fake-agency-for-field-normalizing") ); @@ -137,7 +137,7 @@ void canHandleInvalidSubstitutionPatterns() throws IOException { new Substitution("\\Cir\\b", "Circle") ) ); - FeedSource feedSource = initializeFeedSource(transformation); + initializeFeedSource(transformation); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -165,14 +165,13 @@ private FeedTransformation createTransformation(Transfor /** * Create and persist a feed source using the given transformation. */ - private FeedSource initializeFeedSource(FeedTransformation transformation) { + private void initializeFeedSource(FeedTransformation transformation) { // Create feed source with above transform. - FeedSource feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); + feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); feedSource.deployable = false; feedSource.transformRules.add(new FeedTransformRules(transformation)); Persistence.feedSources.create(feedSource); - return feedSource; } // FIXME: Refactor (almost same code as AutoDeployJobTest in PR #361, From fdac958ab1d9f69d4e60d1166b9da90e30a8c429 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 15 Aug 2023 10:49:44 +0100 Subject: [PATCH 072/133] refactor(NormalizeFieldTransformJobTest): Changed assertion. --- .../jobs/NormalizeFieldTransformJobTest.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index ad0715b76..2494103cf 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.jobs; import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -51,6 +52,7 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { public static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); // Create a project. project = createProject(); @@ -61,6 +63,7 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); // Project delete cascades to feed sources. project.delete(); } @@ -99,7 +102,7 @@ private static Stream createNormalizedFieldCases() { return Stream.of( Arguments.of(new TransformationCase("routes", "route_long_name", "Route", "Rte")), Arguments.of(new TransformationCase("booking_rules", "pickup_message", "message", "msg")), - Arguments.of(new TransformationCase("areas", "area_name", "area", "location")) + Arguments.of(new TransformationCase("areas", "area_name", "area", "place")) ); } @@ -117,10 +120,10 @@ private void checkTableForModification(ZipFile zip, TransformationCase transform String[] columns = reader.readLine().split(","); int fieldIndex = ArrayUtils.indexOf(columns, transformationCase.fieldName); - String row1 = reader.readLine(); - assertNotNull(row1, String.format("First row in table %s is null!", transformationCase.table)); - String[] row1Fields = row1.split(","); - assertTrue(row1Fields[fieldIndex].startsWith(transformationCase.replacement), row1); + String rowOne = reader.readLine(); + assertNotNull(rowOne, String.format("First row in table %s is null!", transformationCase.table)); + String[] row1Fields = rowOne.split(","); + assertTrue(row1Fields[fieldIndex].contains(transformationCase.replacement), rowOne); } } From d0f5421901f7d7b2d881561353a142deb65857f5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 15 Aug 2023 13:41:42 +0100 Subject: [PATCH 073/133] refactor(NormalizeFieldTransformJobTest): Added table name to feed source name --- .../jobs/NormalizeFieldTransformJobTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 2494103cf..8ad66e97f 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -22,6 +22,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.IOException; @@ -39,6 +41,7 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { + private static final Logger LOG = LoggerFactory.getLogger(NormalizeFieldTransformJobTest.class); private static final String TABLE_NAME = "routes"; private static final String FIELD_NAME = "route_long_name"; private static Project project; @@ -84,7 +87,7 @@ public void tearDownTest() { @ParameterizedTest @MethodSource("createNormalizedFieldCases") void canNormalizeField(TransformationCase transformationCase) throws IOException { - initializeFeedSource(createTransformation(transformationCase)); + initializeFeedSource(transformationCase.table, createTransformation(transformationCase)); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -107,8 +110,10 @@ private static Stream createNormalizedFieldCases() { } private void checkTableForModification(ZipFile zip, TransformationCase transformationCase) throws IOException { + String tableName = transformationCase.table + ".txt"; + LOG.info("Getting table {} from zip {}", tableName, zip.getName()); // Check that the new version has been modified. - ZipEntry entry = zip.getEntry(transformationCase.table + ".txt"); + ZipEntry entry = zip.getEntry(tableName); assertNotNull(entry); // Scan the first data row and check that the substitution that was defined in the set-up was done. @@ -140,7 +145,7 @@ void canHandleInvalidSubstitutionPatterns() throws IOException { new Substitution("\\Cir\\b", "Circle") ) ); - initializeFeedSource(transformation); + initializeFeedSource(TABLE_NAME, transformation); // Create target version that the transform will operate on. targetVersion = createFeedVersion( @@ -168,10 +173,10 @@ private FeedTransformation createTransformation(Transfor /** * Create and persist a feed source using the given transformation. */ - private void initializeFeedSource(FeedTransformation transformation) { + private void initializeFeedSource(String table, FeedTransformation transformation) { // Create feed source with above transform. - feedSource = new FeedSource("Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); + feedSource = new FeedSource(table + " Normalize Field Test Feed", project.id, FeedRetrievalMethod.MANUALLY_UPLOADED); feedSource.deployable = false; feedSource.transformRules.add(new FeedTransformRules(transformation)); Persistence.feedSources.create(feedSource); From f768a78bfcc43fef3f3c9303b14a17e1b20e395d Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 15 Aug 2023 15:35:37 +0100 Subject: [PATCH 074/133] refactor(NormalizeFieldTransformJobTest): Changed test data to take caps into consideration --- .../manager/jobs/NormalizeFieldTransformJobTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 8ad66e97f..494893b8b 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -104,8 +104,8 @@ void canNormalizeField(TransformationCase transformationCase) throws IOException private static Stream createNormalizedFieldCases() { return Stream.of( Arguments.of(new TransformationCase("routes", "route_long_name", "Route", "Rte")), - Arguments.of(new TransformationCase("booking_rules", "pickup_message", "message", "msg")), - Arguments.of(new TransformationCase("areas", "area_name", "area", "place")) + Arguments.of(new TransformationCase("booking_rules", "pickup_message", "Message", "Msg")), + Arguments.of(new TransformationCase("areas", "area_name", "Area", "Place")) ); } From 4e8fa0ca2c1535c724fdd72ffaab3ff868b50259 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 07:53:07 +0100 Subject: [PATCH 075/133] refactor(pom.xml): Bumped gtfs-lib version to stop time normalization refactor --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 51333f166..490c8cfb7 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 5b4eec6 + 3c5fd3076a4d0eb31f9bdb3ce791aaf37339ff65 From 8b8ddbbae07719595e8bb472b164f30f96d28f9f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 10:06:44 +0100 Subject: [PATCH 076/133] refactor(pom.xml): Bumped gtfs lib to include stop times normalization refactor with less logging --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 490c8cfb7..84c4a8996 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 3c5fd3076a4d0eb31f9bdb3ce791aaf37339ff65 + 46df240855726753ff9ae58e205b566198c3490a From f503fae58ec2774fb3add2657782a101101cd4ff Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 16 Aug 2023 14:25:07 +0100 Subject: [PATCH 077/133] refactor(NormalizeFieldTransformJobTest): Restored correct feed source declaration --- .../datatools/manager/jobs/NormalizeFieldTransformJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 494893b8b..9264a3daa 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -45,8 +45,8 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { private static final String TABLE_NAME = "routes"; private static final String FIELD_NAME = "route_long_name"; private static Project project; + private static FeedSource feedSource; private FeedVersion targetVersion; - FeedSource feedSource; /** * Initialize Data Tools and set up a simple feed source and project. From 34545cb32b1ab4a2e0c477961662ad2cc500a58a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 21 Aug 2023 16:48:28 +0100 Subject: [PATCH 078/133] refactor(Bumped GTFS-lib version): Version now matches the latest dev-flex commit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 84c4a8996..d64efc580 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ com.github.conveyal gtfs-lib - 46df240855726753ff9ae58e205b566198c3490a + 32c084009930501ce992695be73a83ae7f7f767e From e895aca6bece4275afb907217756f40eaa37f9fb Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 1 Sep 2023 16:04:19 +0100 Subject: [PATCH 079/133] Update src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java Co-authored-by: Philip Cline <63798641+philip-cline@users.noreply.github.com> --- .../conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index b344b6608..795cb967f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -203,8 +203,8 @@ public String getNotificationMessage() { if (validationResult != null) { message.append(String.format("New feed version created for %s (valid from %s - %s). ", feedSource.name, - feedVersion.validationResult.firstCalendarDate, - feedVersion.validationResult.lastCalendarDate + validationResult.firstCalendarDate, + validationResult.lastCalendarDate )); } else { message.append(String.format("New feed version created for %s.", feedSource.name)); From ae95fb005bdc41ca3461c721dd77d2ce5afd8e48 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 1 Sep 2023 16:05:54 +0100 Subject: [PATCH 080/133] refactor(ProcessSingleFeedJob): --- .../conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 795cb967f..697d7ba89 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -201,7 +201,7 @@ public String getNotificationMessage() { if (!status.error) { ValidationResult validationResult = feedVersion.validationResult; if (validationResult != null) { - message.append(String.format("New feed version created for %s (valid from %s - %s). ", + message.append(String.format("New feed version created for %s (valid from %s - %s).", feedSource.name, validationResult.firstCalendarDate, validationResult.lastCalendarDate From 738abc27a7c2cfe4a94f8ba7182790aed76af5c4 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 6 Sep 2023 10:25:47 +0100 Subject: [PATCH 081/133] refactor(Reverted grammer changes to unit tests): Notification message doesn't provide spaces betwee --- .../manager/jobs/ProcessSingleFeedJobNotificationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java index 036cbe543..0b623e2d8 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java @@ -60,7 +60,7 @@ public void canNotifySubscribersOfErrors() { validationResult.errorCount = 100; f.validationResult = validationResult; ProcessSingleFeedJob processSingleFeedJob = new ProcessSingleFeedJob(f, user, true); - String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s). During validation, we found 100 issue(s)", + String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s).During validation, we found 100 issue(s)", validationResult.firstCalendarDate, validationResult.lastCalendarDate); assertThat( @@ -75,7 +75,7 @@ public void canNotifySubscribersOfSuccess() { validationResult.errorCount = 0; f.validationResult = validationResult; ProcessSingleFeedJob processSingleFeedJob = new ProcessSingleFeedJob(f, user, true); - String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s). The validation check found no issues with this new dataset!", + String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s).The validation check found no issues with this new dataset!", validationResult.firstCalendarDate, validationResult.lastCalendarDate); assertThat( From d5b34365450f5e6688e2b6f51671508e33220512 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 24 Oct 2023 09:14:11 +0100 Subject: [PATCH 082/133] refactor(Initial work to merge flex feeds): --- pom.xml | 18 +- .../manager/jobs/FlexMergeFeedsJobTest.java | 163 ++++++++++++++ .../agency.txt | 2 + .../fake-agency-with-flex-version-1/areas.txt | 4 + .../attributions.txt | 2 + .../booking_rules.txt | 2 + .../calendar.txt | 2 + .../calendar_dates.txt | 2 + .../fare_attributes.txt | 2 + .../fare_rules.txt | 2 + .../feed_info.txt | 2 + .../frequencies.txt | 2 + .../locations.geojson | 201 ++++++++++++++++++ .../routes.txt | 2 + .../shapes.txt | 8 + .../stop_areas.txt | 4 + .../stop_times.txt | 7 + .../fake-agency-with-flex-version-1/stops.txt | 6 + .../transfers.txt | 1 + .../translations.txt | 2 + .../fake-agency-with-flex-version-1/trips.txt | 3 + .../agency.txt | 2 + .../fake-agency-with-flex-version-2/areas.txt | 4 + .../attributions.txt | 2 + .../booking_rules.txt | 2 + .../calendar.txt | 2 + .../calendar_dates.txt | 2 + .../fare_attributes.txt | 2 + .../fare_rules.txt | 2 + .../feed_info.txt | 2 + .../frequencies.txt | 2 + .../locations.geojson | 201 ++++++++++++++++++ .../routes.txt | 2 + .../shapes.txt | 8 + .../stop_areas.txt | 4 + .../stop_times.txt | 7 + .../fake-agency-with-flex-version-2/stops.txt | 6 + .../transfers.txt | 1 + .../translations.txt | 2 + .../fake-agency-with-flex-version-2/trips.txt | 3 + 40 files changed, 691 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/agency.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar_dates.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_attributes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/frequencies.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/locations.geojson create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/shapes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/trips.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/agency.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar_dates.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_attributes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/frequencies.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/locations.geojson create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/shapes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/trips.txt diff --git a/pom.xml b/pom.xml index 782449668..0d76ef3b6 100644 --- a/pom.xml +++ b/pom.xml @@ -273,9 +273,9 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.github.conveyal + com.conveyal gtfs-lib - 5c383ab + 7.0.2 @@ -286,6 +286,20 @@ + + + + + + + + + + + + + + diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java new file mode 100644 index 000000000..2623a387c --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -0,0 +1,163 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.TestUtils; +import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0Connection; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; +import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Date; +import java.util.HashSet; +import java.util.Set; + +import static com.conveyal.datatools.TestUtils.assertThatFeedHasNoErrorsOfType; +import static com.conveyal.datatools.TestUtils.createFeedVersion; +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.TestUtils.zipFolderFiles; +import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Tests for the various {@link MergeFeedsJob} merge types. + */ +public class FlexMergeFeedsJobTest extends UnitTest { + private static final Logger LOG = LoggerFactory.getLogger(FlexMergeFeedsJobTest.class); + private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); + private static FeedVersion fakeAgencyWithFlexVersion1; + private static FeedVersion fakeAgencyWithFlexVersion2; + private static Project project; + + /** + * Prepare and start a testing-specific web server + */ + @BeforeAll + public static void setUp() throws IOException { + // start server if it isn't already running + DatatoolsTest.setUp(); + Auth0Connection.setAuthDisabled(true); + + // Create a project, feed sources, and feed versions to merge. + project = new Project(); + project.name = String.format("Test %s", new Date()); + Persistence.projects.create(project); + + FeedSource flexAgencyA = new FeedSource("FLEX-AGENCY-A", project.id, MANUALLY_UPLOADED); + Persistence.feedSources.create(flexAgencyA); + fakeAgencyWithFlexVersion1 = TestUtils.createFeedVersion(flexAgencyA, TestUtils.zipFolderFiles("fake-agency-with-flex-version-1")); + + FeedSource flexAgencyB = new FeedSource("FLEX-AGENCY-B", project.id, MANUALLY_UPLOADED); + Persistence.feedSources.create(flexAgencyB); + fakeAgencyWithFlexVersion2 = TestUtils.createFeedVersion(flexAgencyB, TestUtils.zipFolderFiles("fake-agency-with-flex-version-2")); + } + + /** + * Delete project on tear down (feed sources/versions will also be deleted). + */ + @AfterAll + public static void tearDown() { + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); + if (project != null) { + project.delete(); + } + } + + /** + * Ensures that a regional feed merge will produce a feed that includes all entities from each feed. + */ + @Test + void canMergeRegional() throws SQLException { + // Set up list of feed versions to merge. + Set versions = new HashSet<>(); + versions.add(fakeAgencyWithFlexVersion1); + versions.add(fakeAgencyWithFlexVersion2); + FeedVersion mergedVersion = regionallyMergeVersions(versions); + + // Ensure the feed has the row counts we expect. + assertEquals( + fakeAgencyWithFlexVersion1.feedLoadResult.trips.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.trips.rowCount, + mergedVersion.feedLoadResult.trips.rowCount, + "trips count for merged feed should equal sum of trips for versions merged." + ); + assertEquals( + fakeAgencyWithFlexVersion1.feedLoadResult.routes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.routes.rowCount, + mergedVersion.feedLoadResult.routes.rowCount, + "routes count for merged feed should equal sum of routes for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.stops.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.stops.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stops.rowCount, + "stops count for merged feed should equal sum of stops for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.agency.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.agency.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.agency.rowCount, + "agency count for merged feed should equal sum of agency for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.stopTimes.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.stopTimes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopTimes.rowCount, + "stopTimes count for merged feed should equal sum of stopTimes for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.calendar.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.calendar.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.calendar.rowCount, + "calendar count for merged feed should equal sum of calendar for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.calendarDates.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.calendarDates.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.calendarDates.rowCount, + "calendarDates count for merged feed should equal sum of calendarDates for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.area.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.area.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.area.rowCount, + "area count for merged feed should equal sum of area for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.bookingRules.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.bookingRules.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.bookingRules.rowCount, + "bookingRules count for merged feed should equal sum of area for versions merged." + ); + assertEquals( + mergedVersion.feedLoadResult.stopAreas.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.stopAreas.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopAreas.rowCount, + "stopAreas count for merged feed should equal sum of area for versions merged." + ); + // TODO: locations.geojson. + // Ensure there are no referential integrity errors, duplicate ID, or wrong number of + // fields errors. + assertThatFeedHasNoErrorsOfType( + mergedVersion.namespace, + NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), + NewGTFSErrorType.DUPLICATE_ID.toString(), + NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS.toString() + ); + } + + /** + * Merges a set of FeedVersions and then creates a new FeedSource and FeedVersion of the merged feed. + */ + private FeedVersion regionallyMergeVersions(Set versions) { + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, project.id, MergeFeedsType.REGIONAL); + // Run the job in this thread (we're not concerned about concurrency here). + mergeFeedsJob.run(); + LOG.info("Regional merged file: {}", mergeFeedsJob.mergedVersion.retrieveGtfsFile().getAbsolutePath()); + return mergeFeedsJob.mergedVersion; + } +} diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/agency.txt new file mode 100644 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/areas.txt new file mode 100644 index 000000000..3f8aefce7 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/areas.txt @@ -0,0 +1,4 @@ +area_id,area_name +1,"Area referencing a stop" +2,"Area referencing a location" +3,"Multi areas" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt new file mode 100644 index 000000000..1d77168ac --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt @@ -0,0 +1,2 @@ +attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone +1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt new file mode 100644 index 000000000..a0e5dd79d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt @@ -0,0 +1,2 @@ +booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url +Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://www.info_url.com","http://www.booking_url.com" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar.txt new file mode 100644 index 000000000..4c9b7fd13 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar.txt @@ -0,0 +1,2 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +04100312-8fe1-46a5-a9f2-556f39478f57,1,1,1,1,1,1,1,20170915,20170917 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar_dates.txt new file mode 100644 index 000000000..403ee2bbe --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/calendar_dates.txt @@ -0,0 +1,2 @@ +service_id,date,exception_type +04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_attributes.txt new file mode 100644 index 000000000..3173d016d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_attributes.txt @@ -0,0 +1,2 @@ +fare_id,price,currency_type,payment_method,transfers,transfer_duration +route_based_fare,1.23,USD,0,0,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_rules.txt new file mode 100644 index 000000000..05d2aadf8 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/fare_rules.txt @@ -0,0 +1,2 @@ +fare_id,route_id,origin_id,destination_id,contains_id +route_based_fare,1,,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/frequencies.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/frequencies.txt new file mode 100644 index 000000000..9baceff3a --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/frequencies.txt @@ -0,0 +1,2 @@ +trip_id,start_time,end_time,headway_secs,exact_times +frequency-trip,08:00:00,09:00:00,1800,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/locations.geojson b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/locations.geojson new file mode 100644 index 000000000..efae1fe8c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/locations.geojson @@ -0,0 +1,201 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "id": "area_1469-2", + "properties": { + "stop_name": "Templeboy to Ballisodare", + "stop_desc": "Templeboy to Ballisodare Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [[40.0, 40.0], [20.0, 45.0], [45.0, 30.0], [40.0, 40.0]] + ], + [ + [[20.0, 35.0], [10.0, 30.0], [10.0, 10.0], [30.0, 5.0], [45.0, 20.0], [20.0, 35.0]], + [[30.0, 20.0], [20.0, 15.0], [20.0, 25.0], [30.0, 20.0]] + ] + ] + } + }, + { + "id": "area_1469-1", + "properties": { + "stop_name": "Dromore West to Templeboy", + "stop_desc": "Dromore West to Templeboy Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.810391429999981, + 54.255823500000076 + ], + [ + -8.82575511999994, + 54.252263440000036 + ], + [ + -8.832192419999956, + 54.25331645000006 + ], + [ + -8.835968969999954, + 54.25456999000005 + ], + [ + -8.841462139999976, + 54.25472041000006 + ], + [ + -8.846955299999934, + 54.257578340000066 + ], + [ + -8.854250909999962, + 54.26148887000005 + ], + [ + -8.855624199999966, + 54.26359438000003 + ], + [ + -8.860516549999943, + 54.26173953000006 + ], + [ + -8.865494729999966, + 54.261789660000034 + ], + [ + -8.86849879999994, + 54.26414581000006 + ], + [ + -8.871159549999959, + 54.26569979000004 + ], + [ + -8.87991427999998, + 54.26700308000005 + ], + [ + -8.882403369999963, + 54.26785521000005 + ], + [ + -8.890128139999945, + 54.27041150000008 + ], + [ + -8.89776706999993, + 54.27326834000007 + ], + [ + -8.905320169999982, + 54.26194006000003 + ], + [ + -8.904075619999958, + 54.25258586000007 + ], + [ + -8.922486309999954, + 54.24664695000007 + ], + [ + -8.930468559999952, + 54.24223346000008 + ], + [ + -8.935532569999964, + 54.23235155000003 + ], + [ + -8.919868469999926, + 54.22369314000008 + ], + [ + -8.913688659999934, + 54.21887572000003 + ], + [ + -8.902015689999928, + 54.21566380000007 + ], + [ + -8.893775939999955, + 54.21486078000004 + ], + [ + -8.87729644999996, + 54.212853160000066 + ], + [ + -8.85807036999995, + 54.212853160000066 + ], + [ + -8.838844299999948, + 54.216868300000044 + ], + [ + -8.827171329999942, + 54.21807277000005 + ], + [ + -8.82305144999998, + 54.21847425000004 + ], + [ + -8.822305720999964, + 54.218947282000045 + ], + [ + -8.808914424999955, + 54.21954913800005 + ], + [ + -8.810391429999981, + 54.255823500000076 + ] + ] + ] + } + }, + { + "id": "area_246", + "properties": { + "stop_name": "Moville to Letterkenny", + "stop_desc": "Moville to Letterkenny Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [ + [[10.0, 10.0], [20.0, 20.0], [10.0, 40.0]], + [[40.0, 40.0], [30.0, 30.0], [40.0, 20.0], [30.0, 10.0]] + ] + } + }, + { + "id": "area_251", + "properties": { + "stop_name": "Muff to Moville", + "stop_desc": "Muff to Moville Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [ + [30.0, 10.0], [10.0, 30.0], [40.0, 40.0] + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/routes.txt new file mode 100644 index 000000000..35ea7aa67 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/routes.txt @@ -0,0 +1,2 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/shapes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/shapes.txt new file mode 100644 index 000000000..3f2e3fd13 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/shapes.txt @@ -0,0 +1,8 @@ +shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled +5820f377-f947-4728-ac29-ac0102cbc34e,37.0612132,-122.0074332,1,0.0000000 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0611720,-122.0075000,2,7.4997067 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0613590,-122.0076830,3,33.8739075 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0608780,-122.0082780,4,109.0402932 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0603590,-122.0088280,5,184.6078298 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0597610,-122.0093540,6,265.8053023 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0590660,-122.0099190,7,357.8617018 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_areas.txt new file mode 100644 index 000000000..3f900c131 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_areas.txt @@ -0,0 +1,4 @@ +area_id,stop_id +1,123 +2,area_1469-1 +3,123 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt new file mode 100644 index 000000000..f82e627d8 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt @@ -0,0 +1,7 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt new file mode 100644 index 000000000..49e996baa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt new file mode 100644 index 000000000..357103c47 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt @@ -0,0 +1 @@ +from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt new file mode 100644 index 000000000..da8fe2407 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt @@ -0,0 +1,2 @@ +table_name,field_name,language,translation,record_id,record_sub_id,field_value +stops,stop_desc,FR,en direction du nord,4u6g,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/trips.txt new file mode 100644 index 000000000..eab14b86d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,a30277f8-e50a-4a85-9141-b1e0da9d429d,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/agency.txt new file mode 100644 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/areas.txt new file mode 100644 index 000000000..3f8aefce7 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/areas.txt @@ -0,0 +1,4 @@ +area_id,area_name +1,"Area referencing a stop" +2,"Area referencing a location" +3,"Multi areas" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt new file mode 100644 index 000000000..1d77168ac --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt @@ -0,0 +1,2 @@ +attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone +1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt new file mode 100644 index 000000000..a0e5dd79d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt @@ -0,0 +1,2 @@ +booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url +Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://www.info_url.com","http://www.booking_url.com" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar.txt new file mode 100644 index 000000000..4c9b7fd13 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar.txt @@ -0,0 +1,2 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +04100312-8fe1-46a5-a9f2-556f39478f57,1,1,1,1,1,1,1,20170915,20170917 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar_dates.txt new file mode 100644 index 000000000..403ee2bbe --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/calendar_dates.txt @@ -0,0 +1,2 @@ +service_id,date,exception_type +04100312-8fe1-46a5-a9f2-556f39478f57,20170916,2 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_attributes.txt new file mode 100644 index 000000000..3173d016d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_attributes.txt @@ -0,0 +1,2 @@ +fare_id,price,currency_type,payment_method,transfers,transfer_duration +route_based_fare,1.23,USD,0,0,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_rules.txt new file mode 100644 index 000000000..05d2aadf8 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/fare_rules.txt @@ -0,0 +1,2 @@ +fare_id,route_id,origin_id,destination_id,contains_id +route_based_fare,1,,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/frequencies.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/frequencies.txt new file mode 100644 index 000000000..9baceff3a --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/frequencies.txt @@ -0,0 +1,2 @@ +trip_id,start_time,end_time,headway_secs,exact_times +frequency-trip,08:00:00,09:00:00,1800,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/locations.geojson b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/locations.geojson new file mode 100644 index 000000000..efae1fe8c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/locations.geojson @@ -0,0 +1,201 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "id": "area_1469-2", + "properties": { + "stop_name": "Templeboy to Ballisodare", + "stop_desc": "Templeboy to Ballisodare Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [[40.0, 40.0], [20.0, 45.0], [45.0, 30.0], [40.0, 40.0]] + ], + [ + [[20.0, 35.0], [10.0, 30.0], [10.0, 10.0], [30.0, 5.0], [45.0, 20.0], [20.0, 35.0]], + [[30.0, 20.0], [20.0, 15.0], [20.0, 25.0], [30.0, 20.0]] + ] + ] + } + }, + { + "id": "area_1469-1", + "properties": { + "stop_name": "Dromore West to Templeboy", + "stop_desc": "Dromore West to Templeboy Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.810391429999981, + 54.255823500000076 + ], + [ + -8.82575511999994, + 54.252263440000036 + ], + [ + -8.832192419999956, + 54.25331645000006 + ], + [ + -8.835968969999954, + 54.25456999000005 + ], + [ + -8.841462139999976, + 54.25472041000006 + ], + [ + -8.846955299999934, + 54.257578340000066 + ], + [ + -8.854250909999962, + 54.26148887000005 + ], + [ + -8.855624199999966, + 54.26359438000003 + ], + [ + -8.860516549999943, + 54.26173953000006 + ], + [ + -8.865494729999966, + 54.261789660000034 + ], + [ + -8.86849879999994, + 54.26414581000006 + ], + [ + -8.871159549999959, + 54.26569979000004 + ], + [ + -8.87991427999998, + 54.26700308000005 + ], + [ + -8.882403369999963, + 54.26785521000005 + ], + [ + -8.890128139999945, + 54.27041150000008 + ], + [ + -8.89776706999993, + 54.27326834000007 + ], + [ + -8.905320169999982, + 54.26194006000003 + ], + [ + -8.904075619999958, + 54.25258586000007 + ], + [ + -8.922486309999954, + 54.24664695000007 + ], + [ + -8.930468559999952, + 54.24223346000008 + ], + [ + -8.935532569999964, + 54.23235155000003 + ], + [ + -8.919868469999926, + 54.22369314000008 + ], + [ + -8.913688659999934, + 54.21887572000003 + ], + [ + -8.902015689999928, + 54.21566380000007 + ], + [ + -8.893775939999955, + 54.21486078000004 + ], + [ + -8.87729644999996, + 54.212853160000066 + ], + [ + -8.85807036999995, + 54.212853160000066 + ], + [ + -8.838844299999948, + 54.216868300000044 + ], + [ + -8.827171329999942, + 54.21807277000005 + ], + [ + -8.82305144999998, + 54.21847425000004 + ], + [ + -8.822305720999964, + 54.218947282000045 + ], + [ + -8.808914424999955, + 54.21954913800005 + ], + [ + -8.810391429999981, + 54.255823500000076 + ] + ] + ] + } + }, + { + "id": "area_246", + "properties": { + "stop_name": "Moville to Letterkenny", + "stop_desc": "Moville to Letterkenny Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "MultiLineString", + "coordinates": [ + [[10.0, 10.0], [20.0, 20.0], [10.0, 40.0]], + [[40.0, 40.0], [30.0, 30.0], [40.0, 20.0], [30.0, 10.0]] + ] + } + }, + { + "id": "area_251", + "properties": { + "stop_name": "Muff to Moville", + "stop_desc": "Muff to Moville Door-to-door pickup area" + }, + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": [ + [30.0, 10.0], [10.0, 30.0], [40.0, 40.0] + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/routes.txt new file mode 100644 index 000000000..35ea7aa67 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/routes.txt @@ -0,0 +1,2 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/shapes.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/shapes.txt new file mode 100644 index 000000000..3f2e3fd13 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/shapes.txt @@ -0,0 +1,8 @@ +shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled +5820f377-f947-4728-ac29-ac0102cbc34e,37.0612132,-122.0074332,1,0.0000000 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0611720,-122.0075000,2,7.4997067 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0613590,-122.0076830,3,33.8739075 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0608780,-122.0082780,4,109.0402932 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0603590,-122.0088280,5,184.6078298 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0597610,-122.0093540,6,265.8053023 +5820f377-f947-4728-ac29-ac0102cbc34e,37.0590660,-122.0099190,7,357.8617018 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_areas.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_areas.txt new file mode 100644 index 000000000..3f900c131 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_areas.txt @@ -0,0 +1,4 @@ +area_id,stop_id +1,123 +2,area_1469-1 +3,123 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt new file mode 100644 index 000000000..f82e627d8 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt @@ -0,0 +1,7 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt new file mode 100644 index 000000000..49e996baa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt new file mode 100644 index 000000000..357103c47 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt @@ -0,0 +1 @@ +from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt new file mode 100644 index 000000000..da8fe2407 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt @@ -0,0 +1,2 @@ +table_name,field_name,language,translation,record_id,record_sub_id,field_value +stops,stop_desc,FR,en direction du nord,4u6g,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/trips.txt new file mode 100644 index 000000000..eab14b86d --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,a30277f8-e50a-4a85-9141-b1e0da9d429d,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 +1,frequency-trip,,,0,,5820f377-f947-4728-ac29-ac0102cbc34e,0,0,04100312-8fe1-46a5-a9f2-556f39478f57 \ No newline at end of file From f0890c79a69e13eca63132cb236e9c0ba02d0099 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 25 Oct 2023 08:17:40 +0100 Subject: [PATCH 083/133] refactor(Addressed PR feedback): --- .../datatools/manager/jobs/ProcessSingleFeedJob.java | 10 ++++------ .../models/transform/NormalizeFieldTransformation.java | 2 +- .../manager/jobs/NormalizeFieldTransformJobTest.java | 6 ++---- .../jobs/ProcessSingleFeedJobNotificationTest.java | 4 ++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index 697d7ba89..63bbcfedb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -206,15 +206,13 @@ public String getNotificationMessage() { validationResult.firstCalendarDate, validationResult.lastCalendarDate )); - } else { - message.append(String.format("New feed version created for %s.", feedSource.name)); - } - if (validationResult != null) { if (validationResult.errorCount > 0) { - message.append(String.format("During validation, we found %s issue(s)", validationResult.errorCount)); + message.append(String.format(" During validation, we found %s issue(s)", validationResult.errorCount)); } else { - message.append("The validation check found no issues with this new dataset!"); + message.append(" The validation check found no issues with this new dataset!"); } + } else { + message.append(String.format("New feed version created for %s.", feedSource.name)); } } else { // Processing did not complete. Depending on which sub-task this occurred in, diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java index ccf7b94d5..f06840ed8 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/NormalizeFieldTransformation.java @@ -184,7 +184,7 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // is here in case that changes. throw new UnsupportedOperationException("It is not possible to normalize geo json fields."); } - String tableName = table + ".txt"; + String tableName = getTableName(); try( // Hold output before writing to ZIP StringWriter stringWriter = new StringWriter(); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java index 9264a3daa..85f8f3599 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/NormalizeFieldTransformJobTest.java @@ -42,8 +42,6 @@ public class NormalizeFieldTransformJobTest extends DatatoolsTest { private static final Logger LOG = LoggerFactory.getLogger(NormalizeFieldTransformJobTest.class); - private static final String TABLE_NAME = "routes"; - private static final String FIELD_NAME = "route_long_name"; private static Project project; private static FeedSource feedSource; private FeedVersion targetVersion; @@ -141,11 +139,11 @@ void canHandleInvalidSubstitutionPatterns() throws IOException { // In this test, we provide an invalid pattern '\Cir\b' (instead of '\bCir\b'), // when trying to replace e.g. 'Piedmont Cir' with 'Piedmont Circle'. FeedTransformation transformation = NormalizeFieldTransformationTest.createTransformation( - TABLE_NAME, FIELD_NAME, null, Lists.newArrayList( + "routes", "route_long_name", null, Lists.newArrayList( new Substitution("\\Cir\\b", "Circle") ) ); - initializeFeedSource(TABLE_NAME, transformation); + initializeFeedSource("routes", transformation); // Create target version that the transform will operate on. targetVersion = createFeedVersion( diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java index 0b623e2d8..036cbe543 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJobNotificationTest.java @@ -60,7 +60,7 @@ public void canNotifySubscribersOfErrors() { validationResult.errorCount = 100; f.validationResult = validationResult; ProcessSingleFeedJob processSingleFeedJob = new ProcessSingleFeedJob(f, user, true); - String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s).During validation, we found 100 issue(s)", + String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s). During validation, we found 100 issue(s)", validationResult.firstCalendarDate, validationResult.lastCalendarDate); assertThat( @@ -75,7 +75,7 @@ public void canNotifySubscribersOfSuccess() { validationResult.errorCount = 0; f.validationResult = validationResult; ProcessSingleFeedJob processSingleFeedJob = new ProcessSingleFeedJob(f, user, true); - String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s).The validation check found no issues with this new dataset!", + String expectedResult = String.format("New feed version created for Mock Feed Source (valid from %s - %s). The validation check found no issues with this new dataset!", validationResult.firstCalendarDate, validationResult.lastCalendarDate); assertThat( From a98d0564fa7855df860b9e101d49354a10f11305 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 25 Oct 2023 11:58:30 +0100 Subject: [PATCH 084/133] refactor(pom.xml): Updated gtfs-lib version to that of dev-flex --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d9a2dabee..a608cb4de 100644 --- a/pom.xml +++ b/pom.xml @@ -267,7 +267,7 @@ com.github.conveyal gtfs-lib - 5c383ab + 2d7d77c562557ab442201a54f67d4a7d5d703602 From 42fa3a10a485370df7cb6511cef67cb7def27c3e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 3 Nov 2023 16:05:20 +0000 Subject: [PATCH 085/133] refactor(Added geojson merging): --- .../datatools/manager/jobs/MergeFeedsJob.java | 36 +++++++++++++++++-- .../datatools/manager/models/FeedVersion.java | 9 ++--- .../manager/jobs/FlexMergeFeedsJobTest.java | 13 +++++-- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index ab683b090..101223b00 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -18,8 +18,12 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.ErrorUtils; import com.conveyal.gtfs.loader.Feed; +import com.conveyal.gtfs.loader.JdbcGtfsExporter; import com.conveyal.gtfs.loader.Table; +import com.conveyal.gtfs.model.Location; +import com.conveyal.gtfs.model.LocationShape; import com.conveyal.gtfs.model.StopTime; +import com.conveyal.gtfs.util.GeoJsonUtil; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; @@ -38,13 +42,16 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.*; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergedVersion; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.stopTimesMatchSimplified; +import static com.conveyal.gtfs.loader.Table.LOCATION_GEO_JSON_FILE_NAME; /** * This job handles merging two or more feed versions according to logic specific to the specified merge type. @@ -219,13 +226,14 @@ public void jobLogic() { LOG.error("Merge {} table failed!", table.name); } } + mergeLocations(out, feedMergeContext.feedsToMerge); } catch (IOException e) { String message = "Error creating output stream for feed merge."; logAndReportToBugsnag(e, message); status.fail(message, e); } finally { try { - feedMergeContext.close(); + if (feedMergeContext != null) feedMergeContext.close(); } catch (IOException e) { logAndReportToBugsnag(e, "Error closing FeedMergeContext object"); } @@ -245,6 +253,26 @@ public void jobLogic() { } } + /** + * Merge locations.geojson files. These files are not compatible with the CSV merging strategy. Instead, the + * location.geojson file is flattened into locations and locations shapes. These are then 'merged' based on the + * equal/hash values of each object, converted back into geojson and written to the zip file. + */ + void mergeLocations(ZipOutputStream out, List feedsToMerge) throws IOException { + Set locations = new HashSet<>(); + Set locationShapes = new HashSet<>(); + for (FeedToMerge feed : feedsToMerge) { + ZipEntry locationGeoJsonFile = feed.zipFile.getEntry(LOCATION_GEO_JSON_FILE_NAME); + if (locationGeoJsonFile != null) { + locations.addAll(GeoJsonUtil.getLocationsFromGeoJson(feed.zipFile, locationGeoJsonFile, null)); + locationShapes.addAll(GeoJsonUtil.getLocationShapesFromGeoJson(feed.zipFile, locationGeoJsonFile, null)); + } + } + if (!locations.isEmpty()) { + JdbcGtfsExporter.writeLocationsToFile(out, new ArrayList<>(locations), new ArrayList<>(locationShapes)); + } + } + /** * Obtains trip ids whose entries in the stop_times table differ between the active and future feed. */ @@ -281,6 +309,10 @@ private boolean shouldSkipTable(String tableName) { LOG.warn("Skipping editor-only table {}.", tableName); return true; } + if (tableName.equals(Table.LOCATIONS.name) || tableName.equals(Table.LOCATION_SHAPES.name)) { + LOG.warn("{} detected. Skipping traditional merge in favour of bespoke merge.", LOCATION_GEO_JSON_FILE_NAME); + return true; + } return false; } diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index a69250cd0..051bd6fc6 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -422,7 +422,6 @@ public void validateMobility(MonitorableJob.Status status) { // Wait for the file to be entirely copied into the directory. // 5 seconds + ~1 second per 10mb Thread.sleep(5000 + (this.fileSize / 10000)); - File gtfsZip = this.retrieveGtfsFile(); // Namespace based folders avoid clash for validation being run on multiple versions of a feed. // TODO: do we know that there will always be a namespace? String validatorOutputDirectory = "/tmp/datatools_gtfs/" + this.namespace + "/"; @@ -430,7 +429,7 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 20); // Set up MobilityData validator. ValidationRunnerConfig.Builder builder = ValidationRunnerConfig.builder(); - builder.setGtfsSource(gtfsZip.toURI()); + builder.setGtfsSource(this.retrieveGtfsFile().toURI()); builder.setOutputDirectory(Path.of(validatorOutputDirectory)); ValidationRunnerConfig mbValidatorConfig = builder.build(); @@ -442,8 +441,10 @@ public void validateMobility(MonitorableJob.Status status) { status.update("MobilityData Analysis...", 80); // Read generated report and save to Mongo. String json; - try (FileReader fr = new FileReader(validatorOutputDirectory + "report.json")) { - BufferedReader in = new BufferedReader(fr); + try ( + FileReader fr = new FileReader(validatorOutputDirectory + "report.json"); + BufferedReader in = new BufferedReader(fr) + ) { json = in.lines().collect(Collectors.joining(System.lineSeparator())); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index 2623a387c..a7c894513 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -71,10 +71,10 @@ public static void setUp() throws IOException { */ @AfterAll public static void tearDown() { - Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); if (project != null) { project.delete(); } + Auth0Connection.setAuthDisabled(Auth0Connection.getDefaultAuthDisabled()); } /** @@ -139,7 +139,16 @@ void canMergeRegional() throws SQLException { fakeAgencyWithFlexVersion1.feedLoadResult.stopAreas.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopAreas.rowCount, "stopAreas count for merged feed should equal sum of area for versions merged." ); - // TODO: locations.geojson. + assertEquals( + mergedVersion.feedLoadResult.locations.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.locations.rowCount, + "Merged versions contain the same locations, only one set of locations should remain after merge." + ); + assertEquals( + mergedVersion.feedLoadResult.locationShapes.rowCount, + fakeAgencyWithFlexVersion1.feedLoadResult.locationShapes.rowCount, + "Merged versions contain the same location shapes, only one set of location shapes should remain after merge." + ); // Ensure there are no referential integrity errors, duplicate ID, or wrong number of // fields errors. assertThatFeedHasNoErrorsOfType( From 93c3a3abf7aac462bcb4344a6e39c6172dea5cb5 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 6 Nov 2023 16:25:29 +0000 Subject: [PATCH 086/133] refactor(Corrected gtfs-lib reference): --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9ba684126..c09a0f342 100644 --- a/pom.xml +++ b/pom.xml @@ -265,9 +265,10 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.conveyal + com.github.conveyal gtfs-lib - 7.0.2 + + 87c7a3d53448dce2ce87465d66e001f7215edb3b From c809e83be7286e11bd0e96aaa798af378ba875d2 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 7 Nov 2023 08:19:06 +0000 Subject: [PATCH 087/133] refactor(Updated location merging and removed flex items from stop times from test feeds): flex item --- .../datatools/manager/jobs/MergeFeedsJob.java | 26 +++++++++++++------ .../manager/jobs/FlexMergeFeedsJobTest.java | 12 ++++----- .../stop_times.txt | 14 +++++----- .../stop_times.txt | 14 +++++----- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 101223b00..742dacbe8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -51,6 +51,7 @@ import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergedVersion; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.stopTimesMatchSimplified; +import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.conveyal.gtfs.loader.Table.LOCATION_GEO_JSON_FILE_NAME; /** @@ -255,21 +256,30 @@ public void jobLogic() { /** * Merge locations.geojson files. These files are not compatible with the CSV merging strategy. Instead, the - * location.geojson file is flattened into locations and locations shapes. These are then 'merged' based on the - * equal/hash values of each object, converted back into geojson and written to the zip file. + * location.geojson file is flattened into locations and locations shapes. The location id is then updated with the + * scope id to keep feed locations unique, converted back into geojson and written to the zip file. */ void mergeLocations(ZipOutputStream out, List feedsToMerge) throws IOException { - Set locations = new HashSet<>(); - Set locationShapes = new HashSet<>(); + Set mergedLocations = new HashSet<>(); + Set mergedLocationShapes = new HashSet<>(); for (FeedToMerge feed : feedsToMerge) { ZipEntry locationGeoJsonFile = feed.zipFile.getEntry(LOCATION_GEO_JSON_FILE_NAME); if (locationGeoJsonFile != null) { - locations.addAll(GeoJsonUtil.getLocationsFromGeoJson(feed.zipFile, locationGeoJsonFile, null)); - locationShapes.addAll(GeoJsonUtil.getLocationShapesFromGeoJson(feed.zipFile, locationGeoJsonFile, null)); + String idScope = getCleanName(feed.version.parentFeedSource().name) + feed.version.version; + List locations = GeoJsonUtil.getLocationsFromGeoJson(feed.zipFile, locationGeoJsonFile, null); + for (Location location : locations) { + location.location_id = String.join(":", idScope, location.location_id); + mergedLocations.add(location); + } + List locationShapes = GeoJsonUtil.getLocationShapesFromGeoJson(feed.zipFile, locationGeoJsonFile, null); + for (LocationShape locationShape : locationShapes) { + locationShape.location_id = String.join(":", idScope, locationShape.location_id); + mergedLocationShapes.add(locationShape); + } } } - if (!locations.isEmpty()) { - JdbcGtfsExporter.writeLocationsToFile(out, new ArrayList<>(locations), new ArrayList<>(locationShapes)); + if (!mergedLocations.isEmpty()) { + JdbcGtfsExporter.writeLocationsToFile(out, new ArrayList<>(mergedLocations), new ArrayList<>(mergedLocationShapes)); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index a7c894513..487299f16 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -132,22 +132,22 @@ void canMergeRegional() throws SQLException { assertEquals( mergedVersion.feedLoadResult.bookingRules.rowCount, fakeAgencyWithFlexVersion1.feedLoadResult.bookingRules.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.bookingRules.rowCount, - "bookingRules count for merged feed should equal sum of area for versions merged." + "Booking rules count for merged feed should equal sum of booking rules for versions merged." ); assertEquals( mergedVersion.feedLoadResult.stopAreas.rowCount, fakeAgencyWithFlexVersion1.feedLoadResult.stopAreas.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopAreas.rowCount, - "stopAreas count for merged feed should equal sum of area for versions merged." + "Stop areas count for merged feed should equal sum of stop areas for versions merged." ); assertEquals( mergedVersion.feedLoadResult.locations.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.locations.rowCount, - "Merged versions contain the same locations, only one set of locations should remain after merge." + fakeAgencyWithFlexVersion1.feedLoadResult.locations.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.locations.rowCount, + "Locations count for merged feed should equal sum of locations for versions merged." ); assertEquals( mergedVersion.feedLoadResult.locationShapes.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.locationShapes.rowCount, - "Merged versions contain the same location shapes, only one set of location shapes should remain after merge." + fakeAgencyWithFlexVersion1.feedLoadResult.locationShapes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.locationShapes.rowCount, + "Location shapes count for merged feed should equal sum of location shapes for versions merged." ); // Ensure there are no referential integrity errors, duplicate ID, or wrong number of // fields errors. diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt index f82e627d8..3108fb401 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt @@ -1,7 +1,7 @@ -trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 -frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 -frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 -frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 -frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt index f82e627d8..3108fb401 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt @@ -1,7 +1,7 @@ -trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 -frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 -frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 -frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 -frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1 From d619b1067509b41a54ef8b81ac6b110487b82b8b Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 7 Nov 2023 08:42:02 +0000 Subject: [PATCH 088/133] refactor(pom.xml): Removed commented out dependency --- pom.xml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pom.xml b/pom.xml index c09a0f342..ef50266de 100644 --- a/pom.xml +++ b/pom.xml @@ -279,20 +279,6 @@ - - - - - - - - - - - - - - From 518dcbe88cda37c837b4ffd4494faaec45cfc5f9 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 8 Nov 2023 18:14:28 +0000 Subject: [PATCH 089/133] refactor(Restored full flex feed files and bumped gtfs-lib version): --- pom.xml | 2 +- .../fake-agency-with-flex-version-1/stop_times.txt | 14 +++++++------- .../fake-agency-with-flex-version-2/stop_times.txt | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index ef50266de..456d708c9 100644 --- a/pom.xml +++ b/pom.xml @@ -268,7 +268,7 @@ com.github.conveyal gtfs-lib - 87c7a3d53448dce2ce87465d66e001f7215edb3b + 8a9c3424747b05cb4c3b379e59e7762ef9f58e18 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt index 3108fb401..f82e627d8 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_times.txt @@ -1,7 +1,7 @@ -trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1 -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1 -frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1 -frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1 -frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1 -frequency-trip,,,3,4,,1,1,1,1,341.4491961,1 +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt index 3108fb401..f82e627d8 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_times.txt @@ -1,7 +1,7 @@ -trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1 -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1 -frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1 -frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1 -frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1 -frequency-trip,,,3,4,,1,1,1,1,341.4491961,1 +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,continuous_pickup,continuous_drop_off,shape_dist_traveled,timepoint,pickup_booking_rule_id,drop_off_booking_rule_id,start_pickup_drop_off_window,end_pickup_drop_off_window,mean_duration_factor,mean_duration_offset,safe_duration_factor,safe_duration_offset +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,1,1,0.0000000,1,"1","1",07:00:00,07:00:00,1,60,1,60 +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,1,1,341.4491961,1,"1","1",07:01:00,07:01:00,1,60,1,60 +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,1,1,0.0000000,1,"1","1",07:02:00,07:02:00,1,60,1,60 +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,08:31:00,08:31:00,area_251,3,,0,0,1,1,341.4491961,1,"1","1",07:03:00,07:03:00,1,60,1,60 +frequency-trip,,,3,4,,1,1,1,1,341.4491961,1,"1","1",07:04:00,07:05:00,1,60,1,60 From c13559d00918c6d2b3529ba0b39022fc966ab709 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 8 Nov 2023 18:16:41 +0000 Subject: [PATCH 090/133] refactor(ExportSnapshotToGTFSJob.java): Temp fix for call to JdbcGtfsExporter until dev is merged co --- .../conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java index 3798c2ee8..016c346f4 100644 --- a/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java +++ b/src/main/java/com/conveyal/datatools/editor/jobs/ExportSnapshotToGTFSJob.java @@ -57,7 +57,7 @@ public void jobLogic() { status.fail("Error creating local file for snapshot.", e); return; } - JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true); + JdbcGtfsExporter exporter = new JdbcGtfsExporter(snapshot.namespace, tempFile.getAbsolutePath(), DataManager.GTFS_DATA_SOURCE, true, true); FeedLoadResult result = exporter.exportTables(); if (result.fatalException != null) { status.fail(String.format("Error (%s) encountered while exporting database tables.", result.fatalException)); From 52b194f6328bd3fdbdef60437bdddfc18613d20e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Mon, 20 Nov 2023 16:41:39 +0000 Subject: [PATCH 091/133] refactor(Updates around foreign ref checks): --- pom.xml | 2 +- .../datatools/manager/jobs/MergeFeedsJob.java | 11 ++++-- .../jobs/feedmerge/FeedMergeContext.java | 3 ++ .../jobs/feedmerge/MergeFeedsResult.java | 6 +++ .../jobs/feedmerge/MergeLineContext.java | 32 ++++++++++++++-- .../feedmerge/ReferenceTableDiscovery.java | 38 +++++++++++++++++++ 6 files changed, 85 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 71469bac3..f43bcb5fc 100644 --- a/pom.xml +++ b/pom.xml @@ -271,7 +271,7 @@ com.github.conveyal gtfs-lib - e83292580a44880e1efcf6a35d24c2d46044c6f1 + 6da9e9999b1d41ecdcc7e2f88374327e22d7bffa diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 742dacbe8..284ecdbda 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -212,6 +212,8 @@ public void jobLogic() { } } + mergeLocations(out); + // Loop over GTFS tables and merge each feed one table at a time. for (int i = 0; i < numberOfTables; i++) { Table table = tablesToMerge.get(i); @@ -227,7 +229,6 @@ public void jobLogic() { LOG.error("Merge {} table failed!", table.name); } } - mergeLocations(out, feedMergeContext.feedsToMerge); } catch (IOException e) { String message = "Error creating output stream for feed merge."; logAndReportToBugsnag(e, message); @@ -258,11 +259,14 @@ public void jobLogic() { * Merge locations.geojson files. These files are not compatible with the CSV merging strategy. Instead, the * location.geojson file is flattened into locations and locations shapes. The location id is then updated with the * scope id to keep feed locations unique, converted back into geojson and written to the zip file. + * + * Locations must be processed prior to other CSV files so the location ids are available for foreign reference + * checks. */ - void mergeLocations(ZipOutputStream out, List feedsToMerge) throws IOException { + void mergeLocations(ZipOutputStream out) throws IOException { Set mergedLocations = new HashSet<>(); Set mergedLocationShapes = new HashSet<>(); - for (FeedToMerge feed : feedsToMerge) { + for (FeedToMerge feed : feedMergeContext.feedsToMerge) { ZipEntry locationGeoJsonFile = feed.zipFile.getEntry(LOCATION_GEO_JSON_FILE_NAME); if (locationGeoJsonFile != null) { String idScope = getCleanName(feed.version.parentFeedSource().name) + feed.version.version; @@ -270,6 +274,7 @@ void mergeLocations(ZipOutputStream out, List feedsToMerge) throws for (Location location : locations) { location.location_id = String.join(":", idScope, location.location_id); mergedLocations.add(location); + feedMergeContext.locationIds.add(location.location_id); } List locationShapes = GeoJsonUtil.getLocationShapesFromGeoJson(feed.zipFile, locationGeoJsonFile, null); for (LocationShape locationShape : locationShapes) { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 371112026..da5974856 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -9,6 +9,7 @@ import java.io.Closeable; import java.io.IOException; import java.time.LocalDate; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -23,6 +24,8 @@ public class FeedMergeContext implements Closeable { public final boolean tripIdsMatch; public final LocalDate futureFirstCalendarStartDate; public final Set sharedTripIds; + public Set locationIds = new HashSet<>(); + public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) throws IOException { feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index 881c78051..46f980d09 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -36,6 +36,12 @@ public class MergeFeedsResult implements Serializable { */ public Set calendarDatesServiceIds = new HashSet<>(); + /** + * Track various table ids for resolving foreign references. + */ + public Set stopIds = new HashSet<>(); + public Set stopAreaIds = new HashSet<>(); + /** * Track the set of route IDs to end up in the merged feed in order to determine which route_attributes * records should be retained in the merged result. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index e77a1c47d..15d9a77c2 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -229,6 +229,7 @@ public boolean iterateOverRows() throws IOException { public void startNewRow() throws IOException { keyValue = csvReader.get(keyFieldIndex); + setForeignReferenceKeyValues(); // Get the spec fields to export List specFields = table.specFields(); // Filter the spec fields on the set of fields found in all feeds to be merged. @@ -238,8 +239,24 @@ public void startNewRow() throws IOException { } /** - * Determine which reference table to use. If there is only one reference use this. If there are multiple references - * determine the context and then the correct reference table to use. + * Build a list of table key id values to be used in foreign key field look-ups. + */ + private void setForeignReferenceKeyValues() { + switch (table.name) { + case "stops": + mergeFeedsResult.stopIds.add(getIdWithScope(keyValue)); + break; + case "stop_areas": + mergeFeedsResult.stopAreaIds.add(getIdWithScope(keyValue)); + break; + default: + // nothing. + } + } + + /** + * Determine which reference table to use. If there is only one reference use this. If there are multiple + * references, determine the context and then the correct reference table to use. */ private Table getReferenceTable(FieldContext fieldContext, Field field) { if (field.referenceTables.size() == 1) { @@ -255,8 +272,17 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) { getTableScopedValue(Table.CALENDAR_DATES, fieldContext.getValue()) ); case STOP_TIMES_STOP_ID_KEY: + return ReferenceTableDiscovery.getStopTimeStopIdReferenceTable( + fieldContext.getValueToWrite(), + mergeFeedsResult, + feedMergeContext.locationIds + ); case STOP_AREA_STOP_ID_KEY: - // Include other cases as multiple references are added e.g. flex!. + return ReferenceTableDiscovery.getStopAreaAreaIdReferenceTable( + fieldContext.getValueToWrite(), + mergeFeedsResult, + feedMergeContext.locationIds + ); default: return null; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java index bf96de3cb..00e5f9afe 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java @@ -3,6 +3,7 @@ import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.Table; +import java.util.Set; import java.util.stream.Collectors; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeLineContext.SERVICE_ID; @@ -11,6 +12,9 @@ public class ReferenceTableDiscovery { public static final String REF_TABLE_SEPARATOR = "#~#"; + /** + * Tables that have two or more foreign references. + */ public enum ReferenceTableKey { TRIP_SERVICE_ID_KEY( @@ -106,4 +110,38 @@ public static Table getTripServiceIdReferenceTable( } return null; } + + /** + * Define the reference table for a stop area's area id. This will either be a stop or location. + */ + public static Table getStopAreaAreaIdReferenceTable( + String fieldValue, + MergeFeedsResult mergeFeedsResult, + Set locationIds + ) { + if (mergeFeedsResult.stopIds.contains(fieldValue)) { + return Table.STOPS; + } else if (locationIds.contains(fieldValue)) { + return Table.LOCATIONS; + } + return null; + } + + /** + * Define the reference table for a stop time's stop id. This will either be a stop, location or stop area. + */ + public static Table getStopTimeStopIdReferenceTable( + String fieldValue, + MergeFeedsResult mergeFeedsResult, + Set locationIds + ) { + if (mergeFeedsResult.stopIds.contains(fieldValue)) { + return Table.STOPS; + } else if (locationIds.contains(fieldValue)) { + return Table.LOCATIONS; + } else if (mergeFeedsResult.stopAreaIds.contains(fieldValue)) { + return Table.STOP_AREAS; + } + return null; + } } From 75ad5241158d2272bbda23413e4333a8f006516a Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 22 Nov 2023 07:59:02 +0000 Subject: [PATCH 092/133] chore(Updated GTFS-lib reference to point at the new ibi group repo): --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f43bcb5fc..57e57ddce 100644 --- a/pom.xml +++ b/pom.xml @@ -269,9 +269,9 @@ - AWS S3 SDK - putting/getting objects into/out of S3. --> - com.github.conveyal + com.github.ibi-group gtfs-lib - 6da9e9999b1d41ecdcc7e2f88374327e22d7bffa + 6da9e9999b From 3a5263fab104304d7fe33365590cc4b18fafcdbc Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 29 Nov 2023 10:22:32 +0000 Subject: [PATCH 093/133] refactor(Addressed PR feedback): Tidied up unit test --- pom.xml | 2 +- .../manager/jobs/FlexMergeFeedsJobTest.java | 96 +++++++------------ .../attributions.txt | 2 +- .../booking_rules.txt | 2 +- .../feed_info.txt | 2 +- .../fake-agency-with-flex-version-1/stops.txt | 2 +- .../transfers.txt | 1 - .../attributions.txt | 2 +- .../booking_rules.txt | 2 +- .../feed_info.txt | 2 +- .../fake-agency-with-flex-version-2/stops.txt | 2 +- .../transfers.txt | 1 - 12 files changed, 43 insertions(+), 73 deletions(-) delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt diff --git a/pom.xml b/pom.xml index 57e57ddce..73bd0b51a 100644 --- a/pom.xml +++ b/pom.xml @@ -271,7 +271,7 @@ com.github.ibi-group gtfs-lib - 6da9e9999b + 21d04ec0d0559f434e6a4378821b8fa2046f59c8 diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index 487299f16..a2fb440b6 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -11,6 +11,9 @@ import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.loader.FeedLoadResult; +import com.conveyal.gtfs.loader.Table; +import com.conveyal.gtfs.loader.TableLoadResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -88,69 +91,31 @@ void canMergeRegional() throws SQLException { versions.add(fakeAgencyWithFlexVersion2); FeedVersion mergedVersion = regionallyMergeVersions(versions); + FeedLoadResult r1 = fakeAgencyWithFlexVersion1.feedLoadResult; + FeedLoadResult r2 = fakeAgencyWithFlexVersion2.feedLoadResult; + FeedLoadResult merged = mergedVersion.feedLoadResult; + // Ensure the feed has the row counts we expect. - assertEquals( - fakeAgencyWithFlexVersion1.feedLoadResult.trips.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.trips.rowCount, - mergedVersion.feedLoadResult.trips.rowCount, - "trips count for merged feed should equal sum of trips for versions merged." - ); - assertEquals( - fakeAgencyWithFlexVersion1.feedLoadResult.routes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.routes.rowCount, - mergedVersion.feedLoadResult.routes.rowCount, - "routes count for merged feed should equal sum of routes for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.stops.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.stops.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stops.rowCount, - "stops count for merged feed should equal sum of stops for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.agency.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.agency.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.agency.rowCount, - "agency count for merged feed should equal sum of agency for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.stopTimes.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.stopTimes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopTimes.rowCount, - "stopTimes count for merged feed should equal sum of stopTimes for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.calendar.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.calendar.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.calendar.rowCount, - "calendar count for merged feed should equal sum of calendar for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.calendarDates.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.calendarDates.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.calendarDates.rowCount, - "calendarDates count for merged feed should equal sum of calendarDates for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.area.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.area.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.area.rowCount, - "area count for merged feed should equal sum of area for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.bookingRules.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.bookingRules.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.bookingRules.rowCount, - "Booking rules count for merged feed should equal sum of booking rules for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.stopAreas.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.stopAreas.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.stopAreas.rowCount, - "Stop areas count for merged feed should equal sum of stop areas for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.locations.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.locations.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.locations.rowCount, - "Locations count for merged feed should equal sum of locations for versions merged." - ); - assertEquals( - mergedVersion.feedLoadResult.locationShapes.rowCount, - fakeAgencyWithFlexVersion1.feedLoadResult.locationShapes.rowCount + fakeAgencyWithFlexVersion2.feedLoadResult.locationShapes.rowCount, - "Location shapes count for merged feed should equal sum of location shapes for versions merged." - ); - // Ensure there are no referential integrity errors, duplicate ID, or wrong number of - // fields errors. + assertRowCount(r1.agency, r2.agency, merged.agency, "Agency count for merged feed should equal sum of agency for versions merged."); + assertRowCount(r1.area, r2.area, merged.area, "Area count for merged feed should equal sum of area for versions merged."); + assertRowCount(r1.attributions, r2.attributions, merged.attributions, "Attributions count for merged feed should equal sum of attributions for versions merged."); + assertRowCount(r1.bookingRules, r2.bookingRules, merged.bookingRules, "Booking rules count for merged feed should equal sum of booking rules for versions merged."); + assertRowCount(r1.calendar, r2.calendar, merged.calendar, "Calendar count for merged feed should equal sum of calendar for versions merged."); + assertRowCount(r1.calendarDates, r2.calendarDates, merged.calendarDates, "Calendar dates count for merged feed should equal sum of calendar dates for versions merged."); + assertRowCount(r1.fareAttributes, r2.fareAttributes, merged.fareAttributes, "Fare attributes count for merged feed should equal sum of fare attributes for versions merged."); + assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules count for merged feed should equal sum of fare rules for versions merged."); + assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies count for merged feed should equal sum of frequencies for versions merged."); + assertRowCount(r1.locations, r2.locations, merged.locations, "Locations count for merged feed should equal sum of locations for versions merged."); + assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes count for merged feed should equal sum of location shapes for versions merged."); + assertRowCount(r1.routes, r2.routes, merged.routes, "Routes count for merged feed should equal sum of routes for versions merged."); + assertRowCount(r1.shapes, r2.shapes, merged.shapes, "Shapes count for merged feed should equal sum of shapes for versions merged."); + assertRowCount(r1.stops, r2.stops, merged.stops, "Stops count for merged feed should equal sum of stops for versions merged."); + assertRowCount(r1.stopAreas, r2.stopAreas, merged.stopAreas, "Stop areas count for merged feed should equal sum of stop areas for versions merged."); + assertRowCount(r1.stopTimes, r2.stopTimes, merged.stopTimes, "Stop times count for merged feed should equal sum of stopTimes for versions merged."); + assertRowCount(r1.trips, r2.trips, merged.trips, "Trips count for merged feed should equal sum of trips for versions merged."); + assertRowCount(r1.translations, r2.translations, merged.translations, "Translations count for merged feed should equal sum of translations for versions merged."); + + // Ensure there are no referential integrity errors, duplicate ID, or wrong number of fields errors. assertThatFeedHasNoErrorsOfType( mergedVersion.namespace, NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), @@ -169,4 +134,11 @@ private FeedVersion regionallyMergeVersions(Set versions) { LOG.info("Regional merged file: {}", mergeFeedsJob.mergedVersion.retrieveGtfsFile().getAbsolutePath()); return mergeFeedsJob.mergedVersion; } + + /** + * Helper method to confirm that the sum of the two feed table rows match the merged feed table rows. + */ + private void assertRowCount(TableLoadResult feedOne, TableLoadResult feedTwo, TableLoadResult feedMerged, String message) { + assertEquals(feedOne.rowCount + feedTwo.rowCount, feedMerged.rowCount, message); + } } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt index 1d77168ac..2e082242d 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/attributions.txt @@ -1,2 +1,2 @@ attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone -1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org, \ No newline at end of file +1,1,,,Fake Transit,1,,,https://faketransit.example.com,customer.service@faketransit.example.com, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt index a0e5dd79d..36a5af626 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/booking_rules.txt @@ -1,2 +1,2 @@ booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url -Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://www.info_url.com","http://www.booking_url.com" \ No newline at end of file +Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://info.example.com","http://booking.example.com" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt index ceac60810..0912e2ac5 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/feed_info.txt @@ -1,2 +1,2 @@ feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version -fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file +fake_transit,Conveyal,http://publisher.example.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt index 49e996baa..9d56ffab3 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stops.txt @@ -1,5 +1,5 @@ stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding -4u6g,,Butler Ln,,37.0612132,-122.0074332,,,0,,, +4u6g,,Butler Ln,Heading north,37.0612132,-122.0074332,,,0,,, johv,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, 123,,Parent Station,,37.0666,-122.0777,,,1,,, 1234,,Child Stop,,37.06662,-122.07772,,,0,123,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt deleted file mode 100644 index 357103c47..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/transfers.txt +++ /dev/null @@ -1 +0,0 @@ -from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt index 1d77168ac..2e082242d 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/attributions.txt @@ -1,2 +1,2 @@ attribution_id,agency_id,route_id,trip_id,organization_name,is_producer,is_operator,is_authority,attribution_url,attribution_email,attribution_phone -1,1,,,Fake Transit,1,,,https://www.faketransit.org,customer.service@faketransit.org, \ No newline at end of file +1,1,,,Fake Transit,1,,,https://faketransit.example.com,customer.service@faketransit.example.com, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt index a0e5dd79d..36a5af626 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/booking_rules.txt @@ -1,2 +1,2 @@ booking_rule_id,booking_type,prior_notice_duration_min,prior_notice_duration_max,prior_notice_last_day,prior_notice_last_time,prior_notice_start_day,prior_notice_start_time,prior_notice_service_id,message,pickup_message,drop_off_message,phone_number,info_url,booking_url -Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://www.info_url.com","http://www.booking_url.com" \ No newline at end of file +Booking 1,1,30,60,1,17:00:00,7,00:00:00,"04100312-8fe1-46a5-a9f2-556f39478f57","This is a message","This is a pickup message","This is a drop off message","123456789","http://info.example.com","http://booking.example.com" \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt index ceac60810..0912e2ac5 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/feed_info.txt @@ -1,2 +1,2 @@ feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version -fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file +fake_transit,Conveyal,http://publisher.example.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt index 49e996baa..9d56ffab3 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stops.txt @@ -1,5 +1,5 @@ stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding -4u6g,,Butler Ln,,37.0612132,-122.0074332,,,0,,, +4u6g,,Butler Ln,Heading north,37.0612132,-122.0074332,,,0,,, johv,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, 123,,Parent Station,,37.0666,-122.0777,,,1,,, 1234,,Child Stop,,37.06662,-122.07772,,,0,123,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt deleted file mode 100644 index 357103c47..000000000 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/transfers.txt +++ /dev/null @@ -1 +0,0 @@ -from_stop_id,to_stop_id,transfer_type,min_transfer_time From 9f4a33855ce2eb0d797ecaf77a47169721d8d7b0 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Thu, 30 Nov 2023 08:00:33 +0000 Subject: [PATCH 094/133] refactor(Updated unit test and required test data): --- .../manager/jobs/FlexMergeFeedsJobTest.java | 44 ++++++++++--------- .../translations.txt | 2 +- .../translations.txt | 2 +- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index a2fb440b6..eb021c3b2 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -96,24 +96,24 @@ void canMergeRegional() throws SQLException { FeedLoadResult merged = mergedVersion.feedLoadResult; // Ensure the feed has the row counts we expect. - assertRowCount(r1.agency, r2.agency, merged.agency, "Agency count for merged feed should equal sum of agency for versions merged."); - assertRowCount(r1.area, r2.area, merged.area, "Area count for merged feed should equal sum of area for versions merged."); - assertRowCount(r1.attributions, r2.attributions, merged.attributions, "Attributions count for merged feed should equal sum of attributions for versions merged."); - assertRowCount(r1.bookingRules, r2.bookingRules, merged.bookingRules, "Booking rules count for merged feed should equal sum of booking rules for versions merged."); - assertRowCount(r1.calendar, r2.calendar, merged.calendar, "Calendar count for merged feed should equal sum of calendar for versions merged."); - assertRowCount(r1.calendarDates, r2.calendarDates, merged.calendarDates, "Calendar dates count for merged feed should equal sum of calendar dates for versions merged."); - assertRowCount(r1.fareAttributes, r2.fareAttributes, merged.fareAttributes, "Fare attributes count for merged feed should equal sum of fare attributes for versions merged."); - assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules count for merged feed should equal sum of fare rules for versions merged."); - assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies count for merged feed should equal sum of frequencies for versions merged."); - assertRowCount(r1.locations, r2.locations, merged.locations, "Locations count for merged feed should equal sum of locations for versions merged."); - assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes count for merged feed should equal sum of location shapes for versions merged."); - assertRowCount(r1.routes, r2.routes, merged.routes, "Routes count for merged feed should equal sum of routes for versions merged."); - assertRowCount(r1.shapes, r2.shapes, merged.shapes, "Shapes count for merged feed should equal sum of shapes for versions merged."); - assertRowCount(r1.stops, r2.stops, merged.stops, "Stops count for merged feed should equal sum of stops for versions merged."); - assertRowCount(r1.stopAreas, r2.stopAreas, merged.stopAreas, "Stop areas count for merged feed should equal sum of stop areas for versions merged."); - assertRowCount(r1.stopTimes, r2.stopTimes, merged.stopTimes, "Stop times count for merged feed should equal sum of stopTimes for versions merged."); - assertRowCount(r1.trips, r2.trips, merged.trips, "Trips count for merged feed should equal sum of trips for versions merged."); - assertRowCount(r1.translations, r2.translations, merged.translations, "Translations count for merged feed should equal sum of translations for versions merged."); + assertRowCount(r1.agency, r2.agency, merged.agency, "Agency"); + assertRowCount(r1.area, r2.area, merged.area, "Area"); + assertRowCount(r1.attributions, r2.attributions, merged.attributions, "Attributions"); + assertRowCount(r1.bookingRules, r2.bookingRules, merged.bookingRules, "Booking rules"); + assertRowCount(r1.calendar, r2.calendar, merged.calendar, "Calendar"); + assertRowCount(r1.calendarDates, r2.calendarDates, merged.calendarDates, "Calendar dates"); + assertRowCount(r1.fareAttributes, r2.fareAttributes, merged.fareAttributes, "Fare attributes"); + assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules"); + assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies"); + assertRowCount(r1.locations, r2.locations, merged.locations, "Locations"); + assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes"); + assertRowCount(r1.routes, r2.routes, merged.routes, "Routes"); + assertRowCount(r1.shapes, r2.shapes, merged.shapes, "Shapes"); + assertRowCount(r1.stops, r2.stops, merged.stops, "Stops"); + assertRowCount(r1.stopAreas, r2.stopAreas, merged.stopAreas, "Stop areas"); + assertRowCount(r1.stopTimes, r2.stopTimes, merged.stopTimes, "Stop times"); + assertRowCount(r1.trips, r2.trips, merged.trips, "Trips"); + assertRowCount(r1.translations, r2.translations, merged.translations, "Translations"); // Ensure there are no referential integrity errors, duplicate ID, or wrong number of fields errors. assertThatFeedHasNoErrorsOfType( @@ -138,7 +138,11 @@ private FeedVersion regionallyMergeVersions(Set versions) { /** * Helper method to confirm that the sum of the two feed table rows match the merged feed table rows. */ - private void assertRowCount(TableLoadResult feedOne, TableLoadResult feedTwo, TableLoadResult feedMerged, String message) { - assertEquals(feedOne.rowCount + feedTwo.rowCount, feedMerged.rowCount, message); + private void assertRowCount(TableLoadResult feedOne, TableLoadResult feedTwo, TableLoadResult feedMerged, String entity) { + assertEquals( + feedOne.rowCount + feedTwo.rowCount, + feedMerged.rowCount, + String.format("%s count for merged feed should equal the sum for the versions merged.", entity) + ); } } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt index da8fe2407..8e48c9733 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/translations.txt @@ -1,2 +1,2 @@ table_name,field_name,language,translation,record_id,record_sub_id,field_value -stops,stop_desc,FR,en direction du nord,4u6g,, \ No newline at end of file +stops,stop_desc,FR,Direction nord,4u6g,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt index da8fe2407..8e48c9733 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/translations.txt @@ -1,2 +1,2 @@ table_name,field_name,language,translation,record_id,record_sub_id,field_value -stops,stop_desc,FR,en direction du nord,4u6g,, \ No newline at end of file +stops,stop_desc,FR,Direction nord,4u6g,, \ No newline at end of file From 6279783b56f58fededfeeb0ba7bf1a7e2158d441 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 29 Jan 2024 09:54:48 -0500 Subject: [PATCH 095/133] update gtfs-lib flex --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8fd04028e..84893a3eb 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - b657dfe7d7c75eacfbc165cf37c0c0ff3f8cd23e + e83292580a44880e1efcf6a35d24c2d46044c6f1 From a332cfdd7961223df66f3d414a3aca7e3ae6ad56 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 29 Jan 2024 10:05:40 -0500 Subject: [PATCH 096/133] correctly update gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 84893a3eb..70a3b07ef 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - e83292580a44880e1efcf6a35d24c2d46044c6f1 + 5e24a64c4266c5cc9866eaa4b11a1231270a2498 From 6a764ffac94d6e0b04d7846c69bdfdd64b89e10f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 15 May 2024 11:29:05 +0100 Subject: [PATCH 097/133] refactor(Updates to include location groups and location group stops): --- pom.xml | 2 +- .../datatools/manager/DataManager.java | 5 ++-- .../jobs/feedmerge/MergeFeedsResult.java | 2 +- .../jobs/feedmerge/MergeLineContext.java | 13 +++++----- .../feedmerge/ReferenceTableDiscovery.java | 24 +++++++++---------- .../manager/jobs/FlexMergeFeedsJobTest.java | 4 ++-- .../areas.txt | 2 +- .../fake-agency-with-flex-version-1/areas.txt | 4 ---- .../location_group_stops.txt | 2 ++ .../location_groups.txt | 2 ++ .../locations.geojson | 8 +++---- .../stop_areas.txt | 4 ---- .../fake-agency-with-flex-version-2/areas.txt | 4 ---- .../location_group_stops.txt | 2 ++ .../location_groups.txt | 2 ++ .../locations.geojson | 8 +++---- .../stop_areas.txt | 4 ---- 17 files changed, 40 insertions(+), 52 deletions(-) delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/location_group_stops.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/location_groups.txt delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-1/stop_areas.txt delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/areas.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/location_group_stops.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/location_groups.txt delete mode 100644 src/test/resources/com/conveyal/datatools/gtfs/fake-agency-with-flex-version-2/stop_areas.txt diff --git a/pom.xml b/pom.xml index b525edaba..19d91f679 100644 --- a/pom.xml +++ b/pom.xml @@ -271,7 +271,7 @@ com.github.ibi-group gtfs-lib - 2bd924c6e5f5244a279c9d94298c14c0b3d47b35 + e623aef54deba8c95d34e2f7c6760bc055612193 - dae1fee4cc9140c435b17c8d2118940872846365 + 56a5d0ad7fb5d98b4410683a2c189039fc26ebf3 From 31e53202bb6820508e727b73fcc143d8355f67d6 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Wed, 10 Jun 2026 16:48:05 -0400 Subject: [PATCH 123/133] chore(deps): Upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 05f3f8865..0e75e56c4 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - b115413db386bce4ea70284857284b07671c2906 + 6fa568a187984044c01746cb6607135714671f18 From 353ed213050dd8e2f848d91d949ef2a9544c7bac Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Mon, 6 Jul 2026 10:01:48 -0400 Subject: [PATCH 124/133] chore(deps): Upgrade gtfs-lib --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0e75e56c4..113e49c11 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 6fa568a187984044c01746cb6607135714671f18 + dbbf3d166cf31fc9dae490ef4b8db42ec436f743 From 13426492db828eee544a66c0d8c92a29c7ad0b6f Mon Sep 17 00:00:00 2001 From: Rob Beer Date: Tue, 14 Jul 2026 08:22:58 +0100 Subject: [PATCH 125/133] chore(Config update): disabled mtc extension by default --- configurations/test/server.yml.tmp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/test/server.yml.tmp b/configurations/test/server.yml.tmp index c24f035fa..10602849f 100644 --- a/configurations/test/server.yml.tmp +++ b/configurations/test/server.yml.tmp @@ -33,7 +33,7 @@ modules: gtfsapi: enabled: true load_on_fetch: false - use_extension: mtc + # use_extension: mtc # update_frequency: 30 # in seconds manager: normalizeFieldTransformation: From 81d0288da1c65001a0038f61602b7831c84000cc Mon Sep 17 00:00:00 2001 From: Rob Beer Date: Tue, 14 Jul 2026 08:47:29 +0100 Subject: [PATCH 126/133] chore(config update): enabled mtc extension by default This is required for S3UtilsTest unit test to pass --- configurations/test/server.yml.tmp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/test/server.yml.tmp b/configurations/test/server.yml.tmp index 10602849f..c24f035fa 100644 --- a/configurations/test/server.yml.tmp +++ b/configurations/test/server.yml.tmp @@ -33,7 +33,7 @@ modules: gtfsapi: enabled: true load_on_fetch: false - # use_extension: mtc + use_extension: mtc # update_frequency: 30 # in seconds manager: normalizeFieldTransformation: From 504d08c65f77089f5943b80caae560aa66c56fb0 Mon Sep 17 00:00:00 2001 From: Rob Beer Date: Wed, 15 Jul 2026 08:20:37 +0100 Subject: [PATCH 127/133] improvement(Updated gtfs-lib version): --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0831fdb12..aa23c463a 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 9a3ac87a9fdb96a1516e5badf3bca6071fb6aca7 + 926b910e098fa8aba38ada01ce2f5bb7dd38e917 From efa510f5992c539613e38c179eace48c9ce809fd Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 7 Aug 2026 11:34:18 -0400 Subject: [PATCH 128/133] chore(deps): update gtfs-lib for CSV export fixes. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa23c463a..1cbba7182 100644 --- a/pom.xml +++ b/pom.xml @@ -272,7 +272,7 @@ com.github.ibi-group gtfs-lib - 926b910e098fa8aba38ada01ce2f5bb7dd38e917 + f683b708724329dafe10c6bfbcfa158a012b56d3 From 12ca623d34ee066a9db6d2455e2fa9e904767ff7 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Mon, 10 Aug 2026 17:20:30 -0400 Subject: [PATCH 129/133] test(FeedMergeFeedsJob): Remove unused imports/modifiers. --- .../manager/jobs/FlexMergeFeedsJobTest.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index 99dfc6ec3..d4b14b813 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -12,7 +12,6 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.loader.FeedLoadResult; -import com.conveyal.gtfs.loader.Table; import com.conveyal.gtfs.loader.TableLoadResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -27,19 +26,13 @@ import java.util.Set; import static com.conveyal.datatools.TestUtils.assertThatFeedHasNoErrorsOfType; -import static com.conveyal.datatools.TestUtils.createFeedVersion; -import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; -import static com.conveyal.datatools.TestUtils.zipFolderFiles; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; /** * Tests for the various {@link MergeFeedsJob} merge types. */ -public class FlexMergeFeedsJobTest extends UnitTest { +class FlexMergeFeedsJobTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(FlexMergeFeedsJobTest.class); private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static FeedVersion fakeAgencyWithFlexVersion1; @@ -50,7 +43,7 @@ public class FlexMergeFeedsJobTest extends UnitTest { * Prepare and start a testing-specific web server */ @BeforeAll - public static void setUp() throws IOException { + static void setUp() throws IOException { // start server if it isn't already running DatatoolsTest.setUp(); Auth0Connection.setAuthDisabled(true); @@ -73,7 +66,7 @@ public static void setUp() throws IOException { * Delete project on tear down (feed sources/versions will also be deleted). */ @AfterAll - public static void tearDown() { + static void tearDown() { if (project != null) { project.delete(); } From 2c5d30288f752eb7d79f90f32c818ae3b5c4134f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Mon, 10 Aug 2026 17:22:56 -0400 Subject: [PATCH 130/133] test(FeedMergeFeedsJob): Comment out test on merged locations. --- .../conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index d4b14b813..2cb2c813b 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -97,7 +97,8 @@ void canMergeRegional() throws SQLException { assertRowCount(r1.fareAttributes, r2.fareAttributes, merged.fareAttributes, "Fare attributes"); assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules"); assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies"); - assertRowCount(r1.locations, r2.locations, merged.locations, "Locations"); + // FIXME: Export locations of unsupported types when merging feeds. + // assertRowCount(r1.locations, r2.locations, merged.locations, "Locations"); assertRowCount(r1.locationGroup, r2.locationGroup, merged.locationGroup, "Location Groups"); assertRowCount(r1.locationGroupStops, r2.locationGroupStops, merged.locationGroupStops, "Location Group Stops"); assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes"); From e0990ae832dffc5d8ed77dd54dd5c6f8159be88b Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup Date: Fri, 14 Aug 2026 08:07:35 -0400 Subject: [PATCH 131/133] test(FlexMergeFeedsJob): Update test comments regarding supported location types. --- .../manager/jobs/FlexMergeFeedsJobTest.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java index 2cb2c813b..0d04589dc 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FlexMergeFeedsJobTest.java @@ -97,8 +97,16 @@ void canMergeRegional() throws SQLException { assertRowCount(r1.fareAttributes, r2.fareAttributes, merged.fareAttributes, "Fare attributes"); assertRowCount(r1.fareRules, r2.fareRules, merged.fareRules, "Fare rules"); assertRowCount(r1.frequencies, r2.frequencies, merged.frequencies, "Frequencies"); - // FIXME: Export locations of unsupported types when merging feeds. - // assertRowCount(r1.locations, r2.locations, merged.locations, "Locations"); + // For GeoJSON locations, the change from https://github.com/ibi-group/gtfs-lib/pull/28 + // results in an additional two locations registered into feed versions 1 and 2 each. + // These additional locations are flagged with an error because the location type is not polygon or linestring, + // and they are not stored or exported during merge, so we subtract these locations from the expected count. + // The expected locations in the merged feed are unchanged. + assertEquals( + r1.locations.rowCount - r1.locations.errorCount + r2.locations.rowCount - r2.locations.errorCount, + merged.locations.rowCount + ); + assertRowCount(r1.locationGroup, r2.locationGroup, merged.locationGroup, "Location Groups"); assertRowCount(r1.locationGroupStops, r2.locationGroupStops, merged.locationGroupStops, "Location Group Stops"); assertRowCount(r1.locationShapes, r2.locationShapes, merged.locationShapes, "Location shapes"); From 1c545ee94152080ac946767f31299f71f11ea9d5 Mon Sep 17 00:00:00 2001 From: Rob Beer Date: Mon, 14 Sep 2026 12:37:14 +0100 Subject: [PATCH 132/133] improvement(build-docker.yml): Removed dev-flex branch trigger --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a3ebe75cb..a24ed39a9 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -3,7 +3,7 @@ name: Create and publish a Docker image on: push: - branches: ['dev', 'master', 'dev-flex', 'mtc-deploy'] + branches: ['dev', 'master', 'mtc-deploy'] env: REGISTRY: ghcr.io From f9d0297094dd0d87adb4e9912942f87578d4bb24 Mon Sep 17 00:00:00 2001 From: Rob Beer Date: Mon, 21 Sep 2026 10:01:26 +0100 Subject: [PATCH 133/133] chore(Updated gtfs-lib version): Version now matches the commit hash from merge into dev --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa9c23c0e..4230d9f57 100644 --- a/pom.xml +++ b/pom.xml @@ -296,7 +296,7 @@ com.github.ibi-group gtfs-lib - 89dba089dcc10d41a34b4d5868b055dad4309904 + 065d51c957dfdd6231b3f10e9fddef4dd867146a