Merge Dev-flex into Dev

.github/workflows/build-docker.yml

@@ -3,7 +3,7 @@ name: Create and publish a Docker image
 
 on:
   push:
-    branches: ['dev', 'master', 'dev-flex', 'mtc-deploy']
+    branches: ['dev', 'master', 'mtc-deploy']
 
 env:
   REGISTRY: ghcr.io

pom.xml

@@ -296,7 +296,7 @@
             <groupId>com.github.ibi-group</groupId>
             <artifactId>gtfs-lib</artifactId>
             <!-- Latest dev build on jitpack.io -->
-            <version>d39534084f9df5a798ae8d2b6f61cd8a6ab4947d</version>
+            <version>065d51c957dfdd6231b3f10e9fddef4dd867146a</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->
@@ -424,9 +424,9 @@
         </dependency>
         <!-- Snapshotting library for testing -->
         <dependency>
-            <groupId>com.github.conveyal</groupId>
+            <groupId>com.github.ibi-group</groupId>
             <artifactId>java-snapshot-matcher</artifactId>
-            <version>3495b32f7b4d3f82590e0a2284029214070b6984</version>
+            <version>master</version>
             <scope>test</scope>
         </dependency>
         <!-- wiremock is used to mock http requests -->
@@ -505,6 +505,11 @@
             <artifactId>gson</artifactId>
             <version>2.8.9</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.17.0</version>
+        </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>

src/main/java/com/conveyal/datatools/manager/DataManager.java

@@ -227,6 +227,13 @@ static void registerRoutes() throws IOException {
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_PRODUCTS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FARE_TRANSFER_RULES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.FEED_INFO, DataManager.GTFS_DATA_SOURCE);
+
+            // NOTE: Booking rules, locations and location shapes are GTFS Flex additions.
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.BOOKING_RULES, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATIONS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_GROUP_STOPS, DataManager.GTFS_DATA_SOURCE);
+            new EditorControllerImpl(EDITOR_API_PREFIX, Table.LOCATION_SHAPES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.NETWORKS, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.RIDER_CATEGORIES, DataManager.GTFS_DATA_SOURCE);
             new EditorControllerImpl(EDITOR_API_PREFIX, Table.ROUTES, DataManager.GTFS_DATA_SOURCE);

src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java

@@ -18,8 +18,12 @@
 import com.conveyal.datatools.manager.persistence.Persistence;
 import com.conveyal.datatools.manager.utils.ErrorUtils;
 import com.conveyal.gtfs.loader.Feed;
+import com.conveyal.gtfs.loader.JdbcGtfsExporter;
 import com.conveyal.gtfs.loader.Table;
+import com.conveyal.gtfs.model.Location;
+import com.conveyal.gtfs.model.LocationShape;
 import com.conveyal.gtfs.model.StopTime;
+import com.conveyal.gtfs.util.GeoJsonUtil;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
@@ -39,13 +43,17 @@
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD;
 import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL;
 import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES;
 import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE;
-import static com.conveyal.datatools.manager.utils.MergeFeedUtils.*;
+import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergedVersion;
+import static com.conveyal.datatools.manager.utils.MergeFeedUtils.stopTimesMatchSimplified;
+import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName;
+import static com.conveyal.gtfs.loader.Table.LOCATION_GEO_JSON_FILE_NAME;
 
 /**
  * This job handles merging two or more feed versions according to logic specific to the specified merge type.
@@ -205,6 +213,8 @@ public void jobLogic() {
                 }
             }
 
+            mergeLocations(out);
+
             // Loop over GTFS tables and merge each feed one table at a time.
             for (int i = 0; i < numberOfTables; i++) {
                 Table table = tablesToMerge.get(i);
@@ -226,7 +236,7 @@ public void jobLogic() {
             status.fail(message, e);
         } finally {
             try {
-                feedMergeContext.close();
+                if (feedMergeContext != null) feedMergeContext.close();
             } catch (IOException e) {
                 logAndReportToBugsnag(e, "Error closing FeedMergeContext object");
             }
@@ -246,6 +256,39 @@ public void jobLogic() {
         }
     }
 
+    /**
+     * Merge locations.geojson files. These files are not compatible with the CSV merging strategy. Instead, the
+     * location.geojson file is flattened into locations and locations shapes. The location id is then updated with the
+     * scope id to keep feed locations unique, converted back into geojson and written to the zip file.
+     *
+     * Locations must be processed prior to other CSV files so the location ids are available for foreign reference
+     * checks.
+     */
+    void mergeLocations(ZipOutputStream out) throws IOException {
+        Set<Location> mergedLocations = new HashSet<>();
+        Set<LocationShape> mergedLocationShapes = new HashSet<>();
+        for (FeedToMerge feed : feedMergeContext.feedsToMerge) {
+            ZipEntry locationGeoJsonFile = feed.zipFile.getEntry(LOCATION_GEO_JSON_FILE_NAME);
+            if (locationGeoJsonFile != null) {
+                String idScope = getCleanName(feed.version.parentFeedSource().name) + feed.version.version;
+                List<Location> locations = GeoJsonUtil.getLocationsFromGeoJson(feed.zipFile, locationGeoJsonFile, null);
+                for (Location location : locations) {
+                    location.location_id = String.join(":", idScope, location.location_id);
+                    mergedLocations.add(location);
+                    feedMergeContext.locationIds.add(location.location_id);
+                }
+                List<LocationShape> locationShapes = GeoJsonUtil.getLocationShapesFromGeoJson(feed.zipFile, locationGeoJsonFile, null);
+                for (LocationShape locationShape : locationShapes) {
+                    locationShape.location_id = String.join(":", idScope, locationShape.location_id);
+                    mergedLocationShapes.add(locationShape);
+                }
+            }
+        }
+        if (!mergedLocations.isEmpty()) {
+            JdbcGtfsExporter.writeLocationsToFile(out, new ArrayList<>(mergedLocations), new ArrayList<>(mergedLocationShapes));
+        }
+    }
+
     /**
      * Obtains trip ids whose entries in the stop_times table differ between the active and future feed.
      */
@@ -282,6 +325,10 @@ private boolean shouldSkipTable(String tableName) {
             LOG.warn("Skipping editor-only table {}.", tableName);
             return true;
         }
+        if (tableName.equals(Table.LOCATIONS.name) || tableName.equals(Table.LOCATION_SHAPES.name)) {
+            LOG.warn("{} detected. Skipping traditional merge in favour of bespoke merge.", LOCATION_GEO_JSON_FILE_NAME);
+            return true;
+        }
         return false;
     }
 

src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java

@@ -14,6 +14,7 @@
 import com.conveyal.datatools.manager.models.transform.FeedTransformZipTarget;
 import com.conveyal.datatools.manager.models.transform.RemoveNonRevenueTripsTransformation;
 import com.conveyal.datatools.manager.models.transform.ZipTransformation;
+import com.conveyal.gtfs.validator.ValidationResult;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.slf4j.Logger;
@@ -227,15 +228,20 @@ private String getErrorReasonMessage() {
     public String getNotificationMessage() {
         StringBuilder message = new StringBuilder();
         if (!status.error) {
-            message.append(String.format("New feed version created for %s (valid from %s - %s). ",
-                feedSource.name,
-                feedVersion.validationResult.firstCalendarDate,
-                feedVersion.validationResult.lastCalendarDate));
-            if (feedVersion.validationResult.errorCount > 0) {
-                message.append(String.format("During validation, we found %s issue(s)",
-                    feedVersion.validationResult.errorCount));
+            ValidationResult validationResult = feedVersion.validationResult;
+            if (validationResult != null) {
+                message.append(String.format("New feed version created for %s (valid from %s - %s).",
+                    feedSource.name,
+                    validationResult.firstCalendarDate,
+                    validationResult.lastCalendarDate
+                ));
+                if (validationResult.errorCount > 0) {
+                    message.append(String.format(" During validation, we found %s issue(s)", validationResult.errorCount));
+                } else {
+                    message.append(" The validation check found no issues with this new dataset!");
+                }
             } else {
-                message.append("The validation check found no issues with this new dataset!");
+                message.append(String.format("New feed version created for %s.", feedSource.name));
             }
         } else {
             // Processing did not complete. Depending on which sub-task this occurred in,

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java

@@ -9,6 +9,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.time.LocalDate;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -23,6 +24,8 @@ public class FeedMergeContext implements Closeable {
     public final boolean tripIdsMatch;
     public final LocalDate futureFirstCalendarStartDate;
     public final Set<String> sharedTripIds;
+    public Set<String> locationIds = new HashSet<>();
+
 
     public FeedMergeContext(Set<FeedVersion> feedVersions, Auth0UserProfile owner) throws IOException {
         feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner);

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java

@@ -36,6 +36,12 @@ public class MergeFeedsResult implements Serializable {
      */
     public Set<String> calendarDatesServiceIds = new HashSet<>();
 
+    /**
+     * Track various table ids for resolving foreign references.
+     */
+    public Set<String> stopIds = new HashSet<>();
+    public Set<String> locationGroupStopIds = new HashSet<>();
+
     /**
      * Track the set of route IDs to end up in the merged feed in order to determine which route_attributes
      * records should be retained in the merged result.

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java

@@ -240,6 +240,7 @@ public void startNewRow() throws IOException {
             // For this table, use all fields found in the feeds to merge.
             sharedSpecFields = List.copyOf(allFields);
         } else {
+            setForeignReferenceKeyValues();
             // Get the spec fields and custom/proprietary fields to export
             List<Field> specFields = table.specFields();
             // Filter the spec fields on the set of fields found in all feeds to be merged.
@@ -250,8 +251,24 @@ public void startNewRow() throws IOException {
     }
 
     /**
-     * Determine which reference table to use. If there is only one reference use this. If there are multiple references
-     * determine the context and then the correct reference table to use.
+     * Build a list of table key id values to be used in foreign key field look-ups.
+     */
+    private void setForeignReferenceKeyValues() {
+        switch (table.name) {
+            case "stops":
+                mergeFeedsResult.stopIds.add(getIdWithScope(keyValue));
+                break;
+            case "location_group_stops":
+                mergeFeedsResult.locationGroupStopIds.add(getIdWithScope(keyValue));
+                break;
+            default:
+                // nothing.
+        }
+    }
+
+    /**
+     * Determine which reference table to use. If there is only one reference use this. If there are multiple
+     * references, determine the context and then the correct reference table to use.
      */
     private Table getReferenceTable(FieldContext fieldContext, Field field) {
         if (field.referenceTables.size() == 1) {
@@ -266,7 +283,12 @@ private Table getReferenceTable(FieldContext fieldContext, Field field) {
                     getTableScopedValue(Table.CALENDAR, fieldContext.getValue()),
                     getTableScopedValue(Table.CALENDAR_DATES, fieldContext.getValue())
                 );
-            // Include other cases as multiple references are added e.g. flex!.
+            case STOP_TIMES_STOP_ID_KEY:
+            case LOCATION_GROUP_STOPS_STOP_ID_KEY:
+                return ReferenceTableDiscovery.getStopReferenceTable(
+                    fieldContext.getValueToWrite(),
+                    mergeFeedsResult
+                );
             default:
                 return null;
         }

src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ReferenceTableDiscovery.java

@@ -11,6 +11,9 @@ public class ReferenceTableDiscovery {
 
     public static final String REF_TABLE_SEPARATOR = "#~#";
 
+    /**
+     * Tables that have two or more foreign references.
+     */
     public enum ReferenceTableKey {
 
         TRIP_SERVICE_ID_KEY(
@@ -22,6 +25,22 @@ public enum ReferenceTableKey {
                 Table.CALENDAR_DATES.name,
                 Table.SCHEDULE_EXCEPTIONS.name
             )
+        ),
+        LOCATION_GROUP_STOPS_STOP_ID_KEY(
+            String.join(
+                REF_TABLE_SEPARATOR,
+                Table.LOCATION_GROUP_STOPS.name,
+                "stop_id",
+                Table.STOPS.name
+            )
+        ),
+        STOP_TIMES_STOP_ID_KEY(
+            String.join(
+                REF_TABLE_SEPARATOR,
+                Table.STOP_TIMES.name,
+                "stop_id",
+                Table.STOPS.name
+            )
         );
 
         private final String value;
@@ -87,4 +106,14 @@ public static Table getTripServiceIdReferenceTable(
         }
         return null;
     }
+
+    /**
+     * Defines the reference table as a stop if the field value matches a stop id.
+     */
+    public static Table getStopReferenceTable(String fieldValue, MergeFeedsResult mergeFeedsResult) {
+        if (mergeFeedsResult.stopIds.contains(fieldValue)) {
+            return Table.STOPS;
+        }
+        return null;
+    }
 }

src/main/java/com/conveyal/datatools/manager/models/FeedSource.java

@@ -18,6 +18,7 @@
 import com.conveyal.datatools.manager.utils.connections.ConnectionResponse;
 import com.conveyal.datatools.manager.utils.connections.HttpURLConnectionResponse;
 import com.conveyal.gtfs.GTFS;
+import com.conveyal.gtfs.loader.FeedLoadResult;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -180,6 +181,14 @@ public String organizationId () {
 
     public String editorNamespace;
 
+    /**
+     * If this feed source references a feed version that is GTFS Flex, this value will permanently be set to true.
+     * This is evaluated every time a feed version is retrieved as different feed versions may or may not be flex.
+     */
+    public boolean flex;
+
+    public boolean flexUIFeaturesEnabled;
+
     /**
      * Create a new feed.
      */
@@ -388,6 +397,7 @@ public FeedVersion processFetchResponse(
             String message = String.format("Fetch complete for %s", this.name);
             LOG.info(message);
             status.completeSuccessfully(message);
+            presetFlex(version);
             return version;
         }
     }
@@ -429,10 +439,16 @@ public String toString () {
      */
     @JsonIgnore
     public FeedVersion retrieveLatest() {
-        return Persistence.feedVersions.getOneFiltered(
+        FeedVersion newestVersion = Persistence.feedVersions.getOneFiltered(
             eq("feedSourceId", this.id),
             Sorts.descending("version")
         );
+        if (newestVersion == null) {
+            // Is this what happens if there are none?
+            return null;
+        }
+        presetFlex(newestVersion);
+        return newestVersion;
     }
 
     /**
@@ -448,6 +464,7 @@ public FeedVersion retrievePublishedVersion() {
             // Is this what happens if there are none?
             return null;
         }
+        presetFlex(publishedVersion);
         return publishedVersion;
     }
 
@@ -809,4 +826,19 @@ public <T extends FeedTransformation> List<T> getActiveTransformations(FeedVersi
             .flatMap(Collection::stream)
             .collect(Collectors.toList());
     }
+
+    /**
+     * If a flex feed version is loaded, set this feed source to permanently be a flex feed source. Subsequent feed
+     * versions which are not flex will have no affected.
+     */
+    public void presetFlex(FeedVersion feedVersion) {
+        boolean isFlexFeedVersion = false;
+        if (feedVersion.feedLoadResult != null) isFlexFeedVersion = feedVersion.feedLoadResult.isGTFSFlex();
+        if (!flex && isFlexFeedVersion) {
+            // If the feed version is flex (and the feed source was not flex) permanently enable flex.
+            flex = flexUIFeaturesEnabled = true;
+            Persistence.feedSources.updateField(this.id, "flexUIFeaturesEnabled", true);
+            Persistence.feedSources.updateField(this.id, "flex", true);
+        }
+    }
 }