feat(core): add ScenarioFileFormat SPI for alternative file formats

matsim/src/main/java/org/matsim/core/controler/corelisteners/DumpDataAtEndImpl.java

@@ -28,7 +28,7 @@
 import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.network.Network;
 import org.matsim.api.core.v01.population.Population;
-import org.matsim.api.core.v01.population.PopulationWriter;
+import org.matsim.core.population.PopulationUtils;
 import org.matsim.core.config.Config;
 import org.matsim.core.config.ConfigGroup;
 import org.matsim.core.config.ConfigUtils;
@@ -372,11 +372,9 @@ private void dumpNetwork() {
 	}
 
 	private void dumpPlans() {
-		// dump plans
-
-		final PopulationWriter writer = new PopulationWriter(this.population, this.network);
-		writer.putAttributeConverters(this.attributeConverters);
-		writer.write(this.controlerIO.getOutputFilename(Controler.DefaultFiles.population));
+		PopulationUtils.writePopulation(this.population,
+				this.controlerIO.getOutputFilename(Controler.DefaultFiles.population),
+				this.attributeConverters);
 	}
 
 }

matsim/src/main/java/org/matsim/core/controler/corelisteners/PlansDumpingImpl.java

@@ -23,9 +23,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.matsim.analysis.IterationStopWatch;
-import org.matsim.api.core.v01.network.Network;
 import org.matsim.api.core.v01.population.Population;
-import org.matsim.api.core.v01.population.PopulationWriter;
+import org.matsim.core.population.PopulationUtils;
 import org.matsim.core.config.Config;
 import org.matsim.core.config.groups.ControllerConfigGroup;
 import org.matsim.core.controler.Controler;
@@ -54,7 +53,6 @@ final class PlansDumpingImpl implements PlansDumping, BeforeMobsimListener {
 	static final private Logger log = LogManager.getLogger(PlansDumpingImpl.class);
 
 	@Inject private Config config;
-	@Inject private Network network;
 	@Inject private Population population;
 	@Inject private IterationStopWatch stopwatch;
 	@Inject private OutputDirectoryHierarchy controlerIO;
@@ -79,8 +77,9 @@ public void notifyBeforeMobsim(final BeforeMobsimEvent event) {
 			final String inputCRS = config.plans().getInputCRS();
 			final String internalCRS = config.global().getCoordinateSystem();
 
+			final String filename = controlerIO.getIterationFilename(event.getIteration(), Controler.DefaultFiles.population);
 			if ( inputCRS == null ) {
-				new PopulationWriter(population, network).write(controlerIO.getIterationFilename(event.getIteration(), Controler.DefaultFiles.population));
+				PopulationUtils.writePopulation(population, filename);
 			}
 			else {
 				log.info( "re-projecting population from "+internalCRS+" back to "+inputCRS+" for export" );
@@ -90,7 +89,8 @@ public void notifyBeforeMobsim(final BeforeMobsimEvent event) {
 								internalCRS,
 								inputCRS );
 
-				new PopulationWriter(transformation, population, network).write(controlerIO.getIterationFilename(event.getIteration(), Controler.DefaultFiles.population));
+				PopulationUtils.writePopulation(population, filename,
+						java.util.Collections.emptyMap(), transformation);
 			}
 			log.info("finished plans dump.");
 			stopwatch.endOperation("dump all plans");

matsim/src/main/java/org/matsim/core/population/PopulationUtils.java

@@ -45,12 +45,16 @@
 import org.matsim.core.router.TripStructureUtils;
 import org.matsim.core.router.TripStructureUtils.StageActivityHandling;
 import org.matsim.core.scenario.MutableScenario;
+import org.matsim.core.scenario.ScenarioFileFormat;
+import org.matsim.core.scenario.ScenarioFileFormatRegistry;
 import org.matsim.core.scenario.ScenarioUtils;
+import org.matsim.core.utils.geometry.CoordinateTransformation;
 import org.matsim.core.utils.geometry.CoordUtils;
 import org.matsim.core.utils.io.IOUtils;
 import org.matsim.core.utils.misc.OptionalTime;
 import org.matsim.facilities.ActivityFacilities;
 import org.matsim.facilities.ActivityFacility;
+import org.matsim.utils.objectattributes.AttributeConverter;
 import org.matsim.utils.objectattributes.attributable.Attributes;
 import org.matsim.utils.objectattributes.attributable.AttributesUtils;
 import org.matsim.vehicles.Vehicle;
@@ -1183,7 +1187,27 @@ public static void printPlansCount(StreamingPopulationReader reader) {
 	}
 
 	public static void writePopulation(Population population, String filename) {
-		new PopulationWriter(population).write(filename);
+		writePopulation(population, filename, Collections.emptyMap());
+	}
+
+	public static void writePopulation(Population population, String filename,
+									   Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		writePopulation(population, filename, attributeConverters, null);
+	}
+
+	public static void writePopulation(Population population, String filename,
+									   Map<Class<?>, AttributeConverter<?>> attributeConverters,
+									   CoordinateTransformation coordinateTransformation) {
+		Optional<ScenarioFileFormat> provider = ScenarioFileFormatRegistry.getProvider(filename);
+		if (provider.isPresent()) {
+			provider.get().writePopulation(population, filename, attributeConverters);
+		} else {
+			PopulationWriter writer = coordinateTransformation != null
+					? new PopulationWriter(coordinateTransformation, population)
+					: new PopulationWriter(population);
+			writer.putAttributeConverters(attributeConverters);
+			writer.write(filename);
+		}
 	}
 
 	public static Id<Link> decideOnLinkIdForActivity(Activity act, Scenario sc) {
@@ -1273,10 +1297,12 @@ public static void sampleDown(Population pop, double sampleTo) {
 	public static void readPopulation(Population population, String filename) {
 		MutableScenario scenario = ScenarioUtils.createMutableScenario(ConfigUtils.createConfig());
 		scenario.setPopulation(population);
-		new PopulationReader(scenario).readFile(filename);
-		// (yyyy population reader uses network to retrofit some missing geo information such as route lenth.
-		// In my opinion, that should be done in prepareForSim, not in the parser.  It is commented as such
-		// in the PopulationReader class.  kai, nov'18)
+		Optional<ScenarioFileFormat> provider = ScenarioFileFormatRegistry.getProvider(filename);
+		if (provider.isPresent()) {
+			provider.get().readPopulation(IOUtils.getFileUrl(filename), scenario, null, null, Collections.emptyMap());
+		} else {
+			new PopulationReader(scenario).readFile(filename);
+		}
 	}
 
 	public static Population readPopulation(String filename) {

matsim/src/main/java/org/matsim/core/scenario/ScenarioFileFormat.java

@@ -0,0 +1,70 @@
+package org.matsim.core.scenario;
+
+import org.matsim.api.core.v01.Scenario;
+import org.matsim.api.core.v01.population.Population;
+import org.matsim.utils.objectattributes.AttributeConverter;
+
+import java.net.URL;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * SPI for alternative scenario file formats (e.g. protobuf, parquet).
+ * Implementations are discovered via {@link java.util.ServiceLoader}.
+ * <p>
+ * To register a provider, add a file
+ * {@code META-INF/services/org.matsim.core.scenario.ScenarioFileFormat}
+ * containing the fully qualified class name of the implementation.
+ * <p>
+ * A provider declares which file extensions it handles. When {@link ScenarioLoaderImpl}
+ * encounters a file with a matching extension, it delegates to the provider instead of
+ * using the built-in XML readers.
+ * <p>
+ * All methods have default implementations that throw {@link UnsupportedOperationException},
+ * so providers only need to implement the methods for the types they support.
+ *
+ * @author nkuehnel / MOIA
+ */
+public interface ScenarioFileFormat {
+
+	/**
+	 * @return file extensions this provider handles (without leading dot), e.g. "pb", "pbf", "parquet"
+	 */
+	Set<String> getSupportedExtensions();
+
+	default void readPopulation(URL url, Scenario scenario, String inputCRS, String targetCRS,
+								Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		throw new UnsupportedOperationException("Population reading not supported by " + getClass().getName());
+	}
+
+	default void writePopulation(Population population, String filename,
+								 Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		throw new UnsupportedOperationException("Population writing not supported by " + getClass().getName());
+	}
+
+	default void readNetwork(URL url, Scenario scenario, String inputCRS, String targetCRS,
+							 Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		throw new UnsupportedOperationException("Network reading not supported by " + getClass().getName());
+	}
+
+	default void readFacilities(URL url, Scenario scenario, String inputCRS, String targetCRS,
+								Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		throw new UnsupportedOperationException("Facilities reading not supported by " + getClass().getName());
+	}
+
+	default void readVehicles(URL url, Scenario scenario) {
+		throw new UnsupportedOperationException("Vehicles reading not supported by " + getClass().getName());
+	}
+
+	default void readTransitSchedule(URL url, Scenario scenario, String inputCRS, String targetCRS) {
+		throw new UnsupportedOperationException("Transit schedule reading not supported by " + getClass().getName());
+	}
+
+	default void readTransitVehicles(URL url, Scenario scenario) {
+		throw new UnsupportedOperationException("Transit vehicles reading not supported by " + getClass().getName());
+	}
+
+	default void readHouseholds(URL url, Scenario scenario, Map<Class<?>, AttributeConverter<?>> attributeConverters) {
+		throw new UnsupportedOperationException("Households reading not supported by " + getClass().getName());
+	}
+}

matsim/src/main/java/org/matsim/core/scenario/ScenarioFileFormatRegistry.java

@@ -0,0 +1,58 @@
+package org.matsim.core.scenario;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.ServiceLoader;
+
+/**
+ * Discovers {@link ScenarioFileFormat} implementations via {@link ServiceLoader}
+ * and resolves the appropriate provider for a given filename.
+ *
+ * @author nkuehnel / MOIA
+ */
+public final class ScenarioFileFormatRegistry {
+
+	private static final Logger log = LogManager.getLogger(ScenarioFileFormatRegistry.class);
+
+	private static final Map<String, ScenarioFileFormat> providers = new HashMap<>();
+
+	static {
+		for (ScenarioFileFormat provider : ServiceLoader.load(ScenarioFileFormat.class)) {
+			for (String ext : provider.getSupportedExtensions()) {
+				providers.put(ext, provider);
+				log.info("Registered ScenarioFileFormat provider for extension '." + ext + "': " + provider.getClass().getName());
+			}
+		}
+	}
+
+	private ScenarioFileFormatRegistry() {
+	}
+
+	/**
+	 * Find a provider for the given filename based on its extension.
+	 * Handles double extensions like "population.pb.zst" by stripping compression suffixes first.
+	 */
+	public static Optional<ScenarioFileFormat> getProvider(String filename) {
+		return getEffectiveExtension(filename).map(providers::get);
+	}
+
+	static Optional<String> getEffectiveExtension(String filename) {
+		int lastDot = filename.lastIndexOf('.');
+		if (lastDot < 0) {
+			return Optional.empty();
+		}
+		String ext = filename.substring(lastDot + 1);
+		if (ext.equals("zst") || ext.equals("gz") || ext.equals("bz2")) {
+			String inner = filename.substring(0, lastDot);
+			int innerDot = inner.lastIndexOf('.');
+			if (innerDot >= 0) {
+				ext = inner.substring(innerDot + 1);
+			}
+		}
+		return Optional.of(ext);
+	}
+}

matsim/src/main/java/org/matsim/core/scenario/ScenarioLoaderImpl.java

@@ -54,6 +54,7 @@
 import java.util.*;
 
 import static org.matsim.core.config.groups.PlansConfigGroup.PERSON_ATTRIBUTES_DEPRECATION_MESSAGE;
+import static org.matsim.core.scenario.ScenarioFileFormatRegistry.getProvider;
 
 /**
  * Loads elements of Scenario from file. Non standardized elements
@@ -134,14 +135,16 @@ private void loadNetwork() {
 			URL networkUrl = this.config.network().getInputFileURL(this.config.getContext());
 			log.info("loading network from " + networkUrl);
 			String inputCRS = config.network().getInputCRS();
-
-			MatsimNetworkReader reader =
-					new MatsimNetworkReader(
-							inputCRS,
-							config.global().getCoordinateSystem(),
-							this.scenario.getNetwork());
-            reader.putAttributeConverters( attributeConverters );
-            reader.parse(networkUrl);
+			String targetCRS = config.global().getCoordinateSystem();
+
+			Optional<ScenarioFileFormat> provider = getProvider(this.config.network().getInputFile());
+			if (provider.isPresent()) {
+				provider.get().readNetwork(networkUrl, this.scenario, inputCRS, targetCRS, attributeConverters);
+			} else {
+				MatsimNetworkReader reader = new MatsimNetworkReader(inputCRS, targetCRS, this.scenario.getNetwork());
+				reader.putAttributeConverters(attributeConverters);
+				reader.parse(networkUrl);
+			}
 
 			if ((this.config.network().getChangeEventsInputFile()!= null) && this.config.network().isTimeVariantNetwork()) {
 				log.info("loading network change events from " + this.config.network().getChangeEventsInputFileUrl(this.config.getContext()).getFile());
@@ -162,9 +165,14 @@ private void loadActivityFacilities() {
 			final String inputCRS = config.facilities().getInputCRS();
 			final String internalCRS = config.global().getCoordinateSystem();
 
-            MatsimFacilitiesReader reader = new MatsimFacilitiesReader(inputCRS, internalCRS, this.scenario.getActivityFacilities());
-            reader.putAttributeConverters(attributeConverters);
-            reader.parse(facilitiesFileName);
+			Optional<ScenarioFileFormat> provider = getProvider(this.config.facilities().getInputFile());
+			if (provider.isPresent()) {
+				provider.get().readFacilities(facilitiesFileName, this.scenario, inputCRS, internalCRS, attributeConverters);
+			} else {
+				MatsimFacilitiesReader reader = new MatsimFacilitiesReader(inputCRS, internalCRS, this.scenario.getActivityFacilities());
+				reader.putAttributeConverters(attributeConverters);
+				reader.parse(facilitiesFileName);
+			}
 
 			log.info("loaded " + this.scenario.getActivityFacilities().getFacilities().size() + " facilities from " + facilitiesFileName);
 		}
@@ -195,12 +203,17 @@ private void loadPopulation() {
 			URL populationFileName = this.config.plans().getInputFileURL(this.config.getContext());
 			log.info("loading population from " + populationFileName);
 
-            final String targetCRS = config.global().getCoordinateSystem();
+			final String targetCRS = config.global().getCoordinateSystem();
 			final String internalCRS = config.global().getCoordinateSystem();
 
-            final PopulationReader reader = new PopulationReader(targetCRS, internalCRS, this.scenario);
-            reader.putAttributeConverters( attributeConverters );
-            reader.parse( populationFileName );
+			Optional<ScenarioFileFormat> provider = getProvider(this.config.plans().getInputFile());
+			if (provider.isPresent()) {
+				provider.get().readPopulation(populationFileName, this.scenario, targetCRS, internalCRS, attributeConverters);
+			} else {
+				final PopulationReader reader = new PopulationReader(targetCRS, internalCRS, this.scenario);
+				reader.putAttributeConverters(attributeConverters);
+				reader.parse(populationFileName);
+			}
 
 			PopulationUtils.printPlansCount(this.scenario.getPopulation()) ;
 		}
@@ -249,9 +262,15 @@ private void loadHouseholds() {
 		if ( (this.config.households() != null) && (this.config.households().getInputFile() != null) ) {
 			URL householdsFile = this.config.households().getInputFileURL(this.config.getContext());
 			log.info("loading households from " + householdsFile);
-			HouseholdsReaderV10 reader = new HouseholdsReaderV10(this.scenario.getHouseholds());
-			reader.putAttributeConverters(this.attributeConverters);
-			reader.parse(householdsFile);
+
+			Optional<ScenarioFileFormat> provider = getProvider(this.config.households().getInputFile());
+			if (provider.isPresent()) {
+				provider.get().readHouseholds(householdsFile, this.scenario, attributeConverters);
+			} else {
+				HouseholdsReaderV10 reader = new HouseholdsReaderV10(this.scenario.getHouseholds());
+				reader.putAttributeConverters(this.attributeConverters);
+				reader.parse(householdsFile);
+			}
 			log.info("households loaded.");
 		}
 		else {
@@ -286,7 +305,12 @@ private void loadTransit() throws UncheckedIOException {
 			final String inputCRS = config.transit().getInputScheduleCRS();
 			final String internalCRS = config.global().getCoordinateSystem();
 
-            new TransitScheduleReader( inputCRS, internalCRS, this.scenario).readURL(transitScheduleFile );
+			Optional<ScenarioFileFormat> provider = getProvider(this.config.transit().getTransitScheduleFile());
+			if (provider.isPresent()) {
+				provider.get().readTransitSchedule(transitScheduleFile, this.scenario, inputCRS, internalCRS);
+			} else {
+				new TransitScheduleReader(inputCRS, internalCRS, this.scenario).readURL(transitScheduleFile);
+			}
 		}
 		else {
 			log.info("no transit schedule file set in config, not loading any transit schedule");
@@ -327,7 +351,14 @@ private void loadTransitVehicles() throws UncheckedIOException {
 		final String vehiclesFile = this.config.transit().getVehiclesFile();
 		if ( vehiclesFile != null ) {
 			log.info("loading transit vehicles from " + vehiclesFile);
-			new MatsimVehicleReader(this.scenario.getTransitVehicles()).readURL(this.config.transit().getVehiclesFileURL(this.config.getContext() ) );
+			URL transitVehiclesUrl = this.config.transit().getVehiclesFileURL(this.config.getContext());
+
+			Optional<ScenarioFileFormat> provider = getProvider(vehiclesFile);
+			if (provider.isPresent()) {
+				provider.get().readTransitVehicles(transitVehiclesUrl, this.scenario);
+			} else {
+				new MatsimVehicleReader(this.scenario.getTransitVehicles()).readURL(transitVehiclesUrl);
+			}
 		}
 		else {
 			log.info("no transit vehicles file set in config, not loading any transit vehicles");
@@ -337,7 +368,14 @@ private void loadVehicles() throws UncheckedIOException {
 		final String vehiclesFile = this.config.vehicles().getVehiclesFile();
 		if ( vehiclesFile != null ) {
 			log.info("loading vehicles from " + vehiclesFile );
-			new MatsimVehicleReader(this.scenario.getVehicles()).readURL(IOUtils.extendUrl(this.config.getContext(), vehiclesFile ) );
+			URL vehiclesUrl = IOUtils.extendUrl(this.config.getContext(), vehiclesFile);
+
+			Optional<ScenarioFileFormat> provider = getProvider(vehiclesFile);
+			if (provider.isPresent()) {
+				provider.get().readVehicles(vehiclesUrl, this.scenario);
+			} else {
+				new MatsimVehicleReader(this.scenario.getVehicles()).readURL(vehiclesUrl);
+			}
 		}
 		else {
 			log.info("no vehicles file set in config, not loading any vehicles");