e888c146 by Nathan Lighthart

Add the ability to read additional entity parameters

- Add generic parameter table reader that is capable of reading the additional file format
- Add spatial parameter reader that extends the parameter table reader to read the file into the spatial contexts
- Update model parameters to accept file paths for the additional files
- Update model to read additional files
- Add example additional files to arti_catch
1 parent f4cff065
......@@ -41,6 +41,7 @@ import io.ParameterOverrideReader;
import io.RegionalizationCacher;
import io.RegionalizedClimateReader;
import io.SoilsParameterReader;
import io.SpatialParameterReader;
import io.StationClimateReader;
import io.StationReader;
import io.TmeanStationClimateReader;
......@@ -218,6 +219,7 @@ public class AgESModel {
loadManagement();
readStations();
initializeContexts();
addAdditionalParameters();
addParameterOverrides();
createPrograms();
initializeClimateReaders();
......@@ -573,6 +575,26 @@ public class AgESModel {
outletAggregateContext = new CombinedContext(new MapContext(), temporalContext, agesContext);
}
private void addAdditionalParameters() throws IOException {
addAdditionalHRUParameters();
addAdditionalReachParameters();
}
private void addAdditionalHRUParameters() throws IOException {
if (parameters.hruAdditionalFilePath != null) {
SpatialParameterReader reader = new SpatialParameterReader(parameters.hruAdditionalFilePath, hruContexts);
reader.read();
}
}
private void addAdditionalReachParameters() throws IOException {
// need to check flag reach routing as reaches may not exist
if (parameters.reachAdditionalFilePath != null && parameters.flagReachRouting) {
SpatialParameterReader reader = new SpatialParameterReader(parameters.reachAdditionalFilePath, reachContexts);
reader.read();
}
}
private void addParameterOverrides() throws IOException {
if (parameters.hruOverrideFilePath == null) {
return;
......
package ages;
import annotations.Range;
import gov.usda.jcf.annotations.Description;
import gov.usda.jcf.annotations.Units;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.io.BufferedReader;
import java.io.IOException;
......@@ -10,9 +13,6 @@ import java.nio.file.Path;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.Map;
import gov.usda.jcf.annotations.Description;
import annotations.Range;
import gov.usda.jcf.annotations.Units;
/**
*
......@@ -82,6 +82,12 @@ public class AgESParameters {
@Description("Sub-Surface program file path")
public final Path subSurfaceProgramFilePath;
@Description("HRU additional parameters file path")
public final Path hruAdditionalFilePath;
@Description("Reach additional parameters file path")
public final Path reachAdditionalFilePath;
@Description("ID set for HRU")
public final String idSet_hru;
......@@ -674,6 +680,9 @@ public class AgESParameters {
surfaceProgramFilePath = builder.surfaceProgramFilePath;
subSurfaceProgramFilePath = builder.subSurfaceProgramFilePath;
hruAdditionalFilePath = builder.hruAdditionalFilePath;
reachAdditionalFilePath = builder.reachAdditionalFilePath;
idSet_hru = builder.idSet_hru;
idSet_reach = builder.idSet_reach;
outFile_hru = builder.outFile_hru;
......@@ -897,6 +906,9 @@ public class AgESParameters {
private Path surfaceProgramFilePath;
private Path subSurfaceProgramFilePath;
private Path hruAdditionalFilePath;
private Path reachAdditionalFilePath;
private String idSet_hru;
private String idSet_reach;
private Path outFile_hru;
......@@ -1260,6 +1272,16 @@ public class AgESParameters {
return this;
}
public Builder hruAdditionalFilePath(Path hruAdditionalFilePath) {
this.hruAdditionalFilePath = hruAdditionalFilePath;
return this;
}
public Builder reachAdditionalFilePath(Path reachAdditionalFilePath) {
this.reachAdditionalFilePath = reachAdditionalFilePath;
return this;
}
public Builder idSet_hru(String idSet_hru) {
this.idSet_hru = idSet_hru;
return this;
......
......@@ -37,13 +37,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import lib.ArrayParser;
import lib.ArrayNameParser;
public class OutputWriter {
private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy");
private final DecimalFormat numberFormat = new DecimalFormat("0.#####");
private final DecimalFormat engNumberFormat = new DecimalFormat("0.#####E0");
private final ArrayParser arrayParser;
private final ArrayNameParser arrayParser;
private Path outFilePath;
private List<String> attributes;
......@@ -61,7 +61,7 @@ public class OutputWriter {
this.outFilePath = outFilePath;
this.attributes = parseAttributes(attributes);
this.splitWriters = new HashMap<>();
arrayParser = ArrayParser.minimalParser();
arrayParser = ArrayNameParser.minimalParser();
initialized = false;
disabled = false;
......
package io;
import gov.usda.jcf.core.Context;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class SpatialParameterReader {
private Path spatialParameterFilePath;
private Map<Integer, Context> spatialContexts;
public SpatialParameterReader(Path spatialParamterFilePath, Map<Integer, Context> spatialContexts) {
this.spatialParameterFilePath = spatialParamterFilePath;
this.spatialContexts = spatialContexts;
}
public void read() throws IOException {
try {
read0();
} catch (Exception ex) {
throw new IOException("Error reading spatial parameter file: " + spatialParameterFilePath, ex);
}
}
private void read0() throws Exception {
ParameterTableReader reader = new ParameterTableReader(spatialParameterFilePath);
// Check that first column is ID
if (!"ID".equals(reader.getTable().getColumnName(1))) {
throw new RuntimeException("Invalid additional file: ID needs to be the first column");
}
// Read each line tracking which ID have been seen
Set<Integer> idSet = new HashSet<>();
int rowCount = 1;
while (reader.hasNextLine()) {
Map<String, Object> data = reader.readNextLine();
// Make sure ID object exists
Object idObject = data.get("ID");
if (idObject == null) {
throw new RuntimeException("Missing ID on data row " + rowCount);
}
// Make sure ID is an int
int id = TypeConverters.INSTANCE.convert(idObject, int.class);
// Check if ID is a duplicate
if (!idSet.add(id)) {
throw new RuntimeException("Duplicate id: " + id);
}
// Retrieve the context associated with the ID
Context spatialContext = spatialContexts.get(id);
if (spatialContext == null) {
throw new RuntimeException("Unknown id: " + id);
}
// Add the parameter values and metadata to the context
for (Map.Entry<String, Object> entry : data.entrySet()) {
String parameterName = entry.getKey();
// ID should be skipped as it is not a parameter
if ("ID".equals(parameterName)) {
continue;
}
// Add the parameter to the context
spatialContext.put(parameterName, entry.getValue(), reader.getType(parameterName));
// Add the units to the context if it exists
String units = reader.getUnits(parameterName);
if (units != null) {
spatialContext.putProperty(parameterName, "units", units);
}
}
++rowCount;
}
// verify that id sets are equal
// this makes sure that all entities got all parameters
if (!idSet.equals(spatialContexts.keySet())) {
throw new RuntimeException("Parameter file ID set does not match spatial ID set");
}
}
}
......@@ -11,7 +11,7 @@ import java.util.regex.Pattern;
*
* @author Nathan Lighthart
*/
public class ArrayParser {
public class ArrayNameParser {
private static final String ARRAY_NAME_MINIMAL_PATTERN_STRING = "([^\\[]+)";
private static final String ARRAY_NAME_JAVA_PATTERN_STRING = "([a-zA-Z_$][a-zA-Z0-9_$]*)";
private static final String ARRAY_INDEX_PATTERN_STRING = "\\[([0-9]+)\\]";
......@@ -53,8 +53,8 @@ public class ArrayParser {
*
* @return the array parser
*/
public static ArrayParser minimalParser() {
return new ArrayParser(MINIMAL_ARRAY_PATTERN);
public static ArrayNameParser minimalParser() {
return new ArrayNameParser(MINIMAL_ARRAY_PATTERN);
}
/**
......@@ -63,8 +63,8 @@ public class ArrayParser {
*
* @return the array parser
*/
public static ArrayParser javaParser() {
return new ArrayParser(JAVA_ARRAY_PATTERN);
public static ArrayNameParser javaParser() {
return new ArrayNameParser(JAVA_ARRAY_PATTERN);
}
/**
......@@ -72,7 +72,7 @@ public class ArrayParser {
*
* @param arrayPattern the main array pattern
*/
private ArrayParser(Pattern arrayPattern) {
private ArrayNameParser(Pattern arrayPattern) {
m_arrayPattern = arrayPattern;
}
......
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package lib;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.List;
/**
*
* @author Nathan.Lighthart
*/
public class ArrayValueParser {
public static <T> T parseArray(String s, Class<T> clazz) {
List<Object> list = parseArrayList(s);
T array = convertToArray(list, clazz);
return array;
}
private static List<Object> parseArrayList(String s) {
if (s == null || "".equals(s)) {
return new ArrayList<>();
}
final char[] chars = s.toCharArray();
if (chars[0] != '[') {
throw new IllegalArgumentException("Invalid array syntax: missing starting '['");
}
List<Object> list = new ArrayList<>();
parseList(chars, 1, list);
return list;
}
private static <T> T convertToArray(List<?> list, Class<T> clazz) {
Class<?> componentType = clazz.getComponentType();
Object array = Array.newInstance(componentType, list.size());
for (int i = 0; i < list.size(); ++i) {
Array.set(array, i, convertValue(list.get(i), componentType));
}
return clazz.cast(array);
}
private static <T> T convertValue(Object value, Class<T> clazz) {
if (clazz.isArray()) {
return convertToArray((List<?>) value, clazz);
} else {
return TypeConverters.INSTANCE.convert(value, clazz);
}
}
private static int parseList(final char[] chars, int index, List<Object> list) {
// empty list
if (chars[index] == ']') {
return index + 1;
} else {
CharLoop:
while (index < chars.length) {
index = parseValue(chars, index, list);
switch (chars[index]) {
case ',':
++index;
while (Character.isWhitespace(chars[index])) {
++index;
}
break;
case ']':
break CharLoop;
default:
throw new IllegalArgumentException("Error parsing at: " + index);
}
}
}
return index;
}
private static int parseValue(final char[] chars, int index, List<Object> list) {
if (chars[index] == '[') {
List<Object> innerList = new ArrayList<>();
list.add(innerList);
index = parseList(chars, index + 1, innerList);
++index;
} else {
int startIndex = index;
while (index < chars.length
&& chars[index] != ','
&& chars[index] != ']') {
++index;
}
if (index == chars.length) {
throw new IllegalArgumentException("Invalid array syntax: missing: missing end ]");
}
String value = new String(chars, startIndex, index - startIndex);
list.add(value);
}
return index;
}
}
package lib;
import java.io.File;
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZonedDateTime;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* Static utility methods that help finding the type based on the type name.
*
* @author Nathan.Lighthart
*/
public class TypeFinder {
/**
* Unmodifiable mapping of common simple type names.
*/
private static final Map<String, Class<?>> SIMPLE_TYPES = createSimpleTypes();
/**
* Creates map of common simple type names.
*
* @return an unmodifiable map
*/
private static Map<String, Class<?>> createSimpleTypes() {
Map<String, Class<?>> simpleTypes = new HashMap<>();
// primitive data types
simpleTypes.put("byte", Byte.TYPE);
simpleTypes.put("short", Short.TYPE);
simpleTypes.put("int", Integer.TYPE);
simpleTypes.put("long", Long.TYPE);
simpleTypes.put("float", Float.TYPE);
simpleTypes.put("double", Double.TYPE);
simpleTypes.put("boolean", Boolean.TYPE);
simpleTypes.put("char", Character.TYPE);
// common alternative names for primitive data types
simpleTypes.put("integer", Integer.TYPE);
simpleTypes.put("real", Double.TYPE);
simpleTypes.put("bool", Boolean.TYPE);
simpleTypes.put("character", Character.TYPE);
// wrapper types
simpleTypes.put("Byte", Byte.TYPE);
simpleTypes.put("Short", Short.TYPE);
simpleTypes.put("Integer", Integer.TYPE);
simpleTypes.put("Long", Long.TYPE);
simpleTypes.put("Float", Float.TYPE);
simpleTypes.put("Double", Double.TYPE);
simpleTypes.put("Boolean", Boolean.TYPE);
simpleTypes.put("Character", Character.TYPE);
// common object types
simpleTypes.put("String", String.class);
simpleTypes.put("BigInteger", BigInteger.class);
simpleTypes.put("BigDecimal", BigDecimal.class);
simpleTypes.put("Date", Date.class);
simpleTypes.put("Calendar", Calendar.class);
simpleTypes.put("File", File.class);
simpleTypes.put("Path", Path.class);
simpleTypes.put("Instant", Instant.class);
simpleTypes.put("ZonedDateTime", ZonedDateTime.class);
simpleTypes.put("OffsetDateTime", OffsetDateTime.class);
simpleTypes.put("LocalDateTime", LocalDateTime.class);
simpleTypes.put("LocalDate", LocalDate.class);
simpleTypes.put("LocalTime", LocalTime.class);
// common alternatives for object types
simpleTypes.put("string", String.class);
return Collections.unmodifiableMap(simpleTypes);
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is also
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup
* @return the type represented by this name or {@code null} if type was not
* found
*/
public static Class<?> findType(String typeName) {
if (typeName == null || "".equals(typeName) || "-".equals(typeName)) {
return null;
} else {
return findType0(typeName);
}
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is also
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findType0(String typeName) {
// Count the number of dimensions that the array contains
int count = 0;
for (int i = typeName.length() - 1; i > 0; i -= 2) {
if (typeName.charAt(i - 1) == '[' && typeName.charAt(i) == ']') {
++count;
} else {
break;
}
}
// type is not an array
if (count == 0) {
return findBasicType(typeName);
}
// type is array so find the component type
String componentTypeName = typeName.substring(0, typeName.length() - (2 * count));
Class<?> componentType = findBasicType(componentTypeName);
if (componentType == null) {
// component type could not be found so array type cannot be found
return null;
}
// convert component type to array type with the same number of dimensions
Class<?> arrayType = Array.newInstance(componentType, new int[count]).getClass();
return arrayType;
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is not
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findBasicType(String typeName) {
Class<?> type = findSimpleType(typeName);
if (type == null) {
type = lookupClass(typeName);
}
return type;
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names. This method is not aware of fully qualified class
* names. This method is not aware of array types. This method is not aware
* of generic types.
*
* @param typeName the type name to lookup
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findSimpleType(String typeName) {
return SIMPLE_TYPES.get(typeName);
}
/**
* Returns the type for the specified type name. This method is not able to
* determine types of primitive values. This method is not aware of simple
* class names. This method is aware of fully qualified class names. This
* method is not aware of array types. This method is not aware of generic
* types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> lookupClass(String typeName) {
try {
return Class.forName(typeName);
} catch (ClassNotFoundException ex) {
return null;
}
}
}
......@@ -13,7 +13,7 @@ import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import lib.ArrayParser;
import lib.ArrayNameParser;
/**
*
......@@ -22,10 +22,10 @@ import lib.ArrayParser;
public class CatchmentAggregator {
private final Set<String> attributes;
private final Set<String> weightedAttributes;
private final ArrayParser arrayParser;
private final ArrayNameParser arrayParser;
public CatchmentAggregator(String attributes, String weightedAttributes) {
arrayParser = ArrayParser.minimalParser();
arrayParser = ArrayNameParser.minimalParser();
this.attributes = parseAttributes(attributes);
this.weightedAttributes = new HashSet<>(parseAttributes(weightedAttributes));
......
......@@ -10,7 +10,7 @@ import java.lang.reflect.Array;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lib.ArrayParser;
import lib.ArrayNameParser;
/**
*
......@@ -18,10 +18,10 @@ import lib.ArrayParser;
*/
public class HRUAreaWeigher {
private final Set<String> weightedAttributes;
private final ArrayParser arrayParser;
private final ArrayNameParser arrayParser;
public HRUAreaWeigher(String weightedAttributes) {
arrayParser = ArrayParser.minimalParser();
arrayParser = ArrayNameParser.minimalParser();
this.weightedAttributes = parseAttributes(weightedAttributes);
}
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!