210f783d by Nathan Lighthart

Merge branch 'context-reader'

2 parents 123f7dea c7ca8670
package ages;
import io.DataIO;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import lib.TypeFinder;
import oms3.annotations.Description;
import oms3.annotations.Execute;
import oms3.annotations.In;
......@@ -73,8 +77,11 @@ public class AgES {
private AgESParameters createParameters() {
AgESParameters.Builder builder = new AgESParameters.Builder();
Map<String, Object> parameterMap = generateParameterMap();
Map<String, Class<?>> typeMap = generateTypeMap();
// override output files as the full path is generated by OMS
AgESParameters parameters = builder.parameters(params)
AgESParameters parameters = builder.parameters(parameterMap, typeMap)
.outFile_hru(outFile_hru.toPath())
.outFile_hru_crop(outFile_hru_crop.toPath())
.outFile_hru_layer(outFile_hru_layer.toPath())
......@@ -93,4 +100,33 @@ public class AgES {
.build();
return parameters;
}
private Map<String, Object> generateParameterMap() {
Map<String, Object> parameterMap = new HashMap<>(params);
parameterMap.remove("params");
parameterMap.remove("fmt_date");
parameterMap.remove("fmt_double");
return parameterMap;
}
private Map<String, Class<?>> generateTypeMap() {
Map<String, Class<?>> typeMap = new HashMap<>();
for (String parameterName : params.keySet()) {
Map<String, String> parameterInfo = params.getInfo(parameterName);
if (parameterInfo == null) {
continue;
}
String typeName = parameterInfo.get(DataIO.KEY_TYPE);
Class<?> type = TypeFinder.findType(typeName);
if (type != null) {
typeMap.put(parameterName, type);
}
}
return typeMap;
}
}
......
package ages;
import annotations.Range;
import gov.usda.jcf.annotations.Description;
import gov.usda.jcf.annotations.Units;
import gov.usda.jcf.core.Context;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.io.BufferedReader;
import java.io.IOException;
......@@ -9,10 +13,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDate;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import gov.usda.jcf.annotations.Description;
import annotations.Range;
import gov.usda.jcf.annotations.Units;
import java.util.Set;
/**
*
......@@ -82,6 +85,15 @@ public class AgESParameters {
@Description("Sub-Surface program file path")
public final Path subSurfaceProgramFilePath;
@Description("HRU additional parameters file path")
public final Path hruAdditionalFilePath;
@Description("Reach additional parameters file path")
public final Path reachAdditionalFilePath;
@Description("Temporal additional parameters file path")
public final Path temporalAdditionalFilePath;
@Description("ID set for HRU")
public final String idSet_hru;
......@@ -645,6 +657,14 @@ public class AgESParameters {
@Description("parallel execution flag")
public final String flagParallel;
private Map<String, Parameter> additionalParameterMap;
public void addAdditionalParameters(Context context) {
for (Parameter p : additionalParameterMap.values()) {
p.addToContext(context);
}
}
private AgESParameters(Builder builder) {
// parameters
hruFilePath = builder.hruFilePath;
......@@ -674,6 +694,10 @@ public class AgESParameters {
surfaceProgramFilePath = builder.surfaceProgramFilePath;
subSurfaceProgramFilePath = builder.subSurfaceProgramFilePath;
hruAdditionalFilePath = builder.hruAdditionalFilePath;
reachAdditionalFilePath = builder.reachAdditionalFilePath;
temporalAdditionalFilePath = builder.temporalAdditionalFilePath;
idSet_hru = builder.idSet_hru;
idSet_reach = builder.idSet_reach;
outFile_hru = builder.outFile_hru;
......@@ -866,6 +890,8 @@ public class AgESParameters {
flagSort = builder.flagSort;
flagSplit = builder.flagSplit;
flagParallel = builder.flagParallel;
additionalParameterMap = builder.additionalParameterMap;
}
public static class Builder {
......@@ -897,6 +923,10 @@ public class AgESParameters {
private Path surfaceProgramFilePath;
private Path subSurfaceProgramFilePath;
private Path hruAdditionalFilePath;
private Path reachAdditionalFilePath;
private Path temporalAdditionalFilePath;
private String idSet_hru;
private String idSet_reach;
private Path outFile_hru;
......@@ -1090,6 +1120,8 @@ public class AgESParameters {
private boolean flagSplit;
private String flagParallel;
private Map<String, Parameter> additionalParameterMap = new HashMap<>();
public AgESParameters build() {
flagIrrigation = (irrigationFilePath != null && managementIrrigationFilePath != null);
return new AgESParameters(this);
......@@ -1129,6 +1161,29 @@ public class AgESParameters {
return this;
}
public Builder parameters(Map<String, Object> parameterMap, Map<String, Class<?>> typeMap) {
// handle field values
parameters(parameterMap);
// handle additional parameters
// find additional parameter names
Set<String> additionalParameters = new HashSet<>(parameterMap.keySet());
Field[] fields = Builder.class.getDeclaredFields();
for (Field field : fields) {
additionalParameters.remove(field.getName());
}
// Create Additional parameter objects
for (String parameterName : additionalParameters) {
Parameter p = new Parameter(parameterName,
parameterMap.get(parameterName),
typeMap.get(parameterName));
additionalParameterMap.put(parameterName, p);
}
return this;
}
private void setParameter(Field field, Object rawValue) throws IllegalArgumentException, IllegalAccessException {
Object value = TypeConverters.INSTANCE.convert(rawValue, field.getType());
field.setAccessible(true);
......@@ -1260,6 +1315,21 @@ public class AgESParameters {
return this;
}
public Builder hruAdditionalFilePath(Path hruAdditionalFilePath) {
this.hruAdditionalFilePath = hruAdditionalFilePath;
return this;
}
public Builder reachAdditionalFilePath(Path reachAdditionalFilePath) {
this.reachAdditionalFilePath = reachAdditionalFilePath;
return this;
}
public Builder temporalAdditionalFilePath(Path temporalAdditionalFilePath) {
this.temporalAdditionalFilePath = temporalAdditionalFilePath;
return this;
}
public Builder idSet_hru(String idSet_hru) {
this.idSet_hru = idSet_hru;
return this;
......@@ -2140,4 +2210,25 @@ public class AgESParameters {
return this;
}
}
private static class Parameter {
private String name;
private Object rawValue;
private Class<?> type;
public Parameter(String name, Object rawValue, Class<?> type) {
this.name = name;
this.rawValue = rawValue;
this.type = type;
}
public void addToContext(Context context) {
if (type == null) {
context.put(name, rawValue);
} else {
Object value = TypeConverters.INSTANCE.convert(rawValue, type);
context.put(name, value, type);
}
}
}
}
......
......@@ -37,13 +37,13 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
import lib.ArrayParser;
import lib.ArrayNameParser;
public class OutputWriter {
private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss zzz yyyy");
private final DecimalFormat numberFormat = new DecimalFormat("0.#####");
private final DecimalFormat engNumberFormat = new DecimalFormat("0.#####E0");
private final ArrayParser arrayParser;
private final ArrayNameParser arrayParser;
private Path outFilePath;
private List<String> attributes;
......@@ -61,7 +61,7 @@ public class OutputWriter {
this.outFilePath = outFilePath;
this.attributes = parseAttributes(attributes);
this.splitWriters = new HashMap<>();
arrayParser = ArrayParser.minimalParser();
arrayParser = ArrayNameParser.minimalParser();
initialized = false;
disabled = false;
......
package io;
import gov.usda.jcf.core.Context;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class SpatialParameterReader {
private Path spatialParameterFilePath;
private Map<Integer, Context> spatialContexts;
public SpatialParameterReader(Path spatialParamterFilePath, Map<Integer, Context> spatialContexts) {
this.spatialParameterFilePath = spatialParamterFilePath;
this.spatialContexts = spatialContexts;
}
public void read() throws IOException {
try {
read0();
} catch (Exception ex) {
throw new IOException("Error reading spatial parameter file: " + spatialParameterFilePath, ex);
}
}
private void read0() throws Exception {
ParameterTableReader reader = new ParameterTableReader(spatialParameterFilePath);
// Check that first column is ID
if (!"ID".equals(reader.getTable().getColumnName(1))) {
throw new RuntimeException("Invalid additional file: ID needs to be the first column");
}
// Read each line tracking which ID have been seen
Set<Integer> idSet = new HashSet<>();
int rowCount = 1;
while (reader.hasNextLine()) {
Map<String, Object> data = reader.readNextLine();
// Make sure ID object exists
Object idObject = data.get("ID");
if (idObject == null) {
throw new RuntimeException("Missing ID on data row " + rowCount);
}
// Make sure ID is an int
int id = TypeConverters.INSTANCE.convert(idObject, int.class);
// Check if ID is a duplicate
if (!idSet.add(id)) {
throw new RuntimeException("Duplicate id: " + id);
}
// Retrieve the context associated with the ID
Context spatialContext = spatialContexts.get(id);
if (spatialContext == null) {
throw new RuntimeException("Unknown id: " + id);
}
// Add the parameter values and metadata to the context
for (Map.Entry<String, Object> entry : data.entrySet()) {
String parameterName = entry.getKey();
// ID should be skipped as it is not a parameter
if ("ID".equals(parameterName)) {
continue;
}
// Add the parameter to the context
spatialContext.put(parameterName, entry.getValue(), reader.getType(parameterName));
// Add the units to the context if it exists
String units = reader.getUnits(parameterName);
if (units != null) {
spatialContext.putProperty(parameterName, "units", units);
}
}
++rowCount;
}
// verify that id sets are equal
// this makes sure that all entities got all parameters
if (!idSet.equals(spatialContexts.keySet())) {
throw new RuntimeException("Parameter file ID set does not match spatial ID set");
}
}
}
package io;
import gov.usda.jcf.core.Context;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.io.IOException;
import java.nio.file.Path;
import java.time.LocalDate;
import java.util.Map;
public class TemporalParameterReader {
private Path temporalParameterFilePath;
private Context temporalContext;
private ParameterTableReader reader;
private int rowCount;
public TemporalParameterReader(Path temporalParameterFilePath, Context temporalContext) {
this.temporalParameterFilePath = temporalParameterFilePath;
this.temporalContext = temporalContext;
reader = null;
}
public void read() throws IOException {
try {
read0();
} catch (Exception ex) {
throw new IOException("Error reading temporal parameter file: " + temporalParameterFilePath, ex);
}
}
private void read0() throws Exception {
if (reader == null) {
readFirstDay();
} else {
readNextDay();
}
}
private void readFirstDay() throws Exception {
reader = new ParameterTableReader(temporalParameterFilePath);
if (!"date".equals(reader.getTable().getColumnName(1))) {
throw new RuntimeException("Invalid additional file: date needs to be the first column");
}
LocalDate currentModelTime = getCurrentModelTime();
LocalDate dataFileTime = null;
rowCount = 1;
Map<String, Object> data;
do {
data = reader.readNextLine();
// retrieve actual time form data file
LocalDate actualTime = getActualTime(data);
if (dataFileTime == null) {
// no previous time so it is the first
// verify that actual time is before or equal to current model time
if (actualTime.isAfter(currentModelTime)) {
throw new RuntimeException("Start date in file is after the model start date: "
+ actualTime + " model time: " + currentModelTime);
}
dataFileTime = actualTime;
} else {
// previous time is known so verify that day is not skipped
LocalDate expectedTime = dataFileTime.plusDays(1L);
if (actualTime.equals(expectedTime)) {
// time in the data file is the next day as expected
dataFileTime = expectedTime;
} else {
// time is not what is expected so malformed file
throw new RuntimeException(getUnexpectedTimeString(expectedTime, actualTime));
}
}
++rowCount;
} while (!currentModelTime.equals(dataFileTime));
// current model time found
// so add to context
readIntoContext(data);
}
private void readNextDay() throws Exception {
LocalDate currentModelTime = getCurrentModelTime();
Map<String, Object> data = reader.readNextLine();
LocalDate actualTime = getActualTime(data);
if (currentModelTime.equals(actualTime)) {
// time is as expected so add data to context
readIntoContext(data);
} else {
// time is not what is expected so malformed file
throw new RuntimeException(getUnexpectedTimeString(currentModelTime, actualTime));
}
++rowCount;
}
private LocalDate getCurrentModelTime() {
return temporalContext.get("time", LocalDate.class);
}
private LocalDate getActualTime(Map<String, Object> data) {
Object timeObject = data.get("date");
if (timeObject == null) {
throw new RuntimeException("Missing time on data row " + rowCount);
}
return TypeConverters.INSTANCE.convert(timeObject, LocalDate.class);
}
private void readIntoContext(Map<String, Object> data) {
// Add the parameter values and metadata to the context
for (Map.Entry<String, Object> entry : data.entrySet()) {
String parameterName = entry.getKey();
// ID should be skipped as it is not a parameter
if ("date".equals(parameterName)) {
continue;
}
// Add the parameter to the context
temporalContext.put(parameterName, entry.getValue(), reader.getType(parameterName));
// Add the units to the context if it exists
String units = reader.getUnits(parameterName);
if (units != null) {
temporalContext.putProperty(parameterName, "units", units);
}
}
}
private String getUnexpectedTimeString(LocalDate expectedTime, LocalDate actualTime) {
return "Unexpected time occured on data row "
+ rowCount + ": " + actualTime + " Expected: " + expectedTime;
}
}
......@@ -11,7 +11,7 @@ import java.util.regex.Pattern;
*
* @author Nathan Lighthart
*/
public class ArrayParser {
public class ArrayNameParser {
private static final String ARRAY_NAME_MINIMAL_PATTERN_STRING = "([^\\[]+)";
private static final String ARRAY_NAME_JAVA_PATTERN_STRING = "([a-zA-Z_$][a-zA-Z0-9_$]*)";
private static final String ARRAY_INDEX_PATTERN_STRING = "\\[([0-9]+)\\]";
......@@ -53,8 +53,8 @@ public class ArrayParser {
*
* @return the array parser
*/
public static ArrayParser minimalParser() {
return new ArrayParser(MINIMAL_ARRAY_PATTERN);
public static ArrayNameParser minimalParser() {
return new ArrayNameParser(MINIMAL_ARRAY_PATTERN);
}
/**
......@@ -63,8 +63,8 @@ public class ArrayParser {
*
* @return the array parser
*/
public static ArrayParser javaParser() {
return new ArrayParser(JAVA_ARRAY_PATTERN);
public static ArrayNameParser javaParser() {
return new ArrayNameParser(JAVA_ARRAY_PATTERN);
}
/**
......@@ -72,7 +72,7 @@ public class ArrayParser {
*
* @param arrayPattern the main array pattern
*/
private ArrayParser(Pattern arrayPattern) {
private ArrayNameParser(Pattern arrayPattern) {
m_arrayPattern = arrayPattern;
}
......
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package lib;
import gov.usda.jcf.util.conversion.TypeConverters;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.List;
/**
*
* @author Nathan.Lighthart
*/
public class ArrayValueParser {
public static <T> T parseArray(String s, Class<T> clazz) {
List<Object> list = parseArrayList(s);
T array = convertToArray(list, clazz);
return array;
}
private static List<Object> parseArrayList(String s) {
if (s == null || "".equals(s)) {
return new ArrayList<>();
}
final char[] chars = s.toCharArray();
if (chars[0] != '[') {
throw new IllegalArgumentException("Invalid array syntax: missing starting '['");
}
List<Object> list = new ArrayList<>();
parseList(chars, 1, list);
return list;
}
private static <T> T convertToArray(List<?> list, Class<T> clazz) {
Class<?> componentType = clazz.getComponentType();
Object array = Array.newInstance(componentType, list.size());
for (int i = 0; i < list.size(); ++i) {
Array.set(array, i, convertValue(list.get(i), componentType));
}
return clazz.cast(array);
}
private static <T> T convertValue(Object value, Class<T> clazz) {
if (clazz.isArray()) {
return convertToArray((List<?>) value, clazz);
} else {
return TypeConverters.INSTANCE.convert(value, clazz);
}
}
private static int parseList(final char[] chars, int index, List<Object> list) {
// empty list
if (chars[index] == ']') {
return index + 1;
} else {
CharLoop:
while (index < chars.length) {
index = parseValue(chars, index, list);
switch (chars[index]) {
case ',':
++index;
while (Character.isWhitespace(chars[index])) {
++index;
}
break;
case ']':
break CharLoop;
default:
throw new IllegalArgumentException("Error parsing at: " + index);
}
}
}
return index;
}
private static int parseValue(final char[] chars, int index, List<Object> list) {
if (chars[index] == '[') {
List<Object> innerList = new ArrayList<>();
list.add(innerList);
index = parseList(chars, index + 1, innerList);
++index;
} else {
int startIndex = index;
while (index < chars.length
&& chars[index] != ','
&& chars[index] != ']') {
++index;
}
if (index == chars.length) {
throw new IllegalArgumentException("Invalid array syntax: missing: missing end ]");
}
String value = new String(chars, startIndex, index - startIndex);
list.add(value);
}
return index;
}
}
package lib;
import java.io.File;
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZonedDateTime;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* Static utility methods that help finding the type based on the type name.
*
* @author Nathan.Lighthart
*/
public class TypeFinder {
/**
* Unmodifiable mapping of common simple type names.
*/
private static final Map<String, Class<?>> SIMPLE_TYPES = createSimpleTypes();
/**
* Creates map of common simple type names.
*
* @return an unmodifiable map
*/
private static Map<String, Class<?>> createSimpleTypes() {
Map<String, Class<?>> simpleTypes = new HashMap<>();
// primitive data types
simpleTypes.put("byte", Byte.TYPE);
simpleTypes.put("short", Short.TYPE);
simpleTypes.put("int", Integer.TYPE);
simpleTypes.put("long", Long.TYPE);
simpleTypes.put("float", Float.TYPE);
simpleTypes.put("double", Double.TYPE);
simpleTypes.put("boolean", Boolean.TYPE);
simpleTypes.put("char", Character.TYPE);
// common alternative names for primitive data types
simpleTypes.put("integer", Integer.TYPE);
simpleTypes.put("real", Double.TYPE);
simpleTypes.put("bool", Boolean.TYPE);
simpleTypes.put("character", Character.TYPE);
// wrapper types
simpleTypes.put("Byte", Byte.TYPE);
simpleTypes.put("Short", Short.TYPE);
simpleTypes.put("Integer", Integer.TYPE);
simpleTypes.put("Long", Long.TYPE);
simpleTypes.put("Float", Float.TYPE);
simpleTypes.put("Double", Double.TYPE);
simpleTypes.put("Boolean", Boolean.TYPE);
simpleTypes.put("Character", Character.TYPE);
// common object types
simpleTypes.put("String", String.class);
simpleTypes.put("BigInteger", BigInteger.class);
simpleTypes.put("BigDecimal", BigDecimal.class);
simpleTypes.put("Date", Date.class);
simpleTypes.put("Calendar", Calendar.class);
simpleTypes.put("File", File.class);
simpleTypes.put("Path", Path.class);
simpleTypes.put("Instant", Instant.class);
simpleTypes.put("ZonedDateTime", ZonedDateTime.class);
simpleTypes.put("OffsetDateTime", OffsetDateTime.class);
simpleTypes.put("LocalDateTime", LocalDateTime.class);
simpleTypes.put("LocalDate", LocalDate.class);
simpleTypes.put("LocalTime", LocalTime.class);
// common alternatives for object types
simpleTypes.put("string", String.class);
return Collections.unmodifiableMap(simpleTypes);
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is also
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup
* @return the type represented by this name or {@code null} if type was not
* found
*/
public static Class<?> findType(String typeName) {
if (typeName == null || "".equals(typeName) || "-".equals(typeName)) {
return null;
} else {
return findType0(typeName);
}
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is also
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findType0(String typeName) {
// Count the number of dimensions that the array contains
int count = 0;
for (int i = typeName.length() - 1; i > 0; i -= 2) {
if (typeName.charAt(i - 1) == '[' && typeName.charAt(i) == ']') {
++count;
} else {
break;
}
}
// type is not an array
if (count == 0) {
return findBasicType(typeName);
}
// type is array so find the component type
String componentTypeName = typeName.substring(0, typeName.length() - (2 * count));
Class<?> componentType = findBasicType(componentTypeName);
if (componentType == null) {
// component type could not be found so array type cannot be found
return null;
}
// convert component type to array type with the same number of dimensions
Class<?> arrayType = Array.newInstance(componentType, new int[count]).getClass();
return arrayType;
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names and fully qualified class names. This method is not
* aware of array types. This method is not aware of generic types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findBasicType(String typeName) {
Class<?> type = findSimpleType(typeName);
if (type == null) {
type = lookupClass(typeName);
}
return type;
}
/**
* Returns the type for the specified type name. This method is able to
* determine types of primitive values. This method aware of a few common
* simple class names. This method is not aware of fully qualified class
* names. This method is not aware of array types. This method is not aware
* of generic types.
*
* @param typeName the type name to lookup
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> findSimpleType(String typeName) {
return SIMPLE_TYPES.get(typeName);
}
/**
* Returns the type for the specified type name. This method is not able to
* determine types of primitive values. This method is not aware of simple
* class names. This method is aware of fully qualified class names. This
* method is not aware of array types. This method is not aware of generic
* types.
*
* @param typeName the type name to lookup (cannot be null)
* @return the type represented by this name or {@code null} if type was not
* found
*/
private static Class<?> lookupClass(String typeName) {
try {
return Class.forName(typeName);
} catch (ClassNotFoundException ex) {
return null;
}
}
}
......@@ -13,7 +13,7 @@ import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import lib.ArrayParser;
import lib.ArrayNameParser;
/**
*
......@@ -22,10 +22,10 @@ import lib.ArrayParser;
public class CatchmentAggregator {
private final Set<String> attributes;
private final Set<String> weightedAttributes;
private final ArrayParser arrayParser;
private final ArrayNameParser arrayParser;
public CatchmentAggregator(String attributes, String weightedAttributes) {
arrayParser = ArrayParser.minimalParser();
arrayParser = ArrayNameParser.minimalParser();
this.attributes = parseAttributes(attributes);
this.weightedAttributes = new HashSet<>(parseAttributes(weightedAttributes));
......
......@@ -10,7 +10,7 @@ import java.lang.reflect.Array;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import lib.ArrayParser;
import lib.ArrayNameParser;
/**
*
......@@ -18,10 +18,10 @@ import lib.ArrayParser;
*/
public class HRUAreaWeigher {
private final Set<String> weightedAttributes;
private final ArrayParser arrayParser;