From 3e0141cdac7b62435ab9c8948c2902bd82bc76d5 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Tue, 11 Jun 2019 19:10:32 -0400 Subject: [PATCH 01/10] Add ETo AdjustmentMethod --- .../EToAdjustmentMethod.spec.ts | 43 ++++ .../adjustmentMethods/EToAdjustmentMethod.ts | 224 ++++++++++++++++++ routes/weather.spec.ts | 21 +- routes/weatherProviders/WeatherProvider.ts | 12 + test/etoTest.json | 42 ++++ 5 files changed, 334 insertions(+), 8 deletions(-) create mode 100644 routes/adjustmentMethods/EToAdjustmentMethod.spec.ts create mode 100644 routes/adjustmentMethods/EToAdjustmentMethod.ts create mode 100644 test/etoTest.json diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts b/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts new file mode 100644 index 0000000..4698cc6 --- /dev/null +++ b/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts @@ -0,0 +1,43 @@ +import * as moment from "moment"; +import { expect } from "chai"; +import { GeoCoordinates } from "../../types"; +import { calculateETo, EToData } from "./EToAdjustmentMethod"; + + +const testData: TestData[] = require( "../../test/etoTest.json" ); + +describe( "ETo AdjustmentMethod", () => { + describe( "Should correctly calculate ETo", async () => { + for ( const locationData of testData ) { + it( "Using data from " + locationData.description, async () => { + let date = moment.unix( locationData.startTimestamp ); + for ( const entry of locationData.entries ) { + const etoData: EToData = { + ...entry.data, + precip: 0, + timestamp: date.unix(), + weatherProvider: "mock" + }; + const calculatedETo = calculateETo( etoData, locationData.elevation, locationData.coordinates ); + // Allow a small margin of error for rounding, unit conversions, and approximations. + expect( calculatedETo ).approximately( entry.eto, 0.003 ); + + date = date.add( 1, "days" ); + } + } ); + } + } ); +} ); + +interface TestData { + description: string; + source: string; + startTimestamp: number; + elevation: number; + coordinates: GeoCoordinates; + entries: { + eto: number, + /** This is not actually full EToData - it is missing `timestamp`, `weatherProvider`, and `precip`. */ + data: EToData + }[]; +} diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts new file mode 100644 index 0000000..47e0980 --- /dev/null +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -0,0 +1,224 @@ +import * as SunCalc from "suncalc"; +import * as moment from "moment"; +import { AdjustmentMethod, AdjustmentMethodResponse, AdjustmentOptions } from "./AdjustmentMethod"; +import { GeoCoordinates, WateringData, WeatherProviderId } from "../../types"; +import { WeatherProvider } from "../weatherProviders/WeatherProvider"; + + +/** + * Calculates how much watering should be scaled based on weather and adjustment options by comparing the recent + * potential ETo to the baseline potential ETo that the watering program was designed for. + */ +async function calculateEToWateringScale( + adjustmentOptions: EToScalingAdjustmentOptions, + wateringData: WateringData | undefined, + coordinates: GeoCoordinates, + weatherProvider: WeatherProvider +): Promise< AdjustmentMethodResponse > { + + // Temporarily disabled since OWM forecast data is checking if rain is forecasted for 3 hours in the future. + /* + if ( wateringData && wateringData.raining ) { + return { + scale: 0, + rawData: { raining: 1 } + } + } + */ + + // This will throw an error message if ETo data cannot be retrieved. + const etoData: EToData = await weatherProvider.getEToData( coordinates ); + + let baseETo: number; + // Default elevation is based on data from https://www.pnas.org/content/95/24/14009. + let elevation = 600; + + if ( adjustmentOptions && "baseETo" in adjustmentOptions ) { + baseETo = adjustmentOptions.baseETo + } else { + throw "A baseline potential ETo must be provided."; + } + + if ( adjustmentOptions && "elevation" in adjustmentOptions ) { + elevation = adjustmentOptions.elevation; + } + + const eto: number = calculateETo( etoData, elevation, coordinates ); + + const scale = Math.floor( Math.min( Math.max( 0, ( eto - etoData.precip ) / baseETo * 100 ), 200 ) ); + return { + scale: scale, + rawData: { + eto: Math.round( eto * 1000) / 1000, + radiation: Math.round( etoData.solarRadiation * 100) / 100 + } + } +} + +/* The implementation of this algorithm was guided by a step-by-step breakdown + (http://edis.ifas.ufl.edu/pdffiles/ae/ae45900.pdf) */ +/** + * Calculates the reference potential evapotranspiration using the Penman-Monteith (FAO-56) method + * (http://www.fao.org/3/X0490E/x0490e07.htm). + * + * @param etoData The data to calculate the ETo with. + * @param elevation The elevation above sea level of the watering site (in feet). + * @param coordinates The coordinates of the watering site. + * @return The reference potential evapotranspiration (in inches per day). + */ +export function calculateETo( etoData: EToData, elevation: number, coordinates: GeoCoordinates ): number { + // Convert to Celsius. + const minTemp = ( etoData.minTemp - 32 ) * 5 / 9; + const maxTemp = ( etoData.maxTemp - 32 ) * 5 / 9; + // Convert to meters. + elevation = elevation / 3.281; + // Convert to meters per second. + const windSpeed = etoData.windSpeed / 2.237; + + const avgTemp = ( maxTemp + minTemp ) / 2; + + const saturationVaporPressureCurveSlope = 4098 * 0.6108 * Math.exp( 17.27 * avgTemp / ( avgTemp + 237.3 ) ) / Math.pow( avgTemp + 237.3, 2 ); + + const pressure = 101.3 * Math.pow( ( 293 - 0.0065 * elevation ) / 293, 5.26 ); + + const psychrometricConstant = 0.000665 * pressure; + + const deltaTerm = saturationVaporPressureCurveSlope / ( saturationVaporPressureCurveSlope + psychrometricConstant * ( 1 + 0.34 * windSpeed ) ); + + const psiTerm = psychrometricConstant / ( saturationVaporPressureCurveSlope + psychrometricConstant * ( 1 + 0.34 * windSpeed ) ); + + const tempTerm = ( 900 / ( avgTemp + 273 ) ) * windSpeed; + + const minSaturationVaporPressure = 0.6108 * Math.exp( 17.27 * minTemp / ( minTemp + 237.3 ) ); + + const maxSaturationVaporPressure = 0.6108 * Math.exp( 17.27 * maxTemp / ( maxTemp + 237.3 ) ); + + const avgSaturationVaporPressure = ( minSaturationVaporPressure + maxSaturationVaporPressure ) / 2; + + const actualVaporPressure = ( minSaturationVaporPressure * etoData.maxHumidity / 100 + maxSaturationVaporPressure * etoData.minHumidity / 100 ) / 2; + + const dayOfYear = moment.unix( etoData.timestamp ).dayOfYear(); + + const inverseRelativeEarthSunDistance = 1 + 0.033 * Math.cos( 2 * Math.PI / 365 * dayOfYear ); + + const solarDeclination = 0.409 * Math.sin( 2 * Math.PI / 365 * dayOfYear - 1.39 ); + + const latitudeRads = Math.PI / 180 * coordinates[ 0 ]; + + const sunsetHourAngle = Math.acos( -Math.tan( latitudeRads ) * Math.tan( solarDeclination ) ); + + const extraterrestrialRadiation = 24 * 60 / Math.PI * 0.082 * inverseRelativeEarthSunDistance * ( sunsetHourAngle * Math.sin( latitudeRads ) * Math.sin( solarDeclination ) + Math.cos( latitudeRads ) * Math.cos( solarDeclination ) * Math.sin( sunsetHourAngle ) ); + + const clearSkyRadiation = ( 0.75 + 2e-5 * elevation ) * extraterrestrialRadiation; + + const solarRadiation = etoData.solarRadiation; + + const netShortWaveRadiation = ( 1 - 0.23 ) * solarRadiation; + + const netOutgoingLongWaveRadiation = 4.903e-9 * ( Math.pow( maxTemp + 273.16, 4 ) + Math.pow( minTemp + 273.16, 4 ) ) / 2 * ( 0.34 - 0.14 * Math.sqrt( actualVaporPressure ) ) * ( 1.35 * solarRadiation / clearSkyRadiation - 0.35); + + const netRadiation = netShortWaveRadiation - netOutgoingLongWaveRadiation; + + const radiationTerm = deltaTerm * 0.408 * netRadiation; + + const windTerm = psiTerm * tempTerm * ( avgSaturationVaporPressure - actualVaporPressure ); + + return ( windTerm + radiationTerm ) / 25.4; +} + +/** + * Approximates the wind speed at 2 meters using the wind speed measured at another height. + * @param speed The wind speed measured at the specified height (in miles per hour). + * @param height The height of the measurement (in feet). + * @returns The approximate wind speed at 2 meters (in miles per hour). + */ +export function standardizeWindSpeed( speed: number, height: number ) { + return speed * 4.87 / Math.log( 67.8 * height / 3.281 - 5.42 ); +} + +// The time at which the formula for clear sky isolation will start/stop yielding a non-negative result. +SunCalc.addTime( Math.asin( 30 / 990 ) * 180 / Math.PI, "radiationStart", "radiationEnd" ); + +/** + * Approximates total solar radiation for a day given cloud coverage information using a formula from + * http://www.shodor.org/os411/courses/_master/tools/calculators/solarrad/ + * @param cloudCoverInfo Information about the cloud coverage for several periods that span the entire day. + * @param coordinates The coordinates of the location the data is from. + * @return The total solar radiation for the day (in megajoules per square meter per day). + */ +export function approximateSolarRadiation(cloudCoverInfo: CloudCoverInfo[], coordinates: GeoCoordinates ): number { + return cloudCoverInfo.reduce( ( total, window: CloudCoverInfo ) => { + const radiationStart: moment.Moment = moment( SunCalc.getTimes( window.endTime.toDate(), coordinates[ 0 ], coordinates[ 1 ])[ "radiationStart" ] ); + const radiationEnd: moment.Moment = moment( SunCalc.getTimes( window.startTime.toDate(), coordinates[ 0 ], coordinates[ 1 ])[ "radiationEnd" ] ); + + // Clamp the start and end times of the window within time when the sun was emitting significant radiation. + const startTime: moment.Moment = radiationStart.isAfter( window.startTime ) ? radiationStart : window.startTime; + const endTime: moment.Moment = radiationEnd.isBefore( window.endTime ) ? radiationEnd: window.endTime; + + // The length of the window that will actually be used (in hours). + const windowLength = ( endTime.unix() - startTime.unix() ) / 60 / 60; + + // Skip the window if there is no significant radiation during the time period. + if ( windowLength <= 0 ) { + return total; + } + + const startPosition = SunCalc.getPosition( startTime.toDate(), coordinates[ 0 ], coordinates[ 1 ] ); + const endPosition = SunCalc.getPosition( endTime.toDate(), coordinates[ 0 ], coordinates[ 1 ] ); + const solarElevationAngle = ( startPosition.altitude + endPosition.altitude ) / 2; + + // Calculate radiation and convert from watts to megajoules. + const clearSkyIsolation = ( 990 * Math.sin( solarElevationAngle ) - 30 ) * 0.0036 * windowLength; + + return total + clearSkyIsolation * ( 1 - 0.75 * Math.pow( window.cloudCover, 3.4 ) ); + }, 0 ); +} + +export interface EToScalingAdjustmentOptions extends AdjustmentOptions { + /** The watering site's height above sea level (in feet). */ + elevation?: number; + /** Baseline potential ETo (in inches per day). */ + baseETo?: number; +} + +/** Data about the cloud coverage for a period of time. */ +export interface CloudCoverInfo { + /** The start of this period of time. */ + startTime: moment.Moment; + /** The end of this period of time. */ + endTime: moment.Moment; + /** The average fraction of the sky covered by clouds during this time period. */ + cloudCover: number; +} + +/** + * Data used to calculate ETo. This data should be taken from a 24 hour time window. + */ +export interface EToData { + /** The WeatherProvider that generated this data. */ + weatherProvider: WeatherProviderId; + /** The Unix epoch seconds timestamp of the start of this 24 hour time window. */ + timestamp: number; + /** The minimum temperature over the time period (in Fahrenheit). */ + minTemp: number; + /** The maximum temperature over the time period (in Fahrenheit). */ + maxTemp: number; + /** The minimum relative humidity over the time period (as a percentage). */ + minHumidity: number; + /** The maximum relative humidity over the time period (as a percentage). */ + maxHumidity: number; + /** The solar radiation, accounting for cloud coverage (in megajoules per square meter per day). */ + solarRadiation: number; + /** + * The average wind speed measured at 2 meters over the time period (in miles per hour). A measurement taken at a + * different height can be standardized to 2m using the `standardizeWindSpeed` function in EToAdjustmentMethod. + */ + windSpeed: number; + /** The total precipitation over the time period (in inches). */ + precip: number; +} + +const EToAdjustmentMethod: AdjustmentMethod = { + calculateWateringScale: calculateEToWateringScale +}; +export default EToAdjustmentMethod; diff --git a/routes/weather.spec.ts b/routes/weather.spec.ts index 36e92e5..de6bea1 100644 --- a/routes/weather.spec.ts +++ b/routes/weather.spec.ts @@ -7,6 +7,7 @@ import * as MockDate from 'mockdate'; import { getWateringData } from './weather'; import { GeoCoordinates, WateringData, WeatherData } from "../types"; import { WeatherProvider } from "./weatherProviders/WeatherProvider"; +import { EToData } from "./adjustmentMethods/EToAdjustmentMethod"; const expected = require( '../test/expected.json' ); const replies = require( '../test/replies.json' ); @@ -78,16 +79,19 @@ export class MockWeatherProvider extends WeatherProvider { } public async getWateringData( coordinates: GeoCoordinates ): Promise< WateringData > { - const data = this.mockData.wateringData; - if ( !data.weatherProvider ) { - data.weatherProvider = "mock"; - } - - return data; + return await this.getData( "wateringData" ) as WateringData; } public async getWeatherData( coordinates: GeoCoordinates ): Promise< WeatherData > { - const data = this.mockData.weatherData; + return await this.getData( "weatherData" ) as WeatherData; + } + + public async getEToData( coordinates: GeoCoordinates ): Promise< EToData > { + return await this.getData( "etoData" ) as EToData; + } + + private async getData( type: "wateringData" | "weatherData" | "etoData" ) { + const data = this.mockData[ type ]; if ( !data.weatherProvider ) { data.weatherProvider = "mock"; } @@ -98,5 +102,6 @@ export class MockWeatherProvider extends WeatherProvider { interface MockWeatherData { wateringData?: WateringData, - weatherData?: WeatherData + weatherData?: WeatherData, + etoData?: EToData } diff --git a/routes/weatherProviders/WeatherProvider.ts b/routes/weatherProviders/WeatherProvider.ts index 4826df2..e8ea210 100644 --- a/routes/weatherProviders/WeatherProvider.ts +++ b/routes/weatherProviders/WeatherProvider.ts @@ -1,4 +1,5 @@ import { GeoCoordinates, WateringData, WeatherData } from "../../types"; +import { EToData } from "../adjustmentMethods/EToAdjustmentMethod"; export class WeatherProvider { /** @@ -22,4 +23,15 @@ export class WeatherProvider { getWeatherData( coordinates : GeoCoordinates ): Promise< WeatherData > { throw "Selected WeatherProvider does not support getWeatherData"; } + + /** + * Retrieves the data necessary for calculating potential ETo. + * @param coordinates The coordinates to retrieve the data for. + * @return A Promise that will be resolved with the EToData if it is successfully retrieved, + * or rejected with an error message if an error occurs while retrieving the EToData or the WeatherProvider does + * not support this method. + */ + getEToData( coordinates: GeoCoordinates ): Promise< EToData > { + throw "Selected WeatherProvider does not support getEToData"; + }; } diff --git a/test/etoTest.json b/test/etoTest.json new file mode 100644 index 0000000..7519b82 --- /dev/null +++ b/test/etoTest.json @@ -0,0 +1,42 @@ +[ + { + "description": "Badgerys Creek, AU for May 2019", + "source": "http://www.bom.gov.au/watl/eto/tables/nsw/badgerys_creek/badgerys_creek-201905.csv", + "elevation": 266, + "coordinates": [ -33.90, 150.73 ], + "startTimestamp": 1556668800, + "entries": [ + {"eto":0.075,"data":{"maxTemp":76.46,"minTemp":55.04,"maxHumidity":100,"minHumidity":58,"windSpeed":2.309,"solarRadiation":10.4}}, + {"eto":0.063,"data":{"maxTemp":77,"minTemp":56.84,"maxHumidity":100,"minHumidity":63,"windSpeed":1.707,"solarRadiation":8.66}}, + {"eto":0.035,"data":{"maxTemp":68.36,"minTemp":56.84,"maxHumidity":100,"minHumidity":91,"windSpeed":2.309,"solarRadiation":4.27}}, + {"eto":0.11,"data":{"maxTemp":72.86,"minTemp":58.46,"maxHumidity":100,"minHumidity":36,"windSpeed":5.254,"solarRadiation":12.15}}, + {"eto":0.098,"data":{"maxTemp":69.44,"minTemp":48.56,"maxHumidity":96,"minHumidity":46,"windSpeed":6.324,"solarRadiation":10.61}}, + {"eto":0.098,"data":{"maxTemp":70.16,"minTemp":47.84,"maxHumidity":97,"minHumidity":39,"windSpeed":4.551,"solarRadiation":13.68}}, + {"eto":0.075,"data":{"maxTemp":71.42,"minTemp":39.74,"maxHumidity":100,"minHumidity":37,"windSpeed":2.259,"solarRadiation":13.56}}, + {"eto":0.114,"data":{"maxTemp":68.36,"minTemp":41.36,"maxHumidity":99,"minHumidity":34,"windSpeed":6.676,"solarRadiation":12.96}}, + {"eto":0.063,"data":{"maxTemp":68.72,"minTemp":36.32,"maxHumidity":99,"minHumidity":36,"windSpeed":1.673,"solarRadiation":13.14}}, + {"eto":0.071,"data":{"maxTemp":65.66,"minTemp":41.18,"maxHumidity":100,"minHumidity":43,"windSpeed":3.999,"solarRadiation":6.76}}, + {"eto":0.13,"data":{"maxTemp":69.08,"minTemp":42.08,"maxHumidity":78,"minHumidity":38,"windSpeed":7.88,"solarRadiation":12.99}}, + {"eto":0.071,"data":{"maxTemp":71.6,"minTemp":38.48,"maxHumidity":99,"minHumidity":35,"windSpeed":2.158,"solarRadiation":12.98}}, + {"eto":0.067,"data":{"maxTemp":73.04,"minTemp":38.84,"maxHumidity":100,"minHumidity":51,"windSpeed":2.326,"solarRadiation":12.49}}, + {"eto":0.079,"data":{"maxTemp":75.74,"minTemp":43.52,"maxHumidity":100,"minHumidity":33,"windSpeed":2.242,"solarRadiation":12.75}}, + {"eto":0.067,"data":{"maxTemp":72.68,"minTemp":44.42,"maxHumidity":100,"minHumidity":45,"windSpeed":1.991,"solarRadiation":12.62}}, + {"eto":0.067,"data":{"maxTemp":71.6,"minTemp":44.06,"maxHumidity":100,"minHumidity":47,"windSpeed":2.326,"solarRadiation":12.47}}, + {"eto":0.071,"data":{"maxTemp":73.94,"minTemp":43.16,"maxHumidity":100,"minHumidity":45,"windSpeed":2.393,"solarRadiation":12.28}}, + {"eto":0.071,"data":{"maxTemp":73.4,"minTemp":45.5,"maxHumidity":100,"minHumidity":50,"windSpeed":2.56,"solarRadiation":12.3}}, + {"eto":0.063,"data":{"maxTemp":73.22,"minTemp":51.44,"maxHumidity":100,"minHumidity":51,"windSpeed":2.342,"solarRadiation":10.02}}, + {"eto":0.055,"data":{"maxTemp":74.12,"minTemp":46.58,"maxHumidity":100,"minHumidity":51,"windSpeed":1.69,"solarRadiation":9.74}}, + {"eto":0.067,"data":{"maxTemp":78.44,"minTemp":44.06,"maxHumidity":100,"minHumidity":43,"windSpeed":1.723,"solarRadiation":11.84}}, + {"eto":0.071,"data":{"maxTemp":77.36,"minTemp":47.3,"maxHumidity":100,"minHumidity":40,"windSpeed":2.125,"solarRadiation":11.76}}, + {"eto":0.063,"data":{"maxTemp":74.48,"minTemp":53.06,"maxHumidity":100,"minHumidity":53,"windSpeed":1.991,"solarRadiation":11.43}}, + {"eto":0.059,"data":{"maxTemp":73.58,"minTemp":44.42,"maxHumidity":100,"minHumidity":48,"windSpeed":2.008,"solarRadiation":11.19}}, + {"eto":0.087,"data":{"maxTemp":77.9,"minTemp":42.8,"maxHumidity":100,"minHumidity":26,"windSpeed":2.828,"solarRadiation":11.78}}, + {"eto":0.091,"data":{"maxTemp":72.68,"minTemp":44.24,"maxHumidity":92,"minHumidity":29,"windSpeed":3.865,"solarRadiation":9.89}}, + {"eto":0.13,"data":{"maxTemp":66.02,"minTemp":39.74,"maxHumidity":82,"minHumidity":35,"windSpeed":9.905,"solarRadiation":8.73}}, + {"eto":0.106,"data":{"maxTemp":65.66,"minTemp":37.58,"maxHumidity":69,"minHumidity":31,"windSpeed":5.739,"solarRadiation":11.56}}, + {"eto":0.161,"data":{"maxTemp":65.48,"minTemp":47.66,"maxHumidity":52,"minHumidity":31,"windSpeed":10.859,"solarRadiation":10.79}}, + {"eto":0.102,"data":{"maxTemp":60.08,"minTemp":36.68,"maxHumidity":70,"minHumidity":31,"windSpeed":6.743,"solarRadiation":11.42}}, + {"eto":0.087,"data":{"maxTemp":68,"minTemp":34.34,"maxHumidity":82,"minHumidity":34,"windSpeed":4.149,"solarRadiation":11.34}} + ] + } +] From 69e8d64f41f9ecd3a6e1a6d6ba88fa9791b627ad Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Wed, 12 Jun 2019 11:02:10 -0400 Subject: [PATCH 02/10] Register ETo AdjustmentMethod --- routes/weather.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/routes/weather.ts b/routes/weather.ts index 1401c77..27e24a6 100644 --- a/routes/weather.ts +++ b/routes/weather.ts @@ -11,6 +11,7 @@ import { AdjustmentMethod, AdjustmentMethodResponse, AdjustmentOptions } from ". import ManualAdjustmentMethod from "./adjustmentMethods/ManualAdjustmentMethod"; import ZimmermanAdjustmentMethod from "./adjustmentMethods/ZimmermanAdjustmentMethod"; import RainDelayAdjustmentMethod from "./adjustmentMethods/RainDelayAdjustmentMethod"; +import EToAdjustmentMethod from "./adjustmentMethods/EToAdjustmentMethod"; const weatherProvider: WeatherProvider = new ( require("./weatherProviders/" + ( process.env.WEATHER_PROVIDER || "OWM" ) ).default )(); // Define regex filters to match against location @@ -26,7 +27,8 @@ const filters = { const ADJUSTMENT_METHOD: { [ key: number ] : AdjustmentMethod } = { 0: ManualAdjustmentMethod, 1: ZimmermanAdjustmentMethod, - 2: RainDelayAdjustmentMethod + 2: RainDelayAdjustmentMethod, + 3: EToAdjustmentMethod }; /** @@ -187,6 +189,11 @@ export const getWateringData = async function( req: express.Request, res: expres // the string is split against a comma and the first value is selected remoteAddress = remoteAddress.split( "," )[ 0 ]; + if ( !adjustmentMethod ) { + res.send( "Error: Unknown AdjustmentMethod ID" ); + return; + } + // Parse weather adjustment options try { From 009fa90ec8416ab6336e731fa415983c96154a61 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Mon, 17 Jun 2019 22:24:06 -0400 Subject: [PATCH 03/10] Improve clarity --- routes/adjustmentMethods/EToAdjustmentMethod.spec.ts | 2 +- routes/adjustmentMethods/EToAdjustmentMethod.ts | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts b/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts index 4698cc6..c180f4e 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.spec.ts @@ -15,7 +15,7 @@ describe( "ETo AdjustmentMethod", () => { const etoData: EToData = { ...entry.data, precip: 0, - timestamp: date.unix(), + periodStartTime: date.unix(), weatherProvider: "mock" }; const calculatedETo = calculateETo( etoData, locationData.elevation, locationData.coordinates ); diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts index 47e0980..679cef9 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -97,7 +97,7 @@ export function calculateETo( etoData: EToData, elevation: number, coordinates: const actualVaporPressure = ( minSaturationVaporPressure * etoData.maxHumidity / 100 + maxSaturationVaporPressure * etoData.minHumidity / 100 ) / 2; - const dayOfYear = moment.unix( etoData.timestamp ).dayOfYear(); + const dayOfYear = moment.unix( etoData.periodStartTime ).dayOfYear(); const inverseRelativeEarthSunDistance = 1 + 0.033 * Math.cos( 2 * Math.PI / 365 * dayOfYear ); @@ -136,7 +136,12 @@ export function standardizeWindSpeed( speed: number, height: number ) { return speed * 4.87 / Math.log( 67.8 * height / 3.281 - 5.42 ); } -// The time at which the formula for clear sky isolation will start/stop yielding a non-negative result. +/* For hours where the Sun is too low to emit significant radiation, the formula for clear sky isolation will yield a + * negative value. "radiationStart" marks the times of day when the Sun will rise high for solar isolation formula to + * become positive, and "radiationEnd" marks the time of day when the Sun sets low enough that the equation will yield + * a negative result. For any times outside of these ranges, the formula will yield incorrect results (they should be + * clamped at 0 instead of being negative). + */ SunCalc.addTime( Math.asin( 30 / 990 ) * 180 / Math.PI, "radiationStart", "radiationEnd" ); /** @@ -198,7 +203,7 @@ export interface EToData { /** The WeatherProvider that generated this data. */ weatherProvider: WeatherProviderId; /** The Unix epoch seconds timestamp of the start of this 24 hour time window. */ - timestamp: number; + periodStartTime: number; /** The minimum temperature over the time period (in Fahrenheit). */ minTemp: number; /** The maximum temperature over the time period (in Fahrenheit). */ From f68a9812572f72c88ad78f9249b6712b6a0f2e4e Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Tue, 18 Jun 2019 10:46:51 -0400 Subject: [PATCH 04/10] Change radiation units to kWh --- .../adjustmentMethods/EToAdjustmentMethod.ts | 12 ++-- test/etoTest.json | 62 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts index 679cef9..208495c 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -74,6 +74,8 @@ export function calculateETo( etoData: EToData, elevation: number, coordinates: elevation = elevation / 3.281; // Convert to meters per second. const windSpeed = etoData.windSpeed / 2.237; + // Convert to megajoules. + const solarRadiation = etoData.solarRadiation * 3.6; const avgTemp = ( maxTemp + minTemp ) / 2; @@ -111,8 +113,6 @@ export function calculateETo( etoData: EToData, elevation: number, coordinates: const clearSkyRadiation = ( 0.75 + 2e-5 * elevation ) * extraterrestrialRadiation; - const solarRadiation = etoData.solarRadiation; - const netShortWaveRadiation = ( 1 - 0.23 ) * solarRadiation; const netOutgoingLongWaveRadiation = 4.903e-9 * ( Math.pow( maxTemp + 273.16, 4 ) + Math.pow( minTemp + 273.16, 4 ) ) / 2 * ( 0.34 - 0.14 * Math.sqrt( actualVaporPressure ) ) * ( 1.35 * solarRadiation / clearSkyRadiation - 0.35); @@ -149,7 +149,7 @@ SunCalc.addTime( Math.asin( 30 / 990 ) * 180 / Math.PI, "radiationStart", "radia * http://www.shodor.org/os411/courses/_master/tools/calculators/solarrad/ * @param cloudCoverInfo Information about the cloud coverage for several periods that span the entire day. * @param coordinates The coordinates of the location the data is from. - * @return The total solar radiation for the day (in megajoules per square meter per day). + * @return The total solar radiation for the day (in kilowatt hours per square meter per day). */ export function approximateSolarRadiation(cloudCoverInfo: CloudCoverInfo[], coordinates: GeoCoordinates ): number { return cloudCoverInfo.reduce( ( total, window: CloudCoverInfo ) => { @@ -172,8 +172,8 @@ export function approximateSolarRadiation(cloudCoverInfo: CloudCoverInfo[], coor const endPosition = SunCalc.getPosition( endTime.toDate(), coordinates[ 0 ], coordinates[ 1 ] ); const solarElevationAngle = ( startPosition.altitude + endPosition.altitude ) / 2; - // Calculate radiation and convert from watts to megajoules. - const clearSkyIsolation = ( 990 * Math.sin( solarElevationAngle ) - 30 ) * 0.0036 * windowLength; + // Calculate radiation and convert from watts to kilowatts. + const clearSkyIsolation = ( 990 * Math.sin( solarElevationAngle ) - 30 ) / 1000 * windowLength; return total + clearSkyIsolation * ( 1 - 0.75 * Math.pow( window.cloudCover, 3.4 ) ); }, 0 ); @@ -212,7 +212,7 @@ export interface EToData { minHumidity: number; /** The maximum relative humidity over the time period (as a percentage). */ maxHumidity: number; - /** The solar radiation, accounting for cloud coverage (in megajoules per square meter per day). */ + /** The solar radiation, accounting for cloud coverage (in kilowatt hours per square meter per day). */ solarRadiation: number; /** * The average wind speed measured at 2 meters over the time period (in miles per hour). A measurement taken at a diff --git a/test/etoTest.json b/test/etoTest.json index 7519b82..bc4fce2 100644 --- a/test/etoTest.json +++ b/test/etoTest.json @@ -6,37 +6,37 @@ "coordinates": [ -33.90, 150.73 ], "startTimestamp": 1556668800, "entries": [ - {"eto":0.075,"data":{"maxTemp":76.46,"minTemp":55.04,"maxHumidity":100,"minHumidity":58,"windSpeed":2.309,"solarRadiation":10.4}}, - {"eto":0.063,"data":{"maxTemp":77,"minTemp":56.84,"maxHumidity":100,"minHumidity":63,"windSpeed":1.707,"solarRadiation":8.66}}, - {"eto":0.035,"data":{"maxTemp":68.36,"minTemp":56.84,"maxHumidity":100,"minHumidity":91,"windSpeed":2.309,"solarRadiation":4.27}}, - {"eto":0.11,"data":{"maxTemp":72.86,"minTemp":58.46,"maxHumidity":100,"minHumidity":36,"windSpeed":5.254,"solarRadiation":12.15}}, - {"eto":0.098,"data":{"maxTemp":69.44,"minTemp":48.56,"maxHumidity":96,"minHumidity":46,"windSpeed":6.324,"solarRadiation":10.61}}, - {"eto":0.098,"data":{"maxTemp":70.16,"minTemp":47.84,"maxHumidity":97,"minHumidity":39,"windSpeed":4.551,"solarRadiation":13.68}}, - {"eto":0.075,"data":{"maxTemp":71.42,"minTemp":39.74,"maxHumidity":100,"minHumidity":37,"windSpeed":2.259,"solarRadiation":13.56}}, - {"eto":0.114,"data":{"maxTemp":68.36,"minTemp":41.36,"maxHumidity":99,"minHumidity":34,"windSpeed":6.676,"solarRadiation":12.96}}, - {"eto":0.063,"data":{"maxTemp":68.72,"minTemp":36.32,"maxHumidity":99,"minHumidity":36,"windSpeed":1.673,"solarRadiation":13.14}}, - {"eto":0.071,"data":{"maxTemp":65.66,"minTemp":41.18,"maxHumidity":100,"minHumidity":43,"windSpeed":3.999,"solarRadiation":6.76}}, - {"eto":0.13,"data":{"maxTemp":69.08,"minTemp":42.08,"maxHumidity":78,"minHumidity":38,"windSpeed":7.88,"solarRadiation":12.99}}, - {"eto":0.071,"data":{"maxTemp":71.6,"minTemp":38.48,"maxHumidity":99,"minHumidity":35,"windSpeed":2.158,"solarRadiation":12.98}}, - {"eto":0.067,"data":{"maxTemp":73.04,"minTemp":38.84,"maxHumidity":100,"minHumidity":51,"windSpeed":2.326,"solarRadiation":12.49}}, - {"eto":0.079,"data":{"maxTemp":75.74,"minTemp":43.52,"maxHumidity":100,"minHumidity":33,"windSpeed":2.242,"solarRadiation":12.75}}, - {"eto":0.067,"data":{"maxTemp":72.68,"minTemp":44.42,"maxHumidity":100,"minHumidity":45,"windSpeed":1.991,"solarRadiation":12.62}}, - {"eto":0.067,"data":{"maxTemp":71.6,"minTemp":44.06,"maxHumidity":100,"minHumidity":47,"windSpeed":2.326,"solarRadiation":12.47}}, - {"eto":0.071,"data":{"maxTemp":73.94,"minTemp":43.16,"maxHumidity":100,"minHumidity":45,"windSpeed":2.393,"solarRadiation":12.28}}, - {"eto":0.071,"data":{"maxTemp":73.4,"minTemp":45.5,"maxHumidity":100,"minHumidity":50,"windSpeed":2.56,"solarRadiation":12.3}}, - {"eto":0.063,"data":{"maxTemp":73.22,"minTemp":51.44,"maxHumidity":100,"minHumidity":51,"windSpeed":2.342,"solarRadiation":10.02}}, - {"eto":0.055,"data":{"maxTemp":74.12,"minTemp":46.58,"maxHumidity":100,"minHumidity":51,"windSpeed":1.69,"solarRadiation":9.74}}, - {"eto":0.067,"data":{"maxTemp":78.44,"minTemp":44.06,"maxHumidity":100,"minHumidity":43,"windSpeed":1.723,"solarRadiation":11.84}}, - {"eto":0.071,"data":{"maxTemp":77.36,"minTemp":47.3,"maxHumidity":100,"minHumidity":40,"windSpeed":2.125,"solarRadiation":11.76}}, - {"eto":0.063,"data":{"maxTemp":74.48,"minTemp":53.06,"maxHumidity":100,"minHumidity":53,"windSpeed":1.991,"solarRadiation":11.43}}, - {"eto":0.059,"data":{"maxTemp":73.58,"minTemp":44.42,"maxHumidity":100,"minHumidity":48,"windSpeed":2.008,"solarRadiation":11.19}}, - {"eto":0.087,"data":{"maxTemp":77.9,"minTemp":42.8,"maxHumidity":100,"minHumidity":26,"windSpeed":2.828,"solarRadiation":11.78}}, - {"eto":0.091,"data":{"maxTemp":72.68,"minTemp":44.24,"maxHumidity":92,"minHumidity":29,"windSpeed":3.865,"solarRadiation":9.89}}, - {"eto":0.13,"data":{"maxTemp":66.02,"minTemp":39.74,"maxHumidity":82,"minHumidity":35,"windSpeed":9.905,"solarRadiation":8.73}}, - {"eto":0.106,"data":{"maxTemp":65.66,"minTemp":37.58,"maxHumidity":69,"minHumidity":31,"windSpeed":5.739,"solarRadiation":11.56}}, - {"eto":0.161,"data":{"maxTemp":65.48,"minTemp":47.66,"maxHumidity":52,"minHumidity":31,"windSpeed":10.859,"solarRadiation":10.79}}, - {"eto":0.102,"data":{"maxTemp":60.08,"minTemp":36.68,"maxHumidity":70,"minHumidity":31,"windSpeed":6.743,"solarRadiation":11.42}}, - {"eto":0.087,"data":{"maxTemp":68,"minTemp":34.34,"maxHumidity":82,"minHumidity":34,"windSpeed":4.149,"solarRadiation":11.34}} + {"eto":0.075,"data":{"maxTemp":76.46,"minTemp":55.04,"maxHumidity":100,"minHumidity":58,"windSpeed":2.309,"solarRadiation":2.889}}, + {"eto":0.063,"data":{"maxTemp":77,"minTemp":56.84,"maxHumidity":100,"minHumidity":63,"windSpeed":1.707,"solarRadiation":2.406}}, + {"eto":0.035,"data":{"maxTemp":68.36,"minTemp":56.84,"maxHumidity":100,"minHumidity":91,"windSpeed":2.309,"solarRadiation":1.186}}, + {"eto":0.11,"data":{"maxTemp":72.86,"minTemp":58.46,"maxHumidity":100,"minHumidity":36,"windSpeed":5.254,"solarRadiation":3.375}}, + {"eto":0.098,"data":{"maxTemp":69.44,"minTemp":48.56,"maxHumidity":96,"minHumidity":46,"windSpeed":6.324,"solarRadiation":2.947}}, + {"eto":0.098,"data":{"maxTemp":70.16,"minTemp":47.84,"maxHumidity":97,"minHumidity":39,"windSpeed":4.551,"solarRadiation":3.8}}, + {"eto":0.075,"data":{"maxTemp":71.42,"minTemp":39.74,"maxHumidity":100,"minHumidity":37,"windSpeed":2.259,"solarRadiation":3.767}}, + {"eto":0.114,"data":{"maxTemp":68.36,"minTemp":41.36,"maxHumidity":99,"minHumidity":34,"windSpeed":6.676,"solarRadiation":3.6}}, + {"eto":0.063,"data":{"maxTemp":68.72,"minTemp":36.32,"maxHumidity":99,"minHumidity":36,"windSpeed":1.673,"solarRadiation":3.65}}, + {"eto":0.071,"data":{"maxTemp":65.66,"minTemp":41.18,"maxHumidity":100,"minHumidity":43,"windSpeed":3.999,"solarRadiation":1.878}}, + {"eto":0.13,"data":{"maxTemp":69.08,"minTemp":42.08,"maxHumidity":78,"minHumidity":38,"windSpeed":7.88,"solarRadiation":3.608}}, + {"eto":0.071,"data":{"maxTemp":71.6,"minTemp":38.48,"maxHumidity":99,"minHumidity":35,"windSpeed":2.158,"solarRadiation":3.606}}, + {"eto":0.067,"data":{"maxTemp":73.04,"minTemp":38.84,"maxHumidity":100,"minHumidity":51,"windSpeed":2.326,"solarRadiation":3.469}}, + {"eto":0.079,"data":{"maxTemp":75.74,"minTemp":43.52,"maxHumidity":100,"minHumidity":33,"windSpeed":2.242,"solarRadiation":3.542}}, + {"eto":0.067,"data":{"maxTemp":72.68,"minTemp":44.42,"maxHumidity":100,"minHumidity":45,"windSpeed":1.991,"solarRadiation":3.506}}, + {"eto":0.067,"data":{"maxTemp":71.6,"minTemp":44.06,"maxHumidity":100,"minHumidity":47,"windSpeed":2.326,"solarRadiation":3.464}}, + {"eto":0.071,"data":{"maxTemp":73.94,"minTemp":43.16,"maxHumidity":100,"minHumidity":45,"windSpeed":2.393,"solarRadiation":3.411}}, + {"eto":0.071,"data":{"maxTemp":73.4,"minTemp":45.5,"maxHumidity":100,"minHumidity":50,"windSpeed":2.56,"solarRadiation":3.417}}, + {"eto":0.063,"data":{"maxTemp":73.22,"minTemp":51.44,"maxHumidity":100,"minHumidity":51,"windSpeed":2.342,"solarRadiation":2.783}}, + {"eto":0.055,"data":{"maxTemp":74.12,"minTemp":46.58,"maxHumidity":100,"minHumidity":51,"windSpeed":1.69,"solarRadiation":2.706}}, + {"eto":0.067,"data":{"maxTemp":78.44,"minTemp":44.06,"maxHumidity":100,"minHumidity":43,"windSpeed":1.723,"solarRadiation":3.289}}, + {"eto":0.071,"data":{"maxTemp":77.36,"minTemp":47.3,"maxHumidity":100,"minHumidity":40,"windSpeed":2.125,"solarRadiation":3.267}}, + {"eto":0.063,"data":{"maxTemp":74.48,"minTemp":53.06,"maxHumidity":100,"minHumidity":53,"windSpeed":1.991,"solarRadiation":3.175}}, + {"eto":0.059,"data":{"maxTemp":73.58,"minTemp":44.42,"maxHumidity":100,"minHumidity":48,"windSpeed":2.008,"solarRadiation":3.108}}, + {"eto":0.087,"data":{"maxTemp":77.9,"minTemp":42.8,"maxHumidity":100,"minHumidity":26,"windSpeed":2.828,"solarRadiation":3.272}}, + {"eto":0.091,"data":{"maxTemp":72.68,"minTemp":44.24,"maxHumidity":92,"minHumidity":29,"windSpeed":3.865,"solarRadiation":2.747}}, + {"eto":0.13,"data":{"maxTemp":66.02,"minTemp":39.74,"maxHumidity":82,"minHumidity":35,"windSpeed":9.905,"solarRadiation":2.425}}, + {"eto":0.106,"data":{"maxTemp":65.66,"minTemp":37.58,"maxHumidity":69,"minHumidity":31,"windSpeed":5.739,"solarRadiation":3.211}}, + {"eto":0.161,"data":{"maxTemp":65.48,"minTemp":47.66,"maxHumidity":52,"minHumidity":31,"windSpeed":10.859,"solarRadiation":2.997}}, + {"eto":0.102,"data":{"maxTemp":60.08,"minTemp":36.68,"maxHumidity":70,"minHumidity":31,"windSpeed":6.743,"solarRadiation":3.172}}, + {"eto":0.087,"data":{"maxTemp":68,"minTemp":34.34,"maxHumidity":82,"minHumidity":34,"windSpeed":4.149,"solarRadiation":3.15}} ] } ] From 49db07071f9728ca603a6f7e228ce0276ffdb342 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Tue, 18 Jun 2019 18:19:53 -0400 Subject: [PATCH 05/10] Add more information to EToAdjustmentMethod raw data --- routes/adjustmentMethods/EToAdjustmentMethod.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts index 208495c..128cfe0 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -50,7 +50,12 @@ async function calculateEToWateringScale( scale: scale, rawData: { eto: Math.round( eto * 1000) / 1000, - radiation: Math.round( etoData.solarRadiation * 100) / 100 + radiation: Math.round( etoData.solarRadiation * 100) / 100, + minT: Math.round( etoData.minTemp ), + maxT: Math.round( etoData.maxTemp ), + minH: Math.round( etoData.minHumidity ), + maxH: Math.round( etoData.maxHumidity ), + wind: Math.round( etoData.windSpeed * 10 ) / 10 } } } From ca42db5d8002cfbd9ae8f41c24e16f041594ec28 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Thu, 20 Jun 2019 11:57:03 -0400 Subject: [PATCH 06/10] Add ETo support to Dark Sky --- routes/weatherProviders/DarkSky.ts | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/routes/weatherProviders/DarkSky.ts b/routes/weatherProviders/DarkSky.ts index 7098f39..653ac84 100644 --- a/routes/weatherProviders/DarkSky.ts +++ b/routes/weatherProviders/DarkSky.ts @@ -3,6 +3,7 @@ import * as moment from "moment-timezone"; import { GeoCoordinates, WateringData, WeatherData } from "../../types"; import { httpJSONRequest } from "../weather"; import { WeatherProvider } from "./WeatherProvider"; +import { approximateSolarRadiation, CloudCoverInfo, EToData } from "../adjustmentMethods/EToAdjustmentMethod"; export default class DarkSkyWeatherProvider extends WeatherProvider { @@ -107,4 +108,52 @@ export default class DarkSkyWeatherProvider extends WeatherProvider { return weather; } + + public async getEToData( coordinates: GeoCoordinates ): Promise< EToData > { + // The Unix epoch seconds timestamp of 24 hours ago. + const timestamp: number = moment().subtract( 1, "day" ).unix(); + + const DARKSKY_API_KEY = process.env.DARKSKY_API_KEY, + historicUrl = `https://api.darksky.net/forecast/${DARKSKY_API_KEY}/${coordinates[0]},${coordinates[1]},${timestamp}`; + + let historicData; + try { + historicData = await httpJSONRequest( historicUrl ); + } catch (err) { + throw "An error occurred while retrieving weather information from Dark Sky." + } + + const cloudCoverInfo: CloudCoverInfo[] = historicData.hourly.data.map( ( hour ): CloudCoverInfo => { + return { + startTime: moment.unix( hour.time ), + endTime: moment.unix( hour.time ).add( 1, "hours" ), + cloudCover: hour.cloudCover + }; + } ); + + let minHumidity: number = undefined, maxHumidity: number = undefined; + for ( const hour of historicData.hourly.data ) { + // Skip hours where humidity measurement does not exist to prevent result from being NaN. + if ( hour.humidity === undefined ) { + continue; + } + + // If minHumidity or maxHumidity is undefined, these comparisons will yield false. + minHumidity = minHumidity < hour.humidity ? minHumidity : hour.humidity; + maxHumidity = maxHumidity > hour.humidity ? maxHumidity : hour.humidity; + } + + return { + weatherProvider: "DarkSky", + periodStartTime: historicData.hourly.data[ 0 ].time, + minTemp: historicData.daily.data[ 0 ].temperatureMin, + maxTemp: historicData.daily.data[ 0 ].temperatureMax, + minHumidity: minHumidity * 100, + maxHumidity: maxHumidity * 100, + solarRadiation: approximateSolarRadiation( cloudCoverInfo, coordinates ), + // Assume wind speed measurements are taken at 2 meters. + windSpeed: historicData.daily.data[ 0 ].windSpeed, + precip: ( historicData.daily.data[ 0 ].precipIntensity || 0 ) * 24 + }; + } } From 6aa3b7a5b70424e2a46ec8abe9c97a503deb1fe8 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Thu, 20 Jun 2019 14:03:21 -0400 Subject: [PATCH 07/10] Add ETo support to OWM --- routes/weatherProviders/OWM.ts | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/routes/weatherProviders/OWM.ts b/routes/weatherProviders/OWM.ts index fdb4023..3341878 100644 --- a/routes/weatherProviders/OWM.ts +++ b/routes/weatherProviders/OWM.ts @@ -1,6 +1,8 @@ import { GeoCoordinates, WateringData, WeatherData } from "../../types"; import { httpJSONRequest } from "../weather"; import { WeatherProvider } from "./WeatherProvider"; +import { approximateSolarRadiation, CloudCoverInfo, EToData } from "../adjustmentMethods/EToAdjustmentMethod"; +import * as moment from "moment"; export default class OWMWeatherProvider extends WeatherProvider { @@ -89,4 +91,68 @@ export default class OWMWeatherProvider extends WeatherProvider { return weather; } + + // Uses a rolling window since forecast data from further in the future (i.e. the next full day) would be less accurate. + async getEToData( coordinates: GeoCoordinates ): Promise< EToData > { + const OWM_API_KEY = process.env.OWM_API_KEY, + forecastUrl = "http://api.openweathermap.org/data/2.5/forecast?appid=" + OWM_API_KEY + "&units=imperial&lat=" + coordinates[ 0 ] + "&lon=" + coordinates[ 1 ]; + + // Perform the HTTP request to retrieve the weather data + let forecast; + try { + forecast = await httpJSONRequest( forecastUrl ); + } catch (err) { + console.error( "Error retrieving ETo information from OWM:", err ); + throw "An error occurred while retrieving ETo information from OWM." + } + + // Indicate ETo data could not be retrieved if the forecast data is incomplete. + if ( !forecast || !forecast.list || forecast.list.length < 8 ) { + throw "Insufficient data available from OWM." + } + + // Take a sample over 24 hours. + const samples = forecast.list.slice( 0, 8 ); + + const cloudCoverInfo: CloudCoverInfo[] = samples.map( ( window ): CloudCoverInfo => { + return { + startTime: moment.unix( window.dt ), + endTime: moment.unix( window.dt ).add( 3, "hours" ), + cloudCover: window.clouds.all / 100 + }; + } ); + + let minTemp: number = undefined, maxTemp: number = undefined; + let minHumidity: number = undefined, maxHumidity: number = undefined; + // Skip hours where measurements don't exist to prevent result from being NaN. + for ( const sample of samples ) { + const temp: number = sample.main.temp; + if ( temp !== undefined ) { + // If minTemp or maxTemp is undefined, these comparisons will yield false. + minTemp = minTemp < temp ? minTemp : temp; + maxTemp = maxTemp > temp ? maxTemp : temp; + } + + const humidity: number = sample.main.humidity; + if ( humidity !== undefined ) { + // If minHumidity or maxHumidity is undefined, these comparisons will yield false. + minHumidity = minHumidity < humidity ? minHumidity : humidity; + maxHumidity = maxHumidity > humidity ? maxHumidity : humidity; + } + } + + return { + weatherProvider: "OWM", + periodStartTime: samples[ 0 ].dt, + minTemp: minTemp, + maxTemp: maxTemp, + minHumidity: minHumidity, + maxHumidity: maxHumidity, + solarRadiation: approximateSolarRadiation( cloudCoverInfo, coordinates ), + // Assume wind speed measurements are taken at 2 meters. + windSpeed: samples.reduce( ( sum, window ) => sum + ( window.wind.speed || 0 ), 0) / samples.length, + // OWM always returns precip in mm, so it must be converted. + precip: samples.reduce( ( sum, window ) => sum + ( window.rain ? window.rain[ "3h" ] || 0 : 0 ), 0) / 25.4 + }; + } } From 7a7fde393f0f43bef0fa82d6cb9ea0f7f3a556e2 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Tue, 25 Jun 2019 17:11:03 -0400 Subject: [PATCH 08/10] Add baseline ETo endpoint --- .gitignore | 4 + baselineEToData/.dockerignore | 4 + baselineEToData/Dockerfile | 12 ++ baselineEToData/README.md | 57 ++++++ baselineEToData/dataPreparer.c | 354 +++++++++++++++++++++++++++++++++ baselineEToData/entrypoint.sh | 4 + baselineEToData/prepareData.sh | 18 ++ routes/baselineETo.ts | 182 +++++++++++++++++ routes/weather.ts | 4 +- server.ts | 5 + 10 files changed, 642 insertions(+), 2 deletions(-) create mode 100644 baselineEToData/.dockerignore create mode 100644 baselineEToData/Dockerfile create mode 100644 baselineEToData/README.md create mode 100644 baselineEToData/dataPreparer.c create mode 100644 baselineEToData/entrypoint.sh create mode 100644 baselineEToData/prepareData.sh create mode 100644 routes/baselineETo.ts diff --git a/.gitignore b/.gitignore index 5efa40e..eb69cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ npm-debug.log .idea js weather.zip +baselineEToData/*.bin +baselineEToData/*.png +baselineEToData/*.tif +baselineEToData/dataPreparer[.exe] diff --git a/baselineEToData/.dockerignore b/baselineEToData/.dockerignore new file mode 100644 index 0000000..986901d --- /dev/null +++ b/baselineEToData/.dockerignore @@ -0,0 +1,4 @@ +# Don't send images to the Docker daemon since they are very large. +*.bin +*.tif +*.png diff --git a/baselineEToData/Dockerfile b/baselineEToData/Dockerfile new file mode 100644 index 0000000..dc3acff --- /dev/null +++ b/baselineEToData/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine:3.10 + +VOLUME /output/ +ENTRYPOINT ["/entrypoint.sh"] +# Default to 20 passes. +CMD ["20"] + +COPY dataPreparer.c /dataPreparer.c +COPY prepareData.sh /prepareData.sh +COPY entrypoint.sh /entrypoint.sh + +RUN apk --update add imagemagick gcc libc-dev && chmod +x /entrypoint.sh /prepareData.sh diff --git a/baselineEToData/README.md b/baselineEToData/README.md new file mode 100644 index 0000000..fb3e535 --- /dev/null +++ b/baselineEToData/README.md @@ -0,0 +1,57 @@ +# Baseline ETo Data + +The baseline ETo endpoint determines the baseline ETo for a location by reading a file generated using data from [MOD16](https://www.ntsg.umt.edu/project/modis/mod16.php). +The data is stored in a binary file that has 4 key differences from the GeoTIFF provided by MOD16: +* The bit depth is decreased from 16 bits to 8 bits to reduce the file size. +* Missing data is interpolated using the values of surrounding pixels. +The MOD16 dataset does not contain data for locations that don't have vegetated land cover (such as urban environments), which can be problematic since many users may set their location to nearby cities. +* The data is stored in an uncompressed format so that geographic coordinates can be mapped to the offset of the corresponding pixel in the file. +This means the file can be stored on disk instead of memory, and the pixel for a specified location can be quickly accessed by seeking to the calculated offset in the file. +* A metadata header that contains parameters about the data used to create the file (such as the image dimensions and instructions on how to map a pixel value to an annual ETo value) is added to the beginning of the file. +This header enables the weather server to use datafiles generated from future versions of the MOD16 dataset (even if these versions modify some of these parameters). + +The datafile is to be stored as `baselineEToData/Baseline_ETo_Data.bin`. +The datafile is not included in the repo because it is very large (62 MB zipped, 710 MB uncompressed), but it [can be downloaded separately](http://www.mediafire.com/file/n7z32dbdvgyupk3/Baseline_ETo_Data.zip/file). +This file was generated by making 20 [passes](#passes) over the data from 2000-2013 in the MOD16A3 dataset. +Alternatively, it can be generated by running the data preparer program yourself. + +## Preparing the Datafile + +Since TIFF files do not support streaming, directly using the GeoTIFF images from MOD16 would require loading the entire image into memory. +To avoid this, the file must first be converted to a binary format so the pixels in the image can be read row-by-row. +Running `./prepareData.sh ` will download the required image files using [wget](https://www.gnu.org/software/wget/), convert them to a binary format using [ImageMagick](https://imagemagick.org/index.php), compile the program with [gcc](https://gcc.gnu.org/), and run it . +This process can be simplified by using the included Dockerfile that will perform all of these steps inside a container. +The Dockerfile can be used by running `docker build -t baseline-eto-data-preparer . && docker run --rm -v $(pwd):/output baseline-eto-data-preparer `. + +The `` argument is used to control how much the program should attempt to fill in missing data. + +(#passes) +### Passes +The program fills in missing data by making several successive passes over the entire image, attempting to fill in each missing pixel on each pass. +The value for each missing pixel is interpolated using the values of pixels in the surrounding 5x5 square, and missing pixels that don't have enough data available will be skipped. +However, these pixels may be filled in on a later pass if future passes are able to fill in the surrounding pixels. +Running the program with a higher number of passes will fill in more missing data, but the program will take longer to run and each subsequent pass becomes less accurate (since the interpolations will be based on interpolated data). + +## File Format + +The data will be saved in a binary format beginning with the a 32 byte big-endian header in the following format: + +| Offset | Type | Description | +| --- | --- | --- | +| 0 | uint8 | File format version | +| 1-4 | uint32 | Image width (in pixels) | +| 5-8 | uint32 | Image height (in pixels) | +| 9 | uint8 | Pixel bit depth (the only bit depth currently supported is 8) | +| 10-13 | float | Minimum ETo | +| 14-17 | float | Scaling factor | +| 18-32 | N/A | May be used in future versions | + +The header is immediately followed by a `IMAGE_WIDTH * IMAGE_HEIGHT` bytes of data corresponding to the pixels in the image in row-major order. +Each pixel is interpreted as an 8 bit unsigned integer, and the average annual potential ETo at that location is `PIXEL * SCALING_FACTOR + MINIMUM_ETO` inches/year. +A value of `255` is special and indicates that no data is available for that location. + +## Notes + +* Although the [MOD16 documentation]((http://files.ntsg.umt.edu/data/NTSG_Products/MOD16/MOD16UsersGuide_V1.6_2018Aug.docx)) states that several pixel values are used to indicate the land cover type for locations that are missing data, the image actually only uses the value `65535`. +The program handles this by using a [mask image of water bodies](https://static1.squarespace.com/static/58586fa5ebbd1a60e7d76d3e/t/59394abb37c58179160775fa/1496926933082/Ocean_Mask.png) so it can fill in pixels for urban environments without also filling in data for oceans. +* The map uses an [equirectangular projection](https://en.wikipedia.org/wiki/Equirectangular_projection) with the northernmost 10 degrees and southernmost 30 degrees cropped off. diff --git a/baselineEToData/dataPreparer.c b/baselineEToData/dataPreparer.c new file mode 100644 index 0000000..6ee106f --- /dev/null +++ b/baselineEToData/dataPreparer.c @@ -0,0 +1,354 @@ +#include +#include +#include +#include + +#define IMAGE_WIDTH 43200 +#define IMAGE_HEIGHT 16800 +#define MASK_WIDTH 10800 +#define MASK_HEIGHT 5400 +#define OUTPUT_FILE_TEMPLATE "./Baseline_ETo_Data-Pass_%d.bin" +#define FILENAME_MAX_LENGTH 40 +#define HEADER_SIZE 32 + +long unsigned CROPPED_TOP_PIXELS = (MASK_WIDTH * MASK_HEIGHT * 10 / 180); +// These will be set by findPixelRange(). +uint16_t minPixelValue = 0; +uint16_t maxPixelValue = 0xFFFF; +double bitReductionFactor = 256; + + +/** Copies the big-endian byte representation of the specified value into the specified buffer. */ +void copyBytes(void* input, uint8_t* output, int unsigned length) { + int unsigned isBigEndian = 1; + isBigEndian = *((uint8_t*)(&isBigEndian)) == 0; + + for (int unsigned i = 0; i < length; i++) { + int unsigned index = isBigEndian ? i : length - i - 1; + output[i] = *((uint8_t*) input + index); + } +} + +/** + * Write file header to the specified buffer. The header format is documented in the README. + */ +void setHeader(uint8_t *header) { + for (int unsigned i = 0; i < HEADER_SIZE; i++) { + header[i] = 0; + } + + uint32_t width = IMAGE_WIDTH; + uint32_t height = IMAGE_HEIGHT; + // originally 0.1, then multiplied by a value to compensate for the bit depth reduction and divided by 25.4 to convert to inches. + float scalingFactor = 0.1 * bitReductionFactor / 25.4; + float minimumETo = minPixelValue * 0.1 / 25.4; + + // Version + header[0] = 1; + // Width + copyBytes(&width, &(header[1]), 4); + // Height + copyBytes(&height, &(header[5]), 4); + // Bit depth + header[9] = 8; + // Minimum ETo + copyBytes(&minimumETo, &(header[10]), 4); + // Scaling factor + copyBytes(&scalingFactor, &(header[14]), 4); +} + +/** + * Calculates the minimum and maximum pixel values used in the image. These values can be used to optimally reduce the + * bit depth by mapping the minimum value to 0 and the maximum value to 254 (reserving 255 for fill pixels) and linearly + * interpolating the rest of the values. + */ +void findPixelRange(uint16_t* minPtr, uint16_t* maxPtr, double* bitReductionFactorPtr) { + time_t startTime = clock(); + + uint16_t minValue = 0xFFFF; + uint16_t maxValue = 0; + + FILE* inputFile = fopen("./MOD16A3_PET_2000_to_2013_mean.bin", "rb"); + if (inputFile == NULL) { + printf("An error occurred opening image file while finding min/max value.\n"); + exit(1); + } + uint16_t buffer[IMAGE_WIDTH]; + for (int unsigned y = 0; y < IMAGE_HEIGHT; y++) { + if (y % 1000 == 0) { + printf("Finding pixel range on row %d...\n", y); + } + + fread(buffer, 2, IMAGE_WIDTH, inputFile); + if (ferror(inputFile)) { + printf("An error occurred reading image row %d while finding min/max values.\n", y); + exit(1); + } + if (feof(inputFile)) { + printf("Encountered EOF reading image row %d while finding min/max values.\n", y); + exit(1); + } + + for (unsigned int x = 0; x < IMAGE_WIDTH; x++) { + uint16_t pixel = buffer[x]; + // Skip fill pixels. + if (pixel > 65528) { + continue; + } + + minValue = pixel < minValue ? pixel : minValue; + maxValue = pixel > maxValue ? pixel : maxValue; + } + } + + *minPtr = minValue; + *maxPtr = maxValue; + *bitReductionFactorPtr = (maxValue - minValue + 1) / (float) 256; + + fclose(inputFile); + printf("Found pixel range in %.1f seconds. Min value: %d\t Max value: %d\t Bit reduction factor:%f.\n", (clock() - startTime) / (float) CLOCKS_PER_SEC, minValue, maxValue, *bitReductionFactorPtr); +} + +/** Reduces the image bit depth from 16 bits to 8 bits. */ +void reduceBitDepth() { + clock_t startTime = clock(); + FILE* originalFile = fopen("./MOD16A3_PET_2000_to_2013_mean.bin", "rb"); + if (originalFile == NULL) { + printf("An error occurred opening input image file while reducing bit depth.\n"); + exit(1); + } + + char* reducedFileName = malloc(FILENAME_MAX_LENGTH); + snprintf(reducedFileName, FILENAME_MAX_LENGTH, OUTPUT_FILE_TEMPLATE, 0); + FILE* reducedFile = fopen(reducedFileName, "wb"); + if (reducedFile == NULL) { + printf("An error occurred opening output image file while reducing bit depth.\n"); + exit(1); + } + + // Write the file header. + uint8_t header[32]; + setHeader(header); + fwrite(header, 1, 32, reducedFile); + if (ferror(reducedFile)) { + printf("An error occurred writing file header while reducing bit depth.\n"); + exit(1); + } + + uint16_t inputBuffer[IMAGE_WIDTH]; + uint8_t outputBuffer[IMAGE_WIDTH]; + for (int unsigned y = 0; y < IMAGE_HEIGHT; y++) { + if (y % 1000 == 0) { + printf("Reducing bit depth on row %d...\n", y); + } + + fread(inputBuffer, 2, IMAGE_WIDTH, originalFile); + if (ferror(originalFile)) { + printf("An error occurred reading row %d while reducing bit depth.\n", y); + exit(1); + } + if (feof(originalFile)) { + printf("Encountered EOF reading row %d while reducing bit depth.\n", y); + exit(1); + } + + for (unsigned int x = 0; x < IMAGE_WIDTH; x++) { + uint16_t originalPixel = inputBuffer[x]; + uint8_t reducedPixel = originalPixel > 65528 ? 255 : (uint8_t) ((originalPixel - minPixelValue) / bitReductionFactor); + outputBuffer[x] = reducedPixel; + } + + fwrite(outputBuffer, 1, IMAGE_WIDTH, reducedFile); + if (ferror(reducedFile)) { + printf("An error occurred writing row %d while reducing bit depth.\n", y); + exit(1); + } + } + + fclose(reducedFile); + fclose(originalFile); + + printf("Finished reducing bit depth in %.1f seconds.\n", (clock() - startTime) / (double) CLOCKS_PER_SEC); +} + +void fillMissingPixels(int unsigned pass) { + clock_t startTime = clock(); + + char* inputFileName = malloc(FILENAME_MAX_LENGTH); + snprintf(inputFileName, FILENAME_MAX_LENGTH, OUTPUT_FILE_TEMPLATE, pass - 1); + FILE* inputFile = fopen(inputFileName, "rb"); + if (inputFile == NULL) { + printf("An error occurred opening input image file on pass %d.\n", pass); + exit(1); + } + + char* outputFileName = malloc(FILENAME_MAX_LENGTH); + snprintf(outputFileName, FILENAME_MAX_LENGTH, OUTPUT_FILE_TEMPLATE, pass); + FILE* outputFile = fopen(outputFileName, "wb"); + if (outputFile == NULL) { + printf("An error occurred opening output image file on pass %d.\n", pass); + exit(1); + } + + FILE* maskFile = fopen("./Ocean_Mask.bin", "rb"); + if (maskFile == NULL) { + printf("An error occurred opening mask image on pass %d.\n", pass); + exit(1); + } + + uint8_t outputBuffer[IMAGE_WIDTH]; + + // Skip the header. + fseek(inputFile, 32, SEEK_SET); + if (ferror(inputFile)) { + printf("An error occurred reading header on pass %d.\n", pass); + exit(1); + } + if (feof(inputFile)) { + printf("Encountered EOF reading header on pass %d.\n", pass); + exit(1); + } + + // Write the file header. + uint8_t header[32]; + setHeader(header); + fwrite(header, 1, 32, outputFile); + if (ferror(outputFile)) { + printf("An error occurred writing file header on pass %d.\n", pass); + exit(1); + } + + uint8_t* rows[5] = {0, 0, 0, 0, 0}; + // Read the first 2 rows. + for (int unsigned rowIndex = 3; rowIndex < 5; rowIndex++) { + uint8_t* row = (uint8_t*) malloc(IMAGE_WIDTH); + fread(row, 1, IMAGE_WIDTH, inputFile); + if (ferror(inputFile)) { + printf("An error occurred reading image row %d on pass %d.\n", rowIndex - 3, pass); + exit(1); + } + if (feof(inputFile)) { + printf("Encountered EOF reading image row %d on pass %d.\n", rowIndex - 3, pass); + exit(1); + } + + rows[rowIndex] = row; + } + + long unsigned fixedPixels = 0; + long unsigned unfixablePixels = 0; + long unsigned waterPixels = 0; + + for (int unsigned y = 0; y < IMAGE_HEIGHT; y++) { + if (y % 1000 == 0) { + printf("Filling missing pixels on pass %d row %d.\n", pass, y); + } + + // Read a row from the mask. + uint8_t maskRow[MASK_WIDTH]; + int unsigned maskOffset = y / (IMAGE_WIDTH / MASK_WIDTH) * MASK_WIDTH + CROPPED_TOP_PIXELS; + fseek(maskFile, maskOffset, SEEK_SET); + fread(maskRow, 1, MASK_WIDTH, maskFile); + if (ferror(maskFile)) { + printf("An error occurred reading mask at offset %d on pass %d.\n", maskOffset, pass); + exit(1); + } + if (feof(maskFile)) { + printf("Encountered EOF reading mask at offset %d on pass %d.\n", maskOffset, pass); + exit(1); + } + + // Free the oldest row. + free(rows[0]); + // Shift the previous rows back. + for (int unsigned rowIndex = 1; rowIndex < 5; rowIndex++) { + rows[rowIndex - 1] = rows[rowIndex]; + } + + // Read the next row if one exists. + if (y < IMAGE_HEIGHT - 2) { + uint8_t* row = malloc(IMAGE_WIDTH); + fread(row, 1, IMAGE_WIDTH, inputFile); + if (ferror(inputFile)) { + printf("An error occurred reading image row %d on pass %d.\n", y + 2, pass); + exit(1); + } + if (feof(inputFile)) { + printf("Encountered EOF reading image row %d on pass %d,\n", y + 2, pass); + exit(1); + } + + rows[4] = row; + } + + for (unsigned int x = 0; x < IMAGE_WIDTH; x++) { + uint8_t pixel = *(rows[2] +x); + // Skip water pixels. + if (maskRow[x / (IMAGE_WIDTH / MASK_WIDTH)] > 128) { + if (pixel == 255) { + int unsigned totalWeight = 0; + float neighborTotal = 0; + for (int i = -2; i <= 2; i++) { + for (int j = -2; j <= 2; j++) { + int neighborX = x + i; + int neighborY = y + j; + if (neighborX < 0 || neighborX >= IMAGE_WIDTH || neighborY < 0 || neighborY >= IMAGE_HEIGHT) { + continue; + } + + uint8_t neighbor = *(rows[2 + j] + neighborX); + if (neighbor == 255) { + continue; + } + + int unsigned weight = 5 - (abs(i) + abs(j)); + neighborTotal += weight * neighbor; + totalWeight += weight; + } + } + if (totalWeight > 11) { + pixel = (uint8_t) (neighborTotal / totalWeight); + fixedPixels++; + } else { + unfixablePixels++; + } + } + } else { + waterPixels++; + } + + outputBuffer[x] = pixel; + } + + fwrite(outputBuffer, 1, IMAGE_WIDTH, outputFile); + if (ferror(outputFile)) { + printf("An error occurred writing row %d on pass %d.\n", y, pass); + exit(1); + } + } + + fclose(outputFile); + fclose(inputFile); + fclose(maskFile); + + printf("Finished pass %d in %f seconds. Fixed pixels: %ld\t Unfixable pixels: %ld\t Water pixels: %ld.\n", pass, (clock() - startTime) / (double) CLOCKS_PER_SEC, fixedPixels, unfixablePixels, waterPixels); +} + +int main(int argc, char* argv[]) { + if (argc != 2) { + printf("Proper usage: %s \n", argv[0]); + } + int unsigned passes = strtol(argv[1], NULL, 10); + if (passes <= 0) { + printf("passes argument must be a positive integer.\n"); + exit(1); + } + + findPixelRange(&minPixelValue, &maxPixelValue, &bitReductionFactor); + reduceBitDepth(); + for (int unsigned i = 1; i <= passes; i++) { + fillMissingPixels(i); + } + + return 0; +} diff --git a/baselineEToData/entrypoint.sh b/baselineEToData/entrypoint.sh new file mode 100644 index 0000000..5798630 --- /dev/null +++ b/baselineEToData/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh +/prepareData.sh $1 +# Move the last pass to the output directory. +mv $(ls Baseline_ETo_Data-Pass_*.bin | tail -n1) /output/Baseline_ETo_Data.bin diff --git a/baselineEToData/prepareData.sh b/baselineEToData/prepareData.sh new file mode 100644 index 0000000..a982f2e --- /dev/null +++ b/baselineEToData/prepareData.sh @@ -0,0 +1,18 @@ +#!/bin/sh +echo "Compiling dataPreparer.c..." +gcc -std=c99 -o dataPreparer dataPreparer.c + +echo "Downloading ocean mask image..." +wget http://static1.squarespace.com/static/58586fa5ebbd1a60e7d76d3e/t/59394abb37c58179160775fa/1496926933082/Ocean_Mask.png + +echo "Converting ocean mask image to binary format..." +magick Ocean_Mask.png -depth 8 gray:Ocean_Mask.bin + +echo "Downloading MOD16 GeoTIFF..." +wget http://files.ntsg.umt.edu/data/NTSG_Products/MOD16/MOD16A3.105_MERRAGMAO/Geotiff/MOD16A3_PET_2000_to_2013_mean.tif + +echo "Converting MOD16 GeoTIFF to binary format..." +magick MOD16A3_PET_2000_to_2013_mean.tif -depth 16 gray:MOD16A3_PET_2000_to_2013_mean.bin + +echo "Preparing data..." +./dataPreparer $1 diff --git a/routes/baselineETo.ts b/routes/baselineETo.ts new file mode 100644 index 0000000..0da8520 --- /dev/null +++ b/routes/baselineETo.ts @@ -0,0 +1,182 @@ +/* This script requires the file Baseline_ETo_Data.bin file to be created in the baselineEToData directory. More + * information about this is available in /baselineEToData/README.md. + */ +import * as express from "express"; +import * as fs from "fs"; +import { GeoCoordinates } from "../types"; +import { getParameter, resolveCoordinates } from "./weather"; + +const DATA_FILE = __dirname + "/../../baselineEToData/Baseline_ETo_Data.bin"; +let FILE_META: FileMeta; + +readFileHeader().then( ( fileMeta ) => { + FILE_META = fileMeta; + console.log( "Loaded baseline ETo data." ); +} ).catch( ( err ) => { + console.error( "An error occurred while reading the annual ETo data file header. Baseline ETo endpoint will be unavailable.", err ); +} ); + +export const getBaselineETo = async function( req: express.Request, res: express.Response ) { + const location: string = getParameter( req.query.loc ); + + // Error if the file meta was not read (either the file is still being read or an error occurred and it could not be read). + if ( !FILE_META ) { + res.status( 503 ).send( "Baseline ETo calculation is currently unavailable." ); + return; + } + + // Attempt to resolve provided location to GPS coordinates. + let coordinates: GeoCoordinates; + try { + coordinates = await resolveCoordinates( location ); + } catch (err) { + res.status( 404 ).send( `Error: Unable to resolve coordinates for location (${ err })` ); + return; + } + + let eto: number; + try { + eto = await calculateAverageDailyETo( coordinates ); + } catch ( err ) { + /* Use a 500 error code if a more appropriate error code is not specified, and prefer the error message over the + full error object if a message is defined. */ + res.status( err.code || 500 ).send( err.message || err ); + return; + } + + res.status( 200 ).json( { + eto: Math.round( eto * 1000 ) / 1000 + } ); +}; + +/** + * Retrieves the average daily potential ETo for the specified location. + * @param coordinates The location to retrieve the ETo for. + * @return A Promise that will be resolved with the average potential ETo (in inches per day), or rejected with an error + * (which may include a message and the appropriate HTTP status code to send the user) if the ETo cannot be retrieved. + */ +async function calculateAverageDailyETo( coordinates: GeoCoordinates ): Promise< number > { + // Convert geographic coordinates into image coordinates. + const x = Math.floor( FILE_META.origin.x + FILE_META.width * coordinates[ 1 ] / 360 ); + // Account for the 30+10 cropped degrees. + const y = Math.floor( FILE_META.origin.y - FILE_META.height * coordinates[ 0 ] / ( 180 - 30 - 10 ) ); + + // The offset (from the start of the data block) of the relevant pixel. + const offset = y * FILE_META.width + x; + + /* Check if the specified coordinates were invalid or correspond to a part of the map that was cropped. */ + if ( offset < 0 || offset > FILE_META.width * FILE_META.height ) { + throw { message: "Specified location is out of bounds.", code: 404 }; + } + + let byte: number; + try { + // Skip the 32 byte header. + byte = await getByteAtOffset( offset + 32 ); + } catch ( err ) { + console.error( `An error occurred while reading the baseline ETo data file for coordinates ${ coordinates }:`, err ); + throw { message: "An unexpected error occurred while retrieving the baseline ETo for this location.", code: 500 } + } + + // The maximum value indicates that no data is available for this point. + if ( ( byte === ( 1 << FILE_META.bitDepth ) - 1 ) ) { + throw { message: "ETo data is not available for this location.", code: 404 }; + } + + return ( byte * FILE_META.scalingFactor + FILE_META.minimumETo ) / 365; +} + +/** + * Returns the byte at the specified offset in the baseline ETo data file. + * @param offset The offset from the start of the file (the start of the header, not the start of the data block). + * @return A Promise that will be resolved with the unsigned representation of the byte at the specified offset, or + * rejected with an Error if an error occurs. + */ +function getByteAtOffset( offset: number ): Promise< number > { + return new Promise( ( resolve, reject ) => { + const stream = fs.createReadStream( DATA_FILE, { start: offset, end: offset } ); + + stream.on( "error", ( err ) => { + reject( err ); + } ); + + // There's no need to wait for the "end" event since the "data" event will contain the single byte being read. + stream.on( "data", ( data ) => { + resolve( data[ 0 ] ); + } ); + } ); +} + +/** + * Parses information from the baseline ETo data file from the file header. The header format is documented in the README. + * @return A Promise that will be resolved with the parsed header information, or rejected with an error if the header + * is invalid or cannot be read. + */ +function readFileHeader(): Promise< FileMeta > { + return new Promise( ( resolve, reject) => { + const stream = fs.createReadStream( DATA_FILE, { start: 0, end: 32 } ); + const headerArray: number[] = []; + + stream.on( "error", ( err ) => { + reject( err ); + } ); + + stream.on( "data", ( data: number[] ) => { + headerArray.push( ...data ); + } ); + + stream.on( "end", () => { + const buffer = Buffer.from( headerArray ); + const version = buffer.readUInt8( 0 ); + if ( version !== 1 ) { + reject( `Unsupported data file version ${ version }. The maximum supported version is 1.` ); + return; + } + + const width = buffer.readUInt32BE( 1 ); + const height = buffer.readUInt32BE( 5 ); + const fileMeta: FileMeta = { + version: version, + width: width, + height: height, + bitDepth: buffer.readUInt8( 9 ), + minimumETo: buffer.readFloatBE( 10 ), + scalingFactor: buffer.readFloatBE( 14 ), + origin: { + x: Math.floor( width / 2 ), + // Account for the 30+10 cropped degrees. + y: Math.floor( height / ( 180 - 10 - 30) * ( 90 - 10 ) ) + } + }; + + if ( fileMeta.bitDepth === 8 ) { + resolve( fileMeta ); + } else { + reject( "Bit depths other than 8 are not currently supported." ); + } + } ); + } ); +} + +/** Information about the data file parsed from the file header. */ +interface FileMeta { + version: number; + /** The width of the image (in pixels). */ + width: number; + /** The height of the image (in pixels). */ + height: number; + /** The number of bits used for each pixel. */ + bitDepth: number; + /** The ETo that a pixel value of 0 represents (in inches/year). */ + minimumETo: number; + /** The ratio of an increase in pixel value to an increase in ETo (in inches/year). */ + scalingFactor: number; + /** + * The pixel coordinates of the geographic coordinates origin. These coordinates are off-center because the original + * image excludes the northernmost 10 degrees and the southernmost 30 degrees. + */ + origin: { + x: number; + y: number; + }; +} diff --git a/routes/weather.ts b/routes/weather.ts index 27e24a6..5c79203 100644 --- a/routes/weather.ts +++ b/routes/weather.ts @@ -37,7 +37,7 @@ const ADJUSTMENT_METHOD: { [ key: number ] : AdjustmentMethod } = { * @return A promise that will be resolved with the coordinates of the best match for the specified location, or * rejected with an error message if unable to resolve the location. */ -async function resolveCoordinates( location: string ): Promise< GeoCoordinates > { +export async function resolveCoordinates( location: string ): Promise< GeoCoordinates > { if ( !location ) { throw "No location specified"; @@ -428,7 +428,7 @@ function ipToInt( ip: string ): number { * @param parameter An array of parameters or a single parameter value. * @return The first element in the array of parameter or the single parameter provided. */ -function getParameter( parameter: string | string[] ): string { +export function getParameter( parameter: string | string[] ): string { if ( Array.isArray( parameter ) ) { parameter = parameter[0]; } diff --git a/server.ts b/server.ts index 493dc70..32158b0 100644 --- a/server.ts +++ b/server.ts @@ -6,6 +6,7 @@ import * as cors from "cors"; import * as weather from "./routes/weather"; import * as local from "./routes/weatherProviders/local"; +import * as baselineETo from "./routes/baselineETo"; let host = process.env.HOST || "127.0.0.1", port = parseInt( process.env.PORT ) || 3000; @@ -32,6 +33,10 @@ app.get( "/", function( req, res ) { res.send( process.env.npm_package_description + " v" + process.env.npm_package_version ); } ); +// Handle requests matching /baselineETo +app.options( /baselineETo/, cors() ); +app.get( /baselineETo/, cors(), baselineETo.getBaselineETo ); + // Handle 404 error app.use( function( req, res ) { res.status( 404 ); From 03b1f4c3412c264764b39ba44ae9b028baa37e04 Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Tue, 25 Jun 2019 21:55:21 -0400 Subject: [PATCH 09/10] Add precipitation to EToAdjustmentMethod rawData --- routes/adjustmentMethods/EToAdjustmentMethod.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts index 128cfe0..7af5cef 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -55,7 +55,8 @@ async function calculateEToWateringScale( maxT: Math.round( etoData.maxTemp ), minH: Math.round( etoData.minHumidity ), maxH: Math.round( etoData.maxHumidity ), - wind: Math.round( etoData.windSpeed * 10 ) / 10 + wind: Math.round( etoData.windSpeed * 10 ) / 10, + p: Math.round( wateringData.precip * 100 ) / 100 } } } From 3112498f3c47155559393b27c1b8b6d3a421670d Mon Sep 17 00:00:00 2001 From: Matthew Oslan Date: Sat, 6 Jul 2019 10:17:44 -0400 Subject: [PATCH 10/10] Error if a PWS is used with the ETo AdjustmentMethod --- routes/adjustmentMethods/EToAdjustmentMethod.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/routes/adjustmentMethods/EToAdjustmentMethod.ts b/routes/adjustmentMethods/EToAdjustmentMethod.ts index b8e11de..03a7846 100644 --- a/routes/adjustmentMethods/EToAdjustmentMethod.ts +++ b/routes/adjustmentMethods/EToAdjustmentMethod.ts @@ -16,6 +16,10 @@ async function calculateEToWateringScale( pws?: PWS ): Promise< AdjustmentMethodResponse > { + if ( pws ) { + throw "ETo adjustment method does not support personal weather stations through WUnderground."; + } + // Temporarily disabled since OWM forecast data is checking if rain is forecasted for 3 hours in the future. /* if ( wateringData && wateringData.raining ) {