Move tutorial workflow away from datapoint

Author: wxtim

changes.d/7044.feat.md

@@ -0,0 +1 @@
+Replace use of Met Office Datapoint (Switchoff December 2025) in tutorial with Met Office data via the Amazon Data Sustainability Initative.

conda-environment.yml

@@ -23,3 +23,5 @@ dependencies:
   #- pympler
   #- matplotlib-base
   #- sqlparse
+  #- h5py
+  #- requests

cylc/flow/etc/tutorial/api-keys

@@ -34,22 +34,18 @@
         script = get-observations
         [[[environment]]]
             SITE_ID = 3917
-            API_KEY = DATAPOINT_API_KEY
     [[get_observations_camborne]]
         script = get-observations
         [[[environment]]]
             SITE_ID = 3808
-            API_KEY = DATAPOINT_API_KEY
     [[get_observations_heathrow]]
         script = get-observations
         [[[environment]]]
             SITE_ID = 3772
-            API_KEY = DATAPOINT_API_KEY
     [[get_observations_shetland]]
         script = get-observations
         [[[environment]]]
             SITE_ID = 3005
-            API_KEY = DATAPOINT_API_KEY
 
 
     [[consolidate_observations]]
@@ -58,26 +54,23 @@
             # The dimensions of each grid cell in degrees.
             RESOLUTION = 0.2
             # The area to generate forecasts for (lng1, lat1, lng2, lat2)
-            DOMAIN = -12,48,5,61  # Do not change!
+            DOMAIN = -12,46,12,61  # Do not change!
 
     [[get_rainfall]]
         script = get-rainfall
         [[[environment]]]
-            # The key required to get weather data from the DataPoint service.
-            # To use archived data comment this line out.
-            API_KEY = DATAPOINT_API_KEY
             # The dimensions of each grid cell in degrees.
             RESOLUTION = 0.2
             # The area to generate forecasts for (lng1, lat1, lng2, lat2)
-            DOMAIN = -12,48,5,61  # Do not change!
+            DOMAIN = -12,46,12,61  # Do not change!
 
     [[forecast]]
         script = forecast 60 5  # Generate 5 forecasts at 60 minute intervals.
         [[[environment]]]
             # The dimensions of each grid cell in degrees.
             RESOLUTION = 0.2
             # The area to generate forecasts for (lng1, lat1, lng2, lat2)
-            DOMAIN = -12,48,5,61  # Do not change!
+            DOMAIN = -12,46,12,61  # Do not change!
             # The path to the files containing wind data (the {variables} will
             # get substituted in the forecast script).
             WIND_FILE_TEMPLATE = $CYLC_WORKFLOW_WORK_DIR/{cycle}/consolidate_observations/wind_{xy}.csv
@@ -97,6 +90,6 @@
             # The dimensions of each grid cell in degrees.
             RESOLUTION = 0.2
             # The area to generate forecasts for (lng1, lat1, lng2, lat2)
-            DOMAIN = -12,48,5,61  # Do not change!
+            DOMAIN = -12,46,12,61  # Do not change!
 
 {% include 'etc/python-job.settings' %}

cylc/flow/etc/tutorial/consolidation-tutorial/flow.cylc

@@ -16,11 +16,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 set -eux
-APIKEY="$(head --lines 1 ../api-keys)"
 FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
 cylc lint .
 cylc install --workflow-name "$FLOW_NAME" --no-run-name
-sed -i "s/DATAPOINT_API_KEY/$APIKEY/" "$HOME/cylc-run/$FLOW_NAME/flow.cylc"
 cylc validate --check-circular --icp=2000 "$FLOW_NAME"
 cylc play --no-detach --abort-if-any-task-fails "$FLOW_NAME"
 cylc clean "$FLOW_NAME"

cylc/flow/etc/tutorial/cylc-forecasting-workflow/.validate

@@ -61,7 +61,7 @@ def plot_wind_data(wind_x, wind_y, x_range, y_range, x_coords, y_coords,
                [x[0] for x in z_coords],
                [y[1] for y in z_coords],
                color='red')
-    plt.savefig('wind.png')
+    plt.savefig(f'{os.environ["CYLC_TASK_LOG_DIR"]}/wind.png')
 
 
 def get_wind_fields():

cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/consolidate-observations

@@ -150,11 +150,12 @@ def push_rainfall(rainfall, wind_data, step, resolution, spline_level):
                                 dim_x, dim_y, resolution, resolution,
                                 spline_level)
 
+    domain = util.parse_domain(os.environ['DOMAIN'])
+
     while True:
         out_of_bounds = []
         for itt in range(len(x_values)):
             try:
-                domain = util.parse_domain(os.environ['DOMAIN'])
                 lng = domain['lng1'] + x_values[itt]
                 lat = domain['lat1'] + y_values[itt]
 

cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/forecast

@@ -24,11 +24,8 @@ Usage:
     get-observations
 
 Environment Variables:
-    SITE_ID: The four digit DataPoint site code identifying the weather station
-        we are to fetch results for.
-    API_KEY: The DataPoint API key, required for getting live weather data.
-        If un-specified then get-observations will fall back to archive data
-        from the workflow directory.
+    SITE_ID: The four digit WMO (World Meteorological Organization)
+    site code identifying the weather station we are to fetch results for.
 
 """
 
@@ -43,10 +40,6 @@ import requests
 import util
 
 
-BASE_URL = ('http://datapoint.metoffice.gov.uk/public/data/'
-            'val/wxobs/all/json/{site_id}'
-            '?res=hourly&time={time}&key={api_key}')
-
 # Compass bearings for ordinate directions.
 # NOTE: We measure direction by where the winds have come from!
 ORDINATES = {
@@ -68,22 +61,52 @@ ORDINATES = {
     'NNW': '157.5'
 }
 
+class Meteorology:
+    """
+    Surface winds tend to be 20-30 degrees backed (vector anticlockwise)
+    from the winds which steer weather systems:
+    Friction with the ground surface tends to mean that land surface winds
+    are as slow as half the speed of the wind at 2000ft. This fudge factor is
+    a more conservative 1.5:
 
-class NoDataException(Exception):
-    ...
+    .. seealso::
+
+       [Source Book to the Forecaster's Reference Book](https://digital.nmla
+       .metoffice.gov.uk/IO_011f7cd4-50fc-4903-b556-d24480ea883d/), section
+       1.2
+    """
+    SURFACE_BACKING = 2
+    SURFACE_FRICTION = .66
+    KT_TO_MPH = 1.15078
+
+    @staticmethod
+    def process_direction(direction: str) -> str:
+        """Process raw wind direction:
+
+        * Convert direction from 10s of degrees to degrees.
+        * Convert from Oceanographic (wind to) to Meteorological (wind from)
+          convention.
+        * Surface Friction Correction
+        """
+        return str(
+            ((int(direction) + 18 - Meteorology.SURFACE_BACKING) % 36) * 10)
+
+    @staticmethod
+    def process_speed(speed: str) -> str:
+        """Process Raw wind speed
+
+        * Convert to KT to MPH
+        * Surface Friction Correction
+        """
+        return str(
+            (
+                int(speed) * Meteorology.KT_TO_MPH
+            ) / Meteorology.SURFACE_FRICTION
+        )
 
 
-def get_datapoint_data(site_id, time, api_key):
-    """Get weather data from the DataPoint service."""
-    # The URL required to get the data.
-    time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y-%m-%dT%H:%MZ')
-    url = BASE_URL.format(time=time, site_id=site_id, api_key=api_key)
-    req = requests.get(url)
-    if req.status_code != 200:
-        raise Exception(f'{url} returned exit code {req.status_code}')
-    # Get the data and parse it as JSON.
-    print('Opening URL: %s' % url)
-    return req.json()['SiteRep']['DV']['Location']
+class NoDataException(Exception):
+    ...
 
 
 def get_archived_data(site_id, time):
@@ -152,11 +175,12 @@ def synop_grab(site_id, time):
         raise NoDataException(
             f'Request for data failed, raw request was {req.text}')
 
-    # Parse the direction from 10's of degrees to degrees:
-    data['direction'] = str(int(data['direction']) * 10)
+    # * Parse the direction from 10's of degrees to degrees
+    # * Convert direction from to direction it's blowing to
+    data['direction'] = Meteorology.process_direction(data['direction'])
 
     # Convert data in KT to MPH:
-    data['speed'] = str(int(data['speed']) * 1.15078)
+    data['speed'] = Meteorology.process_speed(data['speed'])
 
     # Get lat and long from MO Data:
     lat, lon = reference_lat_long(site_id)
@@ -185,7 +209,7 @@ def get_nearby_site(site_id, badsites):
     return int(result[0]), dist
 
 
-def main(site_id, api_key=None):
+def main(site_id):
     cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']
 
     # Try to get the information from SYNOPS.
@@ -202,13 +226,8 @@ def main(site_id, api_key=None):
             site_id, dist = get_nearby_site(site_id, badsites)
 
     if obs is None:
-        if api_key:
-            print('Attempting to get weather data from DataPoint...')
-            data = get_datapoint_data(site_id, cycle_point, api_key)
-        else:
-            print('No API key provided, falling back to archived data')
-            data = get_archived_data(site_id, cycle_point)
-
+        print('Obs unavailable, falling back to archived data')
+        data = get_archived_data(site_id, cycle_point)
         obs = extract_observations(data)
 
     # Write observations.
@@ -218,5 +237,4 @@ def main(site_id, api_key=None):
 
 if __name__ == '__main__':
     util.sleep()
-    main(os.environ['SITE_ID'],
-         os.environ.get('API_KEY'))
+    main(os.environ['SITE_ID'])