Fix incompatability issue for moorings products

.github/workflows/test.yml

@@ -2,17 +2,17 @@ name: python-aodntools
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, v2 ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, v2 ]
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8' ]
+        python-version: [ '3.11' ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -22,7 +22,6 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install "numpy<1.19.0"
         pip install -r test_requirements.txt
         pip install pytest-cov
     - name: Test with pytest

aodntools/ncwriter/schema.py

@@ -1,55 +1,57 @@
 """This module holds schema definitions for validating the various :py:class:`dicts` that make up parts of a
 template, and also the helper functions necessary to validate an object against their respective schema.
 """
-
 import json
-
 import numpy as np
 from jsonschema import validators, Draft4Validator, FormatChecker, ValidationError
 from pkg_resources import resource_filename
 
+# helper function that will later be used to tell the schema validator how to validate objects of type "array"
+def is_array(checker, instance):
+    return isinstance(instance, (list, np.ndarray))
 
-# Create a new validator class (based on Draft4Validator) to allow templates to use
-# * Python types or numpy dtypes to specify variable data types; and
-# * numpy arrays to specify variable data.
-TemplateValidator = validators.create(meta_schema=Draft4Validator.META_SCHEMA,
-                                      validators=Draft4Validator.VALIDATORS)
-format_checker = FormatChecker()
+# Extend the default type checker by redefining "array"
+# whenever a schema expects a value of type "array", it will now use the is_array function to check if the value is acceptable.
+custom_type_checker = Draft4Validator.TYPE_CHECKER.redefine("array", is_array)
 
+# Create a custom validator that uses the new type checker.
+# any validation performed with CustomValidator will use the custom array checker 
+CustomValidator = validators.extend(Draft4Validator, type_checker=custom_type_checker)
+format_checker = FormatChecker()
 
+# Define a custom format checker
+# called when a JSON schema specifies that a value should have the format "datatype"
 @format_checker.checks('datatype')
 def is_python_datatype(value):
     """Return whether the given value is a valid data type specification for a NetCDF variable"""
     if isinstance(value, np.dtype):
         return True
     if isinstance(value, type):
         return issubclass(value, np.number)
-
     return False
 
-
-TYPES = {'array': (list, np.ndarray)}
-
+# Load JSON schema file
 TEMPLATE_SCHEMA_JSON = resource_filename(__name__, 'template_schema.json')
 with open(TEMPLATE_SCHEMA_JSON) as f:
     TEMPLATE_SCHEMA = json.load(f)
-TemplateValidator.check_schema(TEMPLATE_SCHEMA)
 
-template_validator = TemplateValidator(TEMPLATE_SCHEMA, types=TYPES, format_checker=format_checker)
+# Use the custom validator to check it is valid according to Draft 4 rules
+CustomValidator.check_schema(TEMPLATE_SCHEMA)
+
+# ready-to-use validator that applies both custom type and format checks
+template_validator = CustomValidator(TEMPLATE_SCHEMA, format_checker=format_checker)
 
 
+# Validation checks
 def validate_template(t):
     template_validator.validate(t)
 
-
 def validate_dimensions(d):
     validate_template({'_dimensions': d})
 
-
 def validate_variables(v):
     validate_template({'_variables': v})
-
-
+
 def validate_global_attributes(a):
     if hasattr(a, 'keys'):
         special = [k for k in a.keys() if k.startswith('_')]

aodntools/ncwriter/template.py

@@ -298,7 +298,7 @@ def create_variables(self, **kwargs):
 
         # variable attributes to convert to the same type as the variable
         # datatype
-        varattrs_to_convert_to_datatype = ['valid_min', 'valid_max', 'valid_range']
+        varattrs_to_convert_to_datatype = ['valid_min', 'valid_max', 'valid_range', 'flag_values']
 
         for varname, varattr in self.variables.items():
             if not varattr['_dimensions']:  # no kwargs in createVariable

aodntools/timeseries_products/aggregated_timeseries.py

@@ -212,7 +212,7 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
     # default name for temporary file. It will be renamed at the end
     fd, temp_outfile = tempfile.mkstemp(suffix='.nc', dir=output_dir)
     os.close(fd)
-    
+
     ## check files and get total number of flattened obs
     n_obs_total = 0
     for file in files_to_agg:
@@ -288,8 +288,8 @@ def main_aggregator(files_to_agg, var_to_agg, site_code, input_dir='', output_di
             LATITUDE[index] = nc.LATITUDE.values
             LONGITUDE[index] = nc.LONGITUDE.values
             NOMINAL_DEPTH[index] = get_nominal_depth(nc)
-            source_file[index] = stringtochar(np.array(file, dtype='S256'))
-            instrument_id[index] = stringtochar(np.array(get_instrument_id(nc), dtype='S256'))
+            source_file[index] = stringtochar(np.array(file, dtype='S256'), encoding='ascii')
+            instrument_id[index] = stringtochar(np.array(get_instrument_id(nc), dtype='S256'), encoding='ascii')
 
         start = end
 

aodntools/timeseries_products/common.py

@@ -2,6 +2,7 @@
 from datetime import datetime, timezone
 
 import numpy as np
+import xarray as xr
 
 # Common date/time format strings
 TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
@@ -179,7 +180,7 @@ def in_water_index(nc):
     """
     time_deployment_start = np.datetime64(nc.attrs['time_deployment_start'][:-1])
     time_deployment_end = np.datetime64(nc.attrs['time_deployment_end'][:-1])
-    TIME = nc['TIME'][:]
+    TIME = nc['TIME'].values
     return (TIME >= time_deployment_start) & (TIME <= time_deployment_end)
 
 def in_water(nc):
@@ -189,8 +190,12 @@ def in_water(nc):
     :param nc: xarray dataset
     :return: xarray dataset
     """
-    return nc.where(in_water_index(nc), drop=True)
-
+    condition = in_water_index(nc)  # NumPy boolean array
+    # Get the integer indices where condition is True.
+    indices = np.nonzero(condition)[0]
+    # Use positional indexing to select the TIME entries that satisfy the condition.
+    return nc.isel(TIME=indices)
 
 def current_utc_timestamp(format=TIMESTAMP_FORMAT):
     return datetime.now(timezone.utc).strftime(format)
+

aodntools/timeseries_products/hourly_timeseries.py

@@ -30,27 +30,27 @@ def check_files(file_list, site_code, parameter_names_accepted, input_dir=''):
     :param input_dir: base path where source files are stored
     :return: dictionary with the file name and list of failed tests, list good files chronologically ordered
     """
-
-    file_list_dataframe = pd.DataFrame(columns=["url", "deployment_date"])
+    rows = []
     error_dict = {}
 
     for file in file_list:
         with xr.open_dataset(os.path.join(input_dir, file)) as nc:
             error_list = check_file(nc, site_code, parameter_names_accepted)
             if error_list:
-                error_dict.update({file: error_list})
+                error_dict[file] = error_list
             else:
-                file_list_dataframe = file_list_dataframe.append({'url': file,
-                                                                  'deployment_date': parse(nc.time_deployment_start)},
-                                                                 ignore_index=True)
+                rows.append({
+                    'url': file,
+                    'deployment_date': parse(nc.time_deployment_start)
+                })
 
+    file_list_dataframe = pd.DataFrame(rows, columns=["url", "deployment_date"])
     file_list_dataframe = file_list_dataframe.sort_values(by='deployment_date')
-    file_list = file_list_dataframe['url'].to_list()
-    if file_list == []:
+    sorted_files = file_list_dataframe['url'].to_list()
+    if not sorted_files:
         raise NoInputFilesError("no valid input files to aggregate")
 
-    return file_list, error_dict
-
+    return sorted_files, error_dict
 
 
 def get_parameter_names(nc):
@@ -308,7 +308,7 @@ def PDresample_by_hour(df, function_dict, function_stats):
     df_data = pd.DataFrame(index=pd.DatetimeIndex([]))
     for variable in varnames:
         ds_var = df[variable]
-        ds_var_resample = ds_var.resample('1H', base=0.5)  # shift by half hour to centre bin on the hour
+        ds_var_resample = ds_var.resample('1h', offset='30min')  # shift by half hour to centre bin on the hour
         ds_var_mean = ds_var_resample.apply(function_dict[variable]).astype(np.float32)
         df_data = pd.concat([df_data, ds_var_mean], axis=1, sort=False)
         for stat_method in function_stats:
@@ -366,8 +366,6 @@ def hourly_aggregator(files_to_aggregate, site_code, qcflags, input_dir='', outp
         variable_attribute_dictionary = json.load(json_file)['_variables']
 
     df_data = pd.DataFrame()
-
-
     ## create empty DF with dtypes
     metadata_df_types = [('source_file', str),
                          ('instrument_id', str),
@@ -380,6 +378,7 @@ def hourly_aggregator(files_to_aggregate, site_code, qcflags, input_dir='', outp
     parameter_names_all = []
     applied_offset = []
     qc_count_all = {}
+    metadata_rows = []
 
     for file_index, file in enumerate(files_to_aggregate):
         print(file_index)
@@ -398,13 +397,16 @@ def hourly_aggregator(files_to_aggregate, site_code, qcflags, input_dir='', outp
             qc_count = get_QCcount(nc_clean, qcflags)
             qc_count_all = update_QCcount(qc_count_all, qc_count)
             nc_clean = good_data_only(nc_clean, qcflags)  # good quality data only
-            df_metadata = df_metadata.append({'source_file': file,
-                                              'instrument_id': utils.get_instrument_id(nc),
-                                              'LONGITUDE': nc.LONGITUDE.squeeze().values,
-                                              'LATITUDE': nc.LATITUDE.squeeze().values,
-                                              'NOMINAL_DEPTH': get_nominal_depth(nc)},
-                                             ignore_index=True)
-
+
+            # Append a new row as a dictionary to the list.
+            metadata_rows.append({
+                'source_file': file,
+                'instrument_id': utils.get_instrument_id(nc),
+                'LONGITUDE': nc.LONGITUDE.squeeze().values,
+                'LATITUDE': nc.LATITUDE.squeeze().values,
+                'NOMINAL_DEPTH': get_nominal_depth(nc)
+            })
+
             # If TIME had out-of-range values before cleaning, nc_clean would now have a CFTimeIndex, which
             # breaks the resampling further down. Here we reset it to a DatetimeIndex as suggested here:
             # https://stackoverflow.com/questions/55786995/converting-cftime-datetimejulian-to-datetime/55787899#55787899
@@ -421,6 +423,7 @@ def hourly_aggregator(files_to_aggregate, site_code, qcflags, input_dir='', outp
             df_temp['instrument_index'] = np.repeat(file_index, len(df_temp)).astype(np.int32)
             df_data = pd.concat([df_data, df_temp.reset_index()], ignore_index=True, sort=False)
 
+    df_metadata = pd.DataFrame(metadata_rows, columns=['source_file', 'instrument_id', 'LONGITUDE', 'LATITUDE', 'NOMINAL_DEPTH'])
     df_metadata.index.rename('INSTRUMENT', inplace=True)
     df_data.index.rename('OBSERVATION', inplace=True)
     ## rename index to TIME

aodntools/timeseries_products/velocity_aggregated_timeseries.py

@@ -73,6 +73,7 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
     # default name for temporary file. It will be renamed at the end
     fd, temp_outfile = tempfile.mkstemp(suffix='.nc', dir=output_dir)
     os.close(fd)
+
     ## check files and get total number of flattened obs
     n_obs_total = 0
     for file in files_to_agg:
@@ -168,8 +169,8 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
             LATITUDE[index] = nc.LATITUDE.values
             LONGITUDE[index] = nc.LONGITUDE.values
             NOMINAL_DEPTH[index] = utils.get_nominal_depth(nc)
-            source_file[index] = stringtochar(np.array(file, dtype='S256'))
-            instrument_id[index] = stringtochar(np.array(utils.get_instrument_id(nc), dtype='S256'))
+            source_file[index] = stringtochar(np.array(file, dtype='S256'), encoding='ascii')
+            instrument_id[index] = stringtochar(np.array(utils.get_instrument_id(nc), dtype='S256'), encoding='ascii')
             ## add time offset to the middle of the measuring window, if it exists
             if 'seconds_to_middle_of_measurement' in nc.TIME.attrs:
                 SECONDS_TO_MIDDLE[index] = nc.TIME.seconds_to_middle_of_measurement
@@ -237,7 +238,6 @@ def velocity_aggregated(files_to_agg, site_code, input_dir='', output_dir='./',
                             ("velocity-"+product_type),
                             ('END-'+ time_end_filename), 'C-' + current_utc_timestamp(DATESTAMP_FORMAT)]) + '.nc'
     ncout_path = os.path.join(output_dir, output_name)
-
     shutil.move(temp_outfile, ncout_path)
 
 

aodntools/timeseries_products/velocity_hourly_timeseries.py

@@ -58,7 +58,7 @@ def append_resampled_values(nc_cell, ds, slice_start, binning_functions):
     # shift the index forward 30min to centre the bins on the hour
     df_cell.index = df_cell.index + pd.Timedelta(minutes=30)
 
-    df_cell_1H = df_cell.resample('1H')
+    df_cell_1H = df_cell.resample('1h')
     slice_end = len(df_cell_1H) + slice_start
 
     # set binned timestamps
@@ -107,6 +107,7 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
     ## default name for temporary file. It will be renamed at the end
     fd, temp_outfile = tempfile.mkstemp(suffix='.nc', dir=output_dir)
     os.close(fd)
+
     ## check files and get total number of flattened obs
     print("CHECKING FILES...")
     for index, file in enumerate(files_to_agg):
@@ -230,8 +231,8 @@ def velocity_hourly_aggregated(files_to_agg, site_code, input_dir='', output_dir
             LATITUDE[index] = nc.LATITUDE.values
             LONGITUDE[index] = nc.LONGITUDE.values
             NOMINAL_DEPTH[index] = np.array(utils.get_nominal_depth(nc))
-            source_file[index] = stringtochar(np.array(file, dtype='S256'))
-            instrument_id[index] = stringtochar(np.array(utils.get_instrument_id(nc), dtype='S256'))
+            source_file[index] = stringtochar(np.array(file, dtype='S256'), encoding='ascii')
+            instrument_id[index] = stringtochar(np.array(utils.get_instrument_id(nc), dtype='S256'), encoding='ascii')
             ## add time offset to the middle of the measuring window, if it exists
             if 'seconds_to_middle_of_measurement' in nc.TIME.attrs:
                 SECONDS_TO_MIDDLE[index] = nc.TIME.seconds_to_middle_of_measurement

constraints.txt

@@ -1,4 +0,0 @@
-cftime<1.1.1;python_version=='3.5'
-netCDF4<1.5.4;python_version=='3.5'
-pandas<0.25.0;python_version=='3.5'
-xarray<0.14.0;python_version=='3.5'

examples/rottnest.py

@@ -46,7 +46,8 @@
     var_type = var['_datatype']
     for attr in ('valid_min', 'valid_max'):
         if attr in var:
-            var[attr] = np.cast[var_type](var[attr])
+            var[attr] = np.array(var[attr], dtype=var_type)
+
 
 # update range attributes
 template.add_extent_attributes()

setup.py

@@ -1,11 +1,11 @@
 from setuptools import setup, find_packages
 
 INSTALL_REQUIRES = [
-    'jsonschema>=2.6.0,<3.0.0',
-    'numpy>=1.13.0',
-    'netCDF4>=1.5.3',
-    'pandas>=0.24.2',
-    'xarray>=0.11.3'
+    'jsonschema>=4.23.0',
+    'numpy>=2.2.4',
+    'netCDF4>=1.7.2',
+    'pandas>=2.2.3',
+    'xarray>=2023.1.0,<=2025.3.1'
 ]
 
 TESTS_REQUIRE = [
@@ -37,7 +37,7 @@
     author_email='projectofficers@emii.org.au',
     description='AODN data tools library',
     zip_safe=False,
-    python_requires='>=3.5',
+    python_requires='>=3.11, <3.12',
     install_requires=INSTALL_REQUIRES,
     tests_require=TESTS_REQUIRE,
     extras_require=EXTRAS_REQUIRE,
@@ -49,8 +49,7 @@
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: Implementation :: CPython',
     ]
 )

test_aodntools/timeseries_products/test_common.py

@@ -5,7 +5,7 @@
 
 import xarray as xr
 
-from aodntools.timeseries_products.common import (check_file, check_velocity_file, get_qc_variable_names,
+from aodntools.timeseries_products.common import (check_file, check_velocity_file, fixed_width_string_to_char, get_qc_variable_names,
                                                   check_imos_flag_conventions, in_water_index, in_water)
 
 TEST_ROOT = os.path.dirname(__file__)
@@ -46,6 +46,15 @@ def test_check_flag_conventions_bad(self):
         self.assertEqual(errors, ['unexpected quality_control_conventions: "WOCE quality control procedure"'])
 
 
+class TestFixedWidthStringToChar(unittest.TestCase):
+    def test_scalar_string_has_fixed_width_character_representation(self):
+        characters = fixed_width_string_to_char('example.nc')
+
+        self.assertEqual(characters.shape, (256,))
+        self.assertEqual(characters.dtype.itemsize, 1)
+        self.assertEqual(characters.tobytes(), b'example.nc' + b'\0' * (256 - len('example.nc')))
+
+
 class TestCheckFile(unittest.TestCase):
     def test_good_temp_file(self):
         with xr.open_dataset(GOOD_TZ_FILE) as nc: