Commit fe40c9e1 authored by lukas leufen's avatar lukas leufen

new data handler that combines kz filter and mixed sampling types

parent f3c6026c
Pipeline #51765 passed with stages
in 8 minutes and 29 seconds
......@@ -24,7 +24,7 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation):
_requirements = remove_items(inspect.getfullargspec(DataHandlerSingleStation).args, ["self", "station"])
def __init__(self, *args, kz_filter_length, kz_filter_iter, **kwargs):
assert kwargs.get("sampling") == "hourly" # This data handler requires hourly data resolution
self._check_sampling(**kwargs)
kz_filter_length = to_list(kz_filter_length)
kz_filter_iter = to_list(kz_filter_iter)
# self.original_data = None # ToDo: implement here something to store unfiltered data
......@@ -34,6 +34,9 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation):
self.cutoff_period_days = None
super().__init__(*args, **kwargs)
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == "hourly" # This data handler requires hourly data resolution
def setup_samples(self):
"""
Setup samples. This method prepares and creates samples X, and labels Y.
......
......@@ -2,6 +2,7 @@ __author__ = 'Lukas Leufen'
__date__ = '2020-11-05'
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler.data_handler_kz_filter import DataHandlerKzFilterSingleStation
from mlair.data_handler import DefaultDataHandler
from mlair.configuration import path_config
from mlair import helpers
......@@ -59,3 +60,38 @@ class DataHandlerMixedSampling(DefaultDataHandler):
data_handler = DataHandlerMixedSamplingSingleStation
data_handler_transformation = DataHandlerMixedSamplingSingleStation
_requirements = data_handler.requirements()
class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSingleStation,
DataHandlerKzFilterSingleStation):
_requirements1 = DataHandlerKzFilterSingleStation.requirements()
_requirements2 = DataHandlerMixedSamplingSingleStation.requirements()
_requirements = list(set(_requirements1 + _requirements2))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == ("hourly", "daily")
def setup_samples(self):
"""
Setup samples. This method prepares and creates samples X, and labels Y.
A KZ filter is applied on the input data that has hourly resolution. Lables Y are provided as aggregated values
with daily resolution.
"""
self._data = list(map(self.load_and_interpolate, [0, 1])) # load input (0) and target (1) data
self.set_inputs_and_targets()
self.apply_kz_filter()
if self.do_transformation is True:
self.call_transform()
self.make_samples()
class DataHandlerMixedSamplingWithFilter(DefaultDataHandler):
"""Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
data_handler = DataHandlerMixedSamplingWithFilterSingleStation
data_handler_transformation = DataHandlerMixedSamplingWithFilterSingleStation
_requirements = data_handler.requirements()
......@@ -4,7 +4,7 @@ __date__ = '2019-11-14'
import argparse
from mlair.workflows import DefaultWorkflow
from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampling
from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampling, DataHandlerMixedSamplingWithFilter
def main(parser_args):
......@@ -12,7 +12,9 @@ def main(parser_args):
sampling_inputs="hourly",
window_history_size=72,
**parser_args.__dict__,
data_handler=DataHandlerMixedSampling,
data_handler=DataHandlerMixedSampling, # WithFilter,
kz_filter_length=[365 * 24, 20 * 24],
kz_filter_iter=[3, 5],
start="2006-01-01",
train_start="2006-01-01",
end="2011-12-31",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment