diff --git a/common.php b/common.php index 5f6e79a..f398f63 100644 --- a/common.php +++ b/common.php @@ -64,10 +64,10 @@ public function validate($processitem) if ($setting['type']=="interval") { // check if interval is valid - if (is_numeric($processitem->key)) { - $processitem->key = (int) $processitem->key; + if (is_numeric($processitem->{$key})) { + $processitem->{$key} = (int) $processitem->{$key}; } - if (!in_array($processitem->key, array('original',10,30,60,1800,3600))) { + if (!in_array($processitem->{$key}, array('original',10,30,60,1800,3600), true)) { return array("success"=>false,"message"=>"invalid interval"); } } @@ -119,120 +119,559 @@ function getmeta($dir, $id) function createmeta($dir, $id, $meta) { - $metafile = fopen($dir . $id . ".meta", 'wb'); - fwrite($metafile, pack("I", 0)); - fwrite($metafile, pack("I", 0)); - fwrite($metafile, pack("I", $meta->interval)); - fwrite($metafile, pack("I", $meta->start_time)); + if (!$metafile = @fopen($dir . $id . ".meta", 'wb')) return false; + $bytes = pack("IIII", 0, 0, $meta->interval, $meta->start_time); + $written = fwrite($metafile, $bytes); fclose($metafile); + return $written === strlen($bytes); } -//compute meta datas of different feeds intended for a preprocessing -function compute_meta(): stdClass +// -------------------------------------------------------------------------------- +// Block based feed access +// +// FeedReader and FeedWriter read and write feeds in blocks rather than one datapoint +// at a time, which is several times faster in PHP. Errors are thrown as +// PostProcessException, which the runner reports as a failed process. +// -------------------------------------------------------------------------------- + +class PostProcessException extends Exception {} + +// Number of datapoints to read or write at a time +const POSTPROCESS_BLOCKSIZE = 65536; + +// Start time of this run of a process for the output feed meta (start_time, interval, npoints) +// recent: from the end of the output feed +// from: from process_start, aligned to the output interval, no later than the end of the output feed +// all: from the start of the output feed +function resolve_start_time($params, $meta) { - $numargs = func_num_args(); - $arg_list = func_get_args(); - $all_intervals = []; - $all_start_times = []; - $all_ending_times = []; - $meta = new stdClass(); - for ($i = 0; $i < $numargs; $i++) { - $all_intervals[] = $arg_list[$i]->interval; - $all_start_times[] = $arg_list[$i]->start_time; - $all_ending_times[] = $arg_list[$i]->start_time + $arg_list[$i]->npoints * $arg_list[$i]->interval; - } - $meta->interval = max($all_intervals); - $meta->start_time = floor(max($all_start_times) / $meta->interval) * $meta->interval; - $meta->writing_end_time = min($all_ending_times); - //print("intervals.....");print_r($all_intervals); - //print("start_times....");print_r($all_start_times); - //print("ending_times.....");print_r($all_ending_times); - print("NOTICE : output interval=$meta->interval, start=$meta->start_time, end=$meta->writing_end_time \n"); - return $meta; + $end_time = $meta->start_time + ($meta->npoints * $meta->interval); + $mode = isset($params->process_mode) ? $params->process_mode : 'recent'; + + if ($mode == 'recent') { + return $end_time; + } + if ($mode == 'from') { + $start_time = max((int) $params->process_start, $meta->start_time); + $start_time = $meta->start_time + floor(($start_time - $meta->start_time) / $meta->interval) * $meta->interval; + return min($start_time, $end_time); + } + return $meta->start_time; +} + +class FeedReader +{ + public $id; + public $meta; + private $fh; + + public function __construct($dir, $id) + { + $this->id = (int) $id; + if (!file_exists($dir.$this->id.".meta") || !file_exists($dir.$this->id.".dat")) { + throw new PostProcessException("feed ".$this->id." does not exist"); + } + $this->meta = getmeta($dir, $this->id); + $this->meta->npoints = (int) $this->meta->npoints; + if (!$this->meta->interval) { + throw new PostProcessException("feed ".$this->id." has no interval"); + } + if (!$this->fh = @fopen($dir.$this->id.".dat", 'rb')) { + throw new PostProcessException("could not open feed ".$this->id); + } + } + + public function __destruct() + { + if ($this->fh) fclose($this->fh); + } + + // Read $n datapoints starting at position $pos, positions outside of the feed are NAN + public function read($pos, $n) + { + $from = max($pos, 0); + $to = min($pos + $n, $this->meta->npoints); + + if ($from == $pos && $to == $pos + $n) { + fseek($this->fh, $from * 4); + $values = array_values(unpack("f*", fread($this->fh, $n * 4))); + if (count($values) == $n) return $values; + } + + $values = array_fill(0, $n, NAN); + if ($to > $from) { + fseek($this->fh, $from * 4); + $i = $from - $pos; + foreach (unpack("f*", fread($this->fh, ($to - $from) * 4)) as $value) { + $values[$i++] = $value; + } + } + return $values; + } + + // Read $n values on the timeline $start_time + i * $interval, which may differ from + // the feed's own start time and interval. Each value covers [time, time + interval): + // - feed interval shorter: mean of the datapoints within the interval, ignoring NAN + // - feed interval longer: the datapoint covering the start of the interval + public function read_time($start_time, $interval, $n) + { + $feed_start = $this->meta->start_time; + $feed_interval = $this->meta->interval; + + if ($feed_interval == $interval && ($start_time - $feed_start) % $feed_interval == 0) { + return $this->read(intdiv($start_time - $feed_start, $feed_interval), $n); + } + + $block_start = (int) floor(($start_time - $feed_start) / $feed_interval); + $block_end = (int) ceil(($start_time + $n * $interval - $feed_start) / $feed_interval); + $data = $this->read($block_start, $block_end - $block_start); + + $values = array(); + for ($i=0; $i<$n; $i++) { + $time = $start_time + $i * $interval; + $pos_start = (int) ceil(($time - $feed_start) / $feed_interval); + $pos_end = (int) ceil(($time + $interval - $feed_start) / $feed_interval); + if ($pos_end <= $pos_start) { + $pos_start = (int) floor(($time - $feed_start) / $feed_interval); + $pos_end = $pos_start + 1; + } + $sum = 0; + $count = 0; + for ($pos=$pos_start; $pos<$pos_end; $pos++) { + $value = $data[$pos - $block_start]; + if (!is_nan($value)) { + $sum += $value; + $count++; + } + } + $values[] = $count ? $sum / $count : NAN; + } + return $values; + } } -/* -format an array for the formula engine -$b is the result of a preg_match on a formula chunk with the regexp /($Xop)?($Xnbr)?($Xf)?/ -$Xop : regexp for operator -$Xnbr : regexp for float or int value -$Xf : regexp for a feed -cf process basic_formula for more details on $Xop,$Xnbr,$Xf -ouputs a formula element as a 3 elements vector : -0 -> type of data ie "feed" ou "value" -1 -> operator -2 -> value or feed number -*/ -function ftoa($b): array +// Result message for a process that has written to $writer +function written_message($bytes, $writer) { - $c = []; - //print_r($b); - if (sizeof(value: $b) == 4) { - $c[0] = "feed"; - $c[2] = intval(value: substr(string: $b[3], offset: 1)); - } else { - $c[0] = "value"; - $c[2] = $b[2]; - } - $c[1] = $b[1]; - if (!$c[2]) $c[2] = 1; - if (!$c[1]) $c[1] = '+'; - return $c; + $message = "bytes written: ".$bytes; + if ($writer->last_time() !== false) { + $message .= ", last time value: ".$writer->last_time()." ".$writer->last_value(); + } + return $message; } -/* -output of a basic formula for a specified time -the formula is described by the $elements array > a third dimensionnal array > datas are on level 2 -for a given level 1, we operate multiplication/division within level 2 elements and give a sign to the result -the final result is the addition of the whole -a formula element is a 3 elements vector : -0 -> type of data ie "feed" ou "value" -1 -> operator -2 -> value or feed number -feeds_meta : array of metas such as produced by getmeta -*/ -function bfo($elements, $feeds_meta, $feeds_dat, $time): float|int +class FeedWriter { - $s = []; - foreach ($elements as $element) { - $values = []; - foreach ($element as $e) { - $value = NAN; - if ($e[0] == "feed") { - $pos = floor(num: ($time - $feeds_meta[$e[2]]->start_time) / $feeds_meta[$e[2]]->interval); - if ($pos >= 0 && $pos < $feeds_meta[$e[2]]->npoints) { - fseek(stream: $feeds_dat[$e[2]], offset: $pos * 4); - $tmp = unpack("f", fread($feeds_dat[$e[2]], 4)); - $value = $tmp[1]; - } + public $id; + public $meta; + private $dir; + private $fh; + private $buffer = ""; + private $pos = 0; + private $meta_pending = false; + private $bytes_written = 0; + private $last_time = false; + private $last_value = NAN; + // Set false when rewriting part of a feed, so that the feed's last time and value are left alone + public $update_redis = true; + + public function __construct($dir, $id) + { + $this->dir = $dir; + $this->id = (int) $id; + if (!file_exists($dir.$this->id.".meta")) { + throw new PostProcessException("feed ".$this->id." does not exist"); + } + if (!$this->fh = @fopen($dir.$this->id.".dat", 'c+')) { + throw new PostProcessException("could not open feed ".$this->id." for writing"); + } + $this->meta = getmeta($dir, $this->id); + $this->meta->npoints = (int) $this->meta->npoints; + } + + public function __destruct() + { + if ($this->fh) fclose($this->fh); + } + + // Set the start time and interval of an output feed. An empty feed takes them (written + // with the first data), a feed with data must already match unless $reset is set, in + // which case existing data is discarded. + public function init($start_time, $interval, $reset = false) + { + if ($this->meta->npoints > 0 && !$reset) { + if ($this->meta->start_time != $start_time) { + throw new PostProcessException("feed ".$this->id." start time mismatch"); + } + if ($this->meta->interval != $interval) { + throw new PostProcessException("feed ".$this->id." interval mismatch"); + } + return; + } + if ($this->meta->npoints > 0) { + if (!ftruncate($this->fh, 0)) throw new PostProcessException("could not truncate feed ".$this->id); + } + $this->meta->start_time = $start_time; + $this->meta->interval = $interval; + $this->meta->npoints = 0; + $this->meta->end_time = $start_time; + $this->pos = 0; + $this->meta_pending = true; + } + + // Move the write position to $time, a time beyond the end of the feed is padded with NAN + public function seek($time) + { + $offset = $time - $this->meta->start_time; + if (!$this->meta->interval || $offset < 0 || $offset % $this->meta->interval != 0) { + throw new PostProcessException("feed ".$this->id." invalid write time $time"); + } + $this->flush(); + $pos = intdiv($offset, $this->meta->interval); + if ($pos > $this->meta->npoints) { + fseek($this->fh, $this->meta->npoints * 4); + $this->pos = $this->meta->npoints; + $this->write_block(array_fill(0, $pos - $this->meta->npoints, NAN)); + } else { + fseek($this->fh, $pos * 4); + $this->pos = $pos; + } + } + + public function write($value) + { + $this->buffer .= pack("f", $value); + if (!is_nan($value)) { + $this->last_time = $this->meta->start_time + $this->pos * $this->meta->interval; + $this->last_value = $value; + } + $this->pos++; + if (strlen($this->buffer) >= POSTPROCESS_BLOCKSIZE * 4) $this->flush(); + } + + public function write_block($values) + { + $n = count($values); + if (!$n) return; + $this->buffer .= pack("f*", ...$values); + for ($i=$n-1; $i>=0; $i--) { + if (!is_nan($values[$i])) { + $this->last_time = $this->meta->start_time + ($this->pos + $i) * $this->meta->interval; + $this->last_value = $values[$i]; + break; + } + } + $this->pos += $n; + if (strlen($this->buffer) >= POSTPROCESS_BLOCKSIZE * 4) $this->flush(); + } + + // Time of the next datapoint to be written + public function time() + { + return $this->meta->start_time + $this->pos * $this->meta->interval; + } + + public function flush() + { + if ($this->buffer === "") return; + if ($this->meta_pending) { + if (!createmeta($this->dir, $this->id, $this->meta)) { + throw new PostProcessException("could not write meta for feed ".$this->id); + } + $this->meta_pending = false; + } + $length = strlen($this->buffer); + $written = fwrite($this->fh, $this->buffer); + if ($written !== $length) { + throw new PostProcessException("feed ".$this->id.": wrote $written of $length bytes, disk full?"); + } + $this->bytes_written += $length; + $this->meta->npoints = max($this->meta->npoints, $this->pos); + $this->meta->end_time = $this->meta->start_time + $this->meta->npoints * $this->meta->interval; + $this->buffer = ""; + } + + // Flush remaining data and close. $truncate removes any existing data after the last + // datapoint written, use when reprocessing so that no stale data is left at the end. + // Updates the feed's last time and value in redis. Returns the number of bytes written. + public function close($truncate = false) + { + $this->flush(); + if ($truncate && $this->pos < $this->meta->npoints) { + if (!ftruncate($this->fh, $this->pos * 4)) { + throw new PostProcessException("could not truncate feed ".$this->id); + } + $this->meta->npoints = $this->pos; + $this->meta->end_time = $this->time(); + } + fclose($this->fh); + $this->fh = false; + if ($this->update_redis && $this->last_time !== false) { + updatetimevalue($this->id, $this->last_time, $this->last_value); + } + return $this->bytes_written; + } + + // Value of the datapoint at $time, NAN if outside of the feed + public function value_at($time) + { + $this->flush(); + $offset = $time - $this->meta->start_time; + if ($offset < 0 || $offset % $this->meta->interval != 0) return NAN; + $pos = intdiv($offset, $this->meta->interval); + if ($pos >= $this->meta->npoints) return NAN; + fseek($this->fh, $pos * 4); + $value = unpack("f", fread($this->fh, 4))[1]; + fseek($this->fh, $this->pos * 4); + return $value; + } + + public function last_time() { return $this->last_time; } + public function last_value() { return $this->last_value; } +} + +// -------------------------------------------------------------------------------- +// Base classes for processes that transform each value of a feed +// +// Subclasses implement value($value), called for each datapoint that is not NAN, or +// values($values) to handle a block of datapoints at a time, e.g. to keep state between +// datapoints. reset() is called at the start of each run to reset any such state. +// Process settings are available as $this->params. +// -------------------------------------------------------------------------------- + +abstract class ValueProcess extends PostProcess_common +{ + protected $params; + + protected function reset() {} + + protected function value($value) + { + return $value; + } + + protected function values($values) + { + foreach ($values as $i => $value) { + if (!is_nan($value)) $values[$i] = $this->value($value); + } + return $values; + } +} + +// Writes the transformed values of feed setting "input" to feed setting "output", with the +// same start time and interval. Supports recent, from and all process modes, reprocessing +// all replaces the output feed. +abstract class MapProcess extends ValueProcess +{ + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $input = new FeedReader($this->dir, $params->input); + $output = new FeedWriter($this->dir, $params->output); + $interval = $input->meta->interval; + $output->init($input->meta->start_time, $interval, $params->process_mode == 'all'); + + $start_time = resolve_start_time($params, $output->meta); + $end_time = $input->meta->end_time; + if ($start_time >= $end_time) { + return array("success"=>true, "message"=>"output feed already up to date"); } - if ($e[0] == "value") $value = $e[2]; - if (!is_nan(num: $value) && $value != 0) { - if ($e[1] == "/") $value = 1 / $value; - if ($e[1] == "-") $value = -$value; + + $this->params = $params; + $this->reset(); + $output->seek($start_time); + while (($time = $output->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $output->write_block($this->values($input->read_time($time, $interval, $n))); } - $values[] = $value; + // Reprocessing removes any output data beyond the end of the input + $bytes = $output->close($params->process_mode != 'recent'); + + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); } - if (!in_array(needle: NAN, haystack: $values)) { - $s[] = array_product(array: $values); - } else $s[] = NAN; + return array("success"=>true, "message"=>written_message($bytes, $output)); } - if (!in_array(needle: NAN, haystack: $s)) { - $sum = array_sum(array: $s); - } else $sum = NAN; +} + +// Transforms the values of feed setting "feedid" in place. Always processes the whole feed, +// only blocks containing changed values are written. +abstract class InPlaceProcess extends ValueProcess +{ + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $input = new FeedReader($this->dir, $params->feedid); + $output = new FeedWriter($this->dir, $params->feedid); + $output->update_redis = false; + $npoints = $input->meta->npoints; + if ($npoints == 0) { + return array("success"=>false, "message"=>"feed is empty"); + } + + $this->params = $params; + $this->reset(); + $changed = 0; + for ($pos=0; $pos<$npoints; $pos+=POSTPROCESS_BLOCKSIZE) { + $values = $input->read($pos, min(POSTPROCESS_BLOCKSIZE, $npoints - $pos)); + $new_values = $this->values($values); + $block_changed = 0; + foreach ($values as $i => $value) { + $new_value = $new_values[$i]; + if (is_nan($value) ? !is_nan($new_value) : (is_nan($new_value) || $new_value != $value)) { + $block_changed++; + } + } + if ($block_changed) { + $output->seek($input->meta->start_time + $pos * $input->meta->interval); + $output->write_block($new_values); + $changed += $block_changed; + } + } + $output->close(); - return $sum; + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); + } + return array("success"=>true, "message"=>"modified: ".$changed." datapoints, ".round(($changed / $npoints) * 100)."%"); + } } +// Combines input feeds into an output feed on a common timeline: the longest input interval, +// from the latest input start time to the earliest input end time. Inputs with a shorter +// interval are averaged into each output interval. Only complete output intervals are +// written so that recent mode picks up partially recorded intervals on the next run. +// Supports recent, from and all process modes, reprocessing all replaces the output feed. +// +// Subclasses set $inputs and $output to the feed setting keys and implement +// row($a, $b, ...), called with one value per input in the order of $inputs (including NAN) +// and returning the output value. Settings are available as $this->params and the output +// interval as $this->interval. reset($previous) is called at the start of each run with the +// output value before the start time (NAN if none), e.g. to continue an accumulation. +abstract class CombineProcess extends PostProcess_common +{ + protected $inputs = array(); + protected $output = "output"; + protected $params; + protected $interval; + + protected function reset($previous) {} + + // Feed ids of the inputs keyed by name, from the $inputs settings by default + protected function input_feeds($params) + { + $feeds = array(); + foreach ($this->inputs as $key) $feeds[$key] = $params->$key; + return $feeds; + } + + // Output values for a block of input values, one array per input + protected function block($blocks) + { + $out = array(); + $columns = array_values($blocks); + switch (count($columns)) { + case 1: + foreach ($columns[0] as $a) $out[] = $this->row($a); + break; + case 2: + foreach ($columns[0] as $i => $a) $out[] = $this->row($a, $columns[1][$i]); + break; + case 3: + foreach ($columns[0] as $i => $a) $out[] = $this->row($a, $columns[1][$i], $columns[2][$i]); + break; + default: + $n = count($columns[0]); + for ($i=0; $i<$n; $i++) $out[] = $this->row(...array_column($columns, $i)); + } + return $out; + } + + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $this->params = $params; + $inputs = array(); + foreach ($this->input_feeds($params) as $key => $feedid) { + $inputs[$key] = new FeedReader($this->dir, $feedid); + } + if (!count($inputs)) { + return array("success"=>false, "message"=>"no input feeds"); + } + $output = new FeedWriter($this->dir, $params->{$this->output}); + + $interval = 0; + $start_time = 0; + $end_time = PHP_INT_MAX; + foreach ($inputs as $input) { + $interval = max($interval, $input->meta->interval); + $start_time = max($start_time, $input->meta->start_time); + $end_time = min($end_time, $input->meta->end_time); + } + $start_time = (int) floor($start_time / $interval) * $interval; + $end_time = $start_time + (int) floor(($end_time - $start_time) / $interval) * $interval; + $this->interval = $interval; + + $output->init($start_time, $interval, $params->process_mode == 'all'); + $time = resolve_start_time($params, $output->meta); + if ($time >= $end_time) { + return array("success"=>true, "message"=>"output feed already up to date"); + } + + $this->reset($output->value_at($time - $interval)); + $output->seek($time); + while (($time = $output->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $blocks = array(); + foreach ($inputs as $key => $input) { + $blocks[$key] = $input->read_time($time, $interval, $n); + } + $output->write_block($this->block($blocks)); + } + $bytes = $output->close($params->process_mode != 'recent'); + + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); + } + return array("success"=>true, "message"=>written_message($bytes, $output)); + } +} + +// Helper for simulation processes that step through time one datapoint at a time with +// several input and output feeds. Inputs are read and outputs written in blocks, each feed +// has a position like a file pointer: read() and write() move it on, seek_to_time() sets it. class ModelHelper { + // Datapoints per input block and per output write. Smaller than POSTPROCESS_BLOCKSIZE as + // values are held as PHP arrays for many feeds at once, 8192 is also slightly faster. + const BLOCKSIZE = 8192; + private $dir; private $params; - private $fh = array(); - private $buffer = array(); - public $meta = array(); - public $value = array(); + private $readers = array(); + private $writers = array(); + // Next position to read or write for each feed + private $pos = array(); + // Block of input values starting at $cache_start + private $cache = array(); + private $cache_start = array(); + private $cache_length = array(); + // Output values not yet passed to the writer, and whether the writer is at $pos + private $pending = array(); + private $pending_length = array(); + private $positioned = array(); + public $meta = array(); public $start_time = false; public $end_time = false; @@ -242,114 +681,201 @@ public function __construct($dir, $params) $this->params = $params; } - private function load($key, $mode) + // Open feed setting $key as an input. The model start and end times are the latest + // start and earliest end of the inputs. Returns false if the setting or feed is missing. + public function input($key) { - $this->meta[$key] = false; - $this->fh[$key] = false; - $this->value[$key] = false; - - // check for valid key if (!isset($this->params->$key)) return false; - $feedid = $this->params->$key; - // load meta - if (!$meta = getmeta($this->dir, $feedid)) return false; - $this->meta[$key] = $meta; - - if (!$this->fh[$key] = @fopen($this->dir . $feedid . ".dat", $mode)) { - echo "ERROR: could not open " . $this->dir . $feedid . ".dat\n"; + try { + $reader = new FeedReader($this->dir, $this->params->$key); + } catch (PostProcessException $e) { return false; } - - // Fetch last value - $this->value[$key] = 0.0; - if ($meta->npoints > 0) { - fseek($this->fh[$key], ($meta->npoints - 1) * 4); - $tmp = unpack("f", fread($this->fh[$key], 4)); - if (!is_nan($tmp[1])) $this->value[$key] = 1 * $tmp[1]; - } + $this->readers[$key] = $reader; + $this->meta[$key] = $reader->meta; + $this->pos[$key] = 0; + $this->cache[$key] = array(); + $this->cache_start[$key] = 0; + $this->cache_length[$key] = 0; + + if ($this->start_time === false || $reader->meta->start_time > $this->start_time) $this->start_time = $reader->meta->start_time; + if ($this->end_time === false || $reader->meta->end_time < $this->end_time) $this->end_time = $reader->meta->end_time; return true; } - public function input($key) + // Open feed setting $key as an output. Returns false if the setting or feed is missing, + // writes to outputs that are not open are ignored. + public function output($key) { - if (!$result = $this->load($key, 'rb')) return false; - if ($this->start_time === false) $this->start_time = $this->meta[$key]->start_time; - else if ($this->meta[$key]->start_time > $this->start_time) $this->start_time = $this->meta[$key]->start_time; - if ($this->end_time === false) $this->end_time = $this->meta[$key]->end_time; - else if ($this->meta[$key]->end_time < $this->end_time) $this->end_time = $this->meta[$key]->end_time; + if (!isset($this->params->$key)) return false; + try { + $writer = new FeedWriter($this->dir, $this->params->$key); + } catch (PostProcessException $e) { + return false; + } + $this->writers[$key] = $writer; + $this->meta[$key] = $writer->meta; + $this->pos[$key] = 0; + $this->pending[$key] = array(); + $this->pending_length[$key] = 0; + $this->positioned[$key] = false; + $this->cache_start[$key] = 0; + $this->cache_length[$key] = 0; return true; } - public function output($key) + // Set the start time and interval of all outputs, outputs with data must already match + public function set_output_meta($start_time, $interval) { - if (!$this->load($key, 'c+')) return false; - $this->buffer[$key] = ""; - return true; + foreach ($this->writers as $key => $writer) { + $writer->init($start_time, $interval); + $this->meta[$key] = $writer->meta; + } } + // Read the value at the position of $key and move on, $value is returned if missing public function read($key, $value) { - if (!$this->fh[$key]) return $value; - $bytes = fread($this->fh[$key], 4); - if (strlen($bytes) != 4) return $value; - $tmp = unpack("f", $bytes); - if ($tmp[1] !== null && !is_nan($tmp[1])) $value = $tmp[1]; + // Fast path: input value in the current block + if (!isset($this->pos[$key])) return $value; + $i = $this->pos[$key] - $this->cache_start[$key]; + if ($i >= 0 && $i < $this->cache_length[$key]) { + $this->pos[$key]++; + $v = $this->cache[$key][$i]; + return is_nan($v) ? $value : $v; + } + + if (isset($this->writers[$key])) { + $v = $this->read_at($key, $this->time_of($key, $this->pos[$key]), NAN); + $this->pos[$key]++; + $this->positioned[$key] = false; + return is_nan($v) ? $value : $v; + } + if (!isset($this->readers[$key])) return $value; + + // Read the next block + $pos = $this->pos[$key]++; + $npoints = $this->meta[$key]->npoints; + if ($pos < 0 || $pos >= $npoints) return $value; + $this->cache[$key] = $this->readers[$key]->read($pos, min(self::BLOCKSIZE, $npoints - $pos)); + $this->cache_start[$key] = $pos; + $this->cache_length[$key] = count($this->cache[$key]); + $v = $this->cache[$key][0]; + return is_nan($v) ? $value : $v; + } + + // Value of $key at $time without moving its position, $value is returned if missing + public function read_at($key, $time, $value) + { + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $v = $this->writers[$key]->value_at($time); + } else if (isset($this->readers[$key])) { + $meta = $this->meta[$key]; + $pos = floor(($time - $meta->start_time) / $meta->interval); + $v = $this->readers[$key]->read((int) $pos, 1)[0]; + } else { + return $value; + } + return is_nan($v) ? $value : $v; + } + + // Values of $key for $n datapoints from $time without moving its position + public function read_range($key, $time, $n) + { + if (!isset($this->meta[$key])) return array_fill(0, $n, NAN); + $meta = $this->meta[$key]; + $pos = (int) floor(($time - $meta->start_time) / $meta->interval); + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $this->writers[$key]->flush(); + $reader = new FeedReader($this->dir, $this->params->$key); + return $reader->read($pos, $n); + } + return $this->readers[$key]->read($pos, $n); + } + + // Last value of input $key before $time that is not missing, looking back up to one + // block, used to continue holding the last value at the start of a run + public function last_valid($key, $time, $value) + { + if (!isset($this->readers[$key])) return $value; + $meta = $this->meta[$key]; + $end = (int) ceil(($time - $meta->start_time) / $meta->interval); + $start = max(0, $end - POSTPROCESS_BLOCKSIZE); + if ($end <= $start) return $value; + $values = $this->readers[$key]->read($start, $end - $start); + for ($i=count($values)-1; $i>=0; $i--) { + if (!is_nan($values[$i])) return $values[$i]; + } return $value; } public function seek_to_time($time) { - foreach (array_keys($this->fh) as $key) { - if (!$this->meta[$key]) continue; - $pos = floor(($time - $this->meta[$key]->start_time) / $this->meta[$key]->interval); - fseek($this->fh[$key], $pos * 4); - } + foreach (array_keys($this->pos) as $key) $this->seek($key, $time); } public function seek_to_time_inputs($time) { - foreach (array_keys($this->fh) as $key) { - if (!isset($this->buffer[$key])) { - $pos = floor(($time - $this->meta[$key]->start_time) / $this->meta[$key]->interval); - fseek($this->fh[$key], $pos * 4); - } + foreach (array_keys($this->readers) as $key) $this->seek($key, $time); + } + + public function seek_to_time_outputs($time) + { + foreach (array_keys($this->writers) as $key) $this->seek($key, $time); + } + + private function seek($key, $time) + { + $meta = $this->meta[$key]; + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $this->positioned[$key] = false; } + $this->pos[$key] = (int) floor(($time - $meta->start_time) / $meta->interval); } + // Write $value at the position of output $key and move on public function write($key, $value) { - // Skip outputs that are not loaded - if (empty($this->fh[$key])) return; - $this->buffer[$key] .= pack("f", $value); - $this->value[$key] = $value; + // Fast path: output positioned with room in the block + if (isset($this->positioned[$key]) && $this->positioned[$key]) { + $this->pending[$key][] = $value; + $this->pos[$key]++; + if (++$this->pending_length[$key] >= self::BLOCKSIZE) $this->flush_pending($key); + return; + } + if (!isset($this->writers[$key])) return; + $this->writers[$key]->seek($this->time_of($key, $this->pos[$key])); + $this->positioned[$key] = true; + $this->write($key, $value); } - public function set_output_meta($start_time, $interval) + private function flush_pending($key) { - foreach (array_keys($this->buffer) as $key) { - $this->meta[$key]->start_time = $start_time; - $this->meta[$key]->interval = $interval; - if ($this->meta[$key]->end_time == 0) $this->meta[$key]->end_time = $start_time; + if ($this->pending_length[$key]) { + $this->writers[$key]->write_block($this->pending[$key]); + $this->pending[$key] = array(); + $this->pending_length[$key] = 0; } } + private function time_of($key, $pos) + { + return $this->meta[$key]->start_time + $pos * $this->meta[$key]->interval; + } + + // Write remaining data, close outputs and update their last time and value. + // Returns the number of bytes written. public function save_all() { $total_size = 0; - foreach (array_keys($this->buffer) as $key) { - $size = strlen($this->buffer[$key]); - if ($size > 0) { - $feedid = $this->params->$key; - // Write meta data - createmeta($this->dir, $feedid, $this->meta[$key]); - // Write data - fwrite($this->fh[$key], $this->buffer[$key]); - fclose($this->fh[$key]); - $total_size += $size; - // Update feed last time and value - updatetimevalue($feedid, time(), $this->value[$key]); - } + foreach ($this->writers as $key => $writer) { + $this->flush_pending($key); + $total_size += $writer->close(); } + $this->writers = array(); return $total_size; } } diff --git a/postprocess_run.php b/postprocess_run.php index adf8182..f1f2cd2 100644 --- a/postprocess_run.php +++ b/postprocess_run.php @@ -24,7 +24,11 @@ if ($process = $postprocess->pop_process_queue()) { print json_encode($process)."\n"; - $result = $process_classes[$process->params->process]->process($process->params); + try { + $result = $process_classes[$process->params->process]->process($process->params); + } catch (Throwable $e) { + $result = array("success"=>false, "message"=>$e->getMessage()); + } if ($result['success']) { print "Success: ".$result['message']."\n"; diff --git a/processes/accumulator.php b/processes/accumulator.php index 577212c..4808787 100644 --- a/processes/accumulator.php +++ b/processes/accumulator.php @@ -1,7 +1,11 @@ "Accumulator", @@ -16,95 +20,23 @@ public function description() { ); } - public function process($params) + protected function reset($previous) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $total = 0; - $value = 0; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } + $this->total = is_nan($previous) ? 0 : $previous; + $this->value = 0; + } - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,($start_pos-1)*4); - $tmp = unpack("f",fread($of,4)); - $total = $tmp[1]*1; - } - - $filtered_count = 0; - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - if (!is_nan($tmp[1])) $value = 1*$tmp[1]; - - // filter spurious values +-1M - if ($value>=$params->min_value && $value<$params->max_value) { - $total += $value; - $buffer .= pack("f",$total); - } else { - $filtered_count++; + // Missing values hold the last value. Values outside of the limits are treated + // as spurious and are not added. + protected function row($value) + { + if (!is_nan($value)) { + if ($value < $this->params->min_value || $value >= $this->params->max_value) { + return $this->total; } + $this->value = $value; } - - fwrite($of,$buffer); - - if ($filtered_count>0) { - print "Filtered count: $filtered_count\n"; - } - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$total); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$total); - + $this->total += $this->value; + return $this->total; } -} \ No newline at end of file +} diff --git a/processes/addfeeds.php b/processes/addfeeds.php index 5ee63d8..ff8c453 100644 --- a/processes/addfeeds.php +++ b/processes/addfeeds.php @@ -1,7 +1,9 @@ "Add feeds", @@ -15,106 +17,11 @@ public function description() { ); } - public function process($params) + // Sum of both feeds, or whichever feed has data + protected function row($a, $b) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $feedA_meta = getmeta($this->dir,$params->feedA); - $feedB_meta = getmeta($this->dir,$params->feedB); - - // Find longest interval - $interval = $feedA_meta->interval; - if ($feedB_meta->interval>$interval) $interval = $feedB_meta->interval; - // Find latest start time - $start_time = $feedA_meta->start_time; - if ($feedB_meta->start_time>$start_time) $start_time = $feedB_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $feedA_end_time = $feedA_meta->start_time + ($feedA_meta->interval * $feedA_meta->npoints); - $feedB_end_time = $feedB_meta->start_time + ($feedB_meta->interval * $feedB_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $feedA_end_time; - if ($feedB_end_time<$end_time) $end_time = $feedB_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$feedA_fh = @fopen($this->dir.$params->feedA.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedA feed"); - } - if (!$feedB_fh = @fopen($this->dir.$params->feedB.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedB feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_feedA = floor(($time - $feedA_meta->start_time) / $feedA_meta->interval); - $pos_feedB = floor(($time - $feedB_meta->start_time) / $feedB_meta->interval); - - $valueA = NAN; - $valueB = NAN; - - if ($pos_feedA>=0 && $pos_feedA<$feedA_meta->npoints) { - fseek($feedA_fh,$pos_feedA*4); - $tmp = unpack("f",fread($feedA_fh,4)); - $valueA = $tmp[1]; - } - - if ($pos_feedB>=0 && $pos_feedB<$feedB_meta->npoints) { - fseek($feedB_fh,$pos_feedB*4); - $tmp = unpack("f",fread($feedB_fh,4)); - $valueB = $tmp[1]; - } - - $outval = NAN; - if (!is_nan($valueA)) $outval = $valueA; - if (!is_nan($valueB)) $outval = $valueB; - if (!is_nan($valueA) && !is_nan($valueB)) $outval = $valueB + $valueA; - $buffer .= pack("f",$outval); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($feedA_fh); - fclose($feedB_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$outval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$outval); + if (is_nan($a)) return $b; + if (is_nan($b)) return $a; + return $a + $b; } -} \ No newline at end of file +} diff --git a/processes/allownegative.php b/processes/allownegative.php index 603060e..5e8519a 100644 --- a/processes/allownegative.php +++ b/processes/allownegative.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - if ($value>0) $value = 0; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value > 0 ? 0 : $value; } -} \ No newline at end of file +} diff --git a/processes/allowpositive.php b/processes/allowpositive.php index 03200a5..102e50b 100644 --- a/processes/allowpositive.php +++ b/processes/allowpositive.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - if ($value<0) $value = 0; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value < 0 ? 0 : $value; } -} \ No newline at end of file +} diff --git a/processes/backfillfeed.php b/processes/backfillfeed.php index aea63d5..887dddd 100644 --- a/processes/backfillfeed.php +++ b/processes/backfillfeed.php @@ -23,82 +23,34 @@ public function process($params) return array("success"=>false, "message"=>"feed and source must be different feeds"); } - $target_meta = getmeta($this->dir,$params->feed); - $source_meta = getmeta($this->dir,$params->source); - $interval = $target_meta->interval; + $target = new FeedWriter($this->dir,$params->feed); + $source = new FeedReader($this->dir,$params->source); + $interval = $target->meta->interval; + + if ($target->meta->npoints==0) { + return array("success"=>false, "message"=>"feed to backfill is empty"); + } // Always start from the current end of the target feed, regardless of process mode, // so that existing data in the target feed is never modified - $start_time = $target_meta->end_time; + $start_time = $target->meta->end_time; // Only write target intervals that are fully covered by the source feed, // so that a partially recorded interval is completed on the next run - $end_time = $start_time + floor(($source_meta->end_time - $start_time) / $interval) * $interval; + $end_time = $start_time + floor(($source->meta->end_time - $start_time) / $interval) * $interval; if ($start_time>=$end_time) { return array("success"=>true, "message"=>"no new data to process"); } - if (!$source_fh = @fopen($this->dir.$params->source.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open source feed"); - } - if (!$target_fh = @fopen($this->dir.$params->feed.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open target feed"); + // Source values are averaged into each target interval + $target->seek($start_time); + while (($time = $target->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $target->write_block($source->read_time($time,$interval,$n)); } - fseek($target_fh, $target_meta->npoints * 4); - - $buffer = ""; - $byteswritten = 0; - $last_time = false; - $last_value = NAN; - - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - // Source datapoints that fall within this target interval - $pos_start = ceil(($time - $source_meta->start_time) / $source_meta->interval); - $pos_end = ceil(($time + $interval - $source_meta->start_time) / $source_meta->interval); - // Source interval longer than target interval: use the datapoint covering this interval - if ($pos_end<=$pos_start) { - $pos_start = floor(($time - $source_meta->start_time) / $source_meta->interval); - $pos_end = $pos_start + 1; - } - if ($pos_start<0) $pos_start = 0; - if ($pos_end>$source_meta->npoints) $pos_end = $source_meta->npoints; - - $sum = 0; - $n = 0; - if ($pos_end>$pos_start) { - fseek($source_fh,$pos_start*4); - $values = unpack("f*",fread($source_fh,($pos_end-$pos_start)*4)); - foreach ($values as $value) { - if (!is_nan($value)) { - $sum += $value; - $n++; - } - } - } + $byteswritten = $target->close(); - $outval = NAN; - if ($n>0) { - $outval = $sum / $n; - $last_time = $time; - $last_value = $outval; - } - $buffer .= pack("f",$outval); - - if (strlen($buffer)>=400000) { - $byteswritten += fwrite($target_fh,$buffer); - $buffer = ""; - } - } - - $byteswritten += fwrite($target_fh,$buffer); - fclose($target_fh); - fclose($source_fh); - - if ($last_time!==false) { - updatetimevalue($params->feed,$last_time,$last_value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$last_time." ".$last_value); + return array("success"=>true, "message"=>written_message($byteswritten, $target)); } } diff --git a/processes/basic_formula.php b/processes/basic_formula.php index 9a8ef38..80f2f98 100644 --- a/processes/basic_formula.php +++ b/processes/basic_formula.php @@ -1,5 +1,33 @@ type of data ie "feed" ou "value" +1 -> operator +2 -> value or feed number +a missing value is 1, e.g. a lone operator +*/ +function ftoa($b): array +{ + $c = []; + if (sizeof(value: $b) == 4) { + $c[0] = "feed"; + $c[2] = intval(value: substr(string: $b[3], offset: 1)); + } else { + $c[0] = "value"; + $c[2] = $b[2]; + if ($c[2] === "") $c[2] = "1"; + } + $c[1] = $b[1]; + if (!$c[1]) $c[1] = '+'; + return $c; +} + function formula_to_array($formula_string, $reg_exp): array { //process a formula string to an array @@ -7,16 +35,18 @@ function formula_to_array($formula_string, $reg_exp): array foreach (preg_split(pattern: "@(?=(\*|\/))@",subject: $formula_string) as $piece) { if (preg_match(pattern: $reg_exp, subject: $piece, matches: $b)){ if (count(value: $b)>2){ - $c=ftoa(b: $b); - if($c[2]) $fly[]=$c; + $fly[]=ftoa(b: $b); } } } return $fly; } -class PostProcess_basic_formula extends PostProcess_common +class PostProcess_basic_formula extends CombineProcess { + private $elements = []; + private $feed_ids = []; + public function description(): array { $bfdescription="Enter your formula as a symbolic expression - allows brackets and the max function
@@ -39,13 +69,23 @@ public function description(): array { ]; } - public function process($processitem): array + public function process($params): array { - $DEBUG = 0; - $result = $this->validate(processitem: $processitem); + $result = $this->validate(processitem: $params); if (!$result["success"]) return $result; - $dir = $this->dir; + if (!$parsed = $this->parse_formula(formula: $params->formula)) { + return ["success"=>false, "message"=>"could not understand your formula SORRY...."]; + } + $this->elements=$parsed["elements"]; + $this->feed_ids=$parsed["feed_ids"]; + return parent::process($params); + } + + // parse a formula into elements and the feed ids it uses, false if not understood + public function parse_formula($formula) + { + $DEBUG = 0; // regular expression to recognize a float or int value // use of ?: not to perturb things in creating useless references $Xnbr="(?:[0-9]+\.[0-9]+|[0-9]+)"; @@ -73,8 +113,6 @@ public function process($processitem): array $nbf=count(value: $functions); - //retrieving the formula - $formula=$processitem->formula; //removing useless brackets $pos=0; $original=$formula; @@ -121,12 +159,8 @@ public function process($processitem): array $original=$formula; //we catch the distinct feed numbers involved in the formula - $feed_ids=[]; - while(preg_match(pattern: "/$Xf/", subject: $formula, matches: $b)){ - //removing the f... - $feed_ids[]=substr(string: $b[0], offset: 1); - $formula=str_replace(search: $b[0], replace: "", subject: $formula); - } + preg_match_all(pattern: "/f(\d+)/", subject: $formula, matches: $b); + $feed_ids=array_values(array_unique(array: $b[1])); $formula=$original; $array= []; @@ -213,20 +247,7 @@ public function process($processitem): array print $original_copy; print "\n\n"; } - if ($original_copy=="") print "formula is OK!!\n"; else { - return ["success"=>false, "message"=>"could not understand your formula SORRY...."]; - } - - //checking the output feed - $fopen_mode='ab'; - if ($processitem->process_mode=='all') { - $fopen_mode='wb'; - } - $out=$processitem->output; - if(!$out_meta = getmeta(dir: $dir,id: $out)) return array("success"=>false, "message"=>"could not get meta for $out"); - if (!$out_fh = @fopen(filename: $dir.$out.".dat", mode: $fopen_mode)) { - return ["success"=>false, "message"=>"could not open $dir $out.dat"]; - } + if ($original_copy!="") return false; $elements=[]; foreach ($array as $a){ @@ -270,100 +291,67 @@ public function process($processitem): array } if ($DEBUG) print_r(value: $elements); - //we retrieve the meta and open the dat files - foreach ($feed_ids as $id){ - if(!$meta = getmeta(dir: $dir, id: $id)) return array("success"=>false, "message"=>"could not get meta for $id"); - $feeds_meta[$id]=$meta; - if (!$fh = @fopen(filename: "$dir$id.dat", mode: 'rb')) { - return ["success"=>false, "message"=>"could not open $dir $id.dat"]; - } - $feeds_dat[$id]=$fh; - } - - $compute_meta= call_user_func_array(callback: "compute_meta", args: $feeds_meta); + return ["elements"=>$elements, "feed_ids"=>$feed_ids]; + } - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); + protected function input_feeds($params) + { + $feeds=[]; + foreach ($this->feed_ids as $id) $feeds[$id]=$id; + return $feeds; + } - if ($processitem->process_mode=='recent') { - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - } else { - $writing_start_time=$out_meta->start_time; + // formula values for a block of datapoints, the formula is evaluated one operation at a time + // over the whole block. missing values and division by zero give NAN + protected function block($blocks) + { + $n=count(value: reset($blocks)); + $total=array_fill(0,$n,0); + foreach ($this->elements as $element) { + $s1=$this->sum_block(terms: [$element->scale], blocks: $blocks, n: $n); + $s2=$this->sum_block(terms: $element->formula, blocks: $blocks, n: $n); + $s3=$this->sum_block(terms: [$element->scale_right], blocks: $blocks, n: $n); + if ($element->function=="max") { + $arg2_constant=is_numeric(value: $element->arg2); + $arg2=$arg2_constant ? null : $this->sum_block(terms: $element->arg2, blocks: $blocks, n: $n); + for ($i=0;$i<$n;$i++) { + $a2=$arg2_constant ? $element->arg2 : $arg2[$i]; + if (is_nan($s1[$i]) || is_nan($s2[$i]) || is_nan($s3[$i]) || is_nan($a2)) $total[$i]=NAN; + else $total[$i]+=$s1[$i]*max($s2[$i],$a2)*$s3[$i]; + } + } else { + for ($i=0;$i<$n;$i++) $total[$i]+=$s1[$i]*$s2[$i]*$s3[$i]; + } } - - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - - $buffer=""; + return $total; + } - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - $s=[]; - foreach($elements as $element){ - $s1=bfo( - elements: [$element->scale], - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s2=bfo( - elements: $element->formula, - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s3=bfo( - elements: [$element->scale_right], - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - //print("$s1-----$s2-----$s3"); - if (!is_nan(num: $s1) && !is_nan(num: $s2) && !is_nan(num: $s3)) { - if($element->function=="max") { - $arg2 = (is_numeric(value: $element->arg2)) ? $element->arg2 : bfo( - elements: $element->arg2, - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s[] = (!is_nan(num: $arg2)) ? $s1*max([$s2,$arg2])*$s3 : NAN; - } - if($element->function=="brackets" || $element->function=="none") { - $s[]=$s1*$s2*$s3; + // for each datapoint of the block, the sum of the products of each term + // a term is a list of factors [type "feed" or "value", operator, feed id or value] + private function sum_block($terms,$blocks,$n) + { + $sum=array_fill(0,$n,0); + foreach ($terms as $term) { + $product=array_fill(0,$n,1); + foreach ($term as $e) { + if ($e[0]=="value") { + $value=(float) $e[2]; + if ($e[1]=="/") $value=($value==0) ? NAN : 1/$value; + if ($e[1]=="-") $value=-$value; + for ($i=0;$i<$n;$i++) $product[$i]*=$value; + } else { + $values=$blocks[$e[2]]; + if ($e[1]=="/") { + foreach ($values as $i=>$value) $product[$i]*=($value==0) ? NAN : 1/$value; + } else if ($e[1]=="-") { + foreach ($values as $i=>$value) $product[$i]*=-$value; + } else { + foreach ($values as $i=>$value) $product[$i]*=$value; } - } else $s[] = NAN; + } } - $sum = (!in_array(needle: NAN, haystack: $s)) ? array_sum(array: $s) : NAN; - $buffer.=pack("f", $sum); + for ($i=0;$i<$n;$i++) $sum[$i]+=$product[$i]; } - - if(!$buffer) { - return ["success"=>false, "message"=>"nothing to write - all is up to date"]; - } - - if(!$written_bytes=fwrite(stream: $out_fh, data: $buffer)){ - foreach ($feeds_dat as $f) fclose(stream: $f); - fclose(stream: $out_fh); - return ["success"=>false, "message"=>"unable to write to the file with id=$out"]; - } - $nbdataswritten=$written_bytes/4; - print("NOTICE: basic_formula() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta(dir: $dir, id: $out, meta: $out_meta); - foreach ($feeds_dat as $f) fclose(stream: $f); - fclose(stream: $out_fh); - print("last time value: $time / $sum \n"); - updatetimevalue(id: $out, time: $time, value: $sum); - return [ - "success"=>true, - "message"=>"bytes written: $written_bytes, last time value: $time, last written value $sum" - ]; + return $sum; } } diff --git a/processes/batterysimulator.php b/processes/batterysimulator.php index 26edcea..753de6b 100644 --- a/processes/batterysimulator.php +++ b/processes/batterysimulator.php @@ -145,54 +145,42 @@ public function process($p) // Note: implementation only allows for same meta for all output feeds $model->set_output_meta($start_time,$interval); - // Process new data since last run - if (!$recalc) $start_time = $model->meta['grid']->end_time-$interval; + // Continue from the end of the output feeds + if (!$recalc) $start_time = $model->meta['grid']->end_time; if ($start_time<$model->start_time) $start_time = $model->start_time; - if ($start_time==$end_time) { + if ($start_time>=$end_time) { return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - - $solar = 0; - $use = 0; + + // Starting values from the datapoint before the start + $prev = $start_time - $interval; + $charge_kwh = $model->read_at('charge_kwh',$prev,0); + $discharge_kwh = $model->read_at('discharge_kwh',$prev,0); + $import_kwh = $model->read_at('import_kwh',$prev,0); + $export_kwh = $model->read_at('export_kwh',$prev,0); + $solar_to_load_kwh = $model->read_at('solar_to_load_kwh',$prev,0); + $solar_to_grid_kwh = $model->read_at('solar_to_grid_kwh',$prev,0); + $solar_to_battery_kwh = $model->read_at('solar_to_battery_kwh',$prev,0); + $battery_to_load_kwh = $model->read_at('battery_to_load_kwh',$prev,0); + $battery_to_grid_kwh = $model->read_at('battery_to_grid_kwh',$prev,0); + $grid_to_load_kwh = $model->read_at('grid_to_load_kwh',$prev,0); + $grid_to_battery_kwh = $model->read_at('grid_to_battery_kwh',$prev,0); + $soc = $p->capacity * 0.5; - $charge_kwh = 0; - $discharge_kwh = 0; - $import_kwh = 0; - $export_kwh = 0; - $solar_to_load_kwh = 0; - $solar_to_grid_kwh = 0; - $solar_to_battery_kwh = 0; - $battery_to_load_kwh = 0; - $battery_to_grid_kwh = 0; - $grid_to_load_kwh = 0; - $grid_to_battery_kwh = 0; - - // Get starting values - $model->seek_to_time($start_time); - $charge_kwh = $model->read('charge_kwh',$charge_kwh); - $discharge_kwh = $model->read('discharge_kwh',$discharge_kwh); - $import_kwh = $model->read('import_kwh',$import_kwh); - $export_kwh = $model->read('export_kwh',$export_kwh); - $solar_to_load_kwh = $model->read('solar_to_load_kwh',$solar_to_load_kwh); - $solar_to_grid_kwh = $model->read('solar_to_grid_kwh',$solar_to_grid_kwh); - $solar_to_battery_kwh = $model->read('solar_to_battery_kwh',$solar_to_battery_kwh); - $battery_to_load_kwh = $model->read('battery_to_load_kwh',$battery_to_load_kwh); - $battery_to_grid_kwh = $model->read('battery_to_grid_kwh',$battery_to_grid_kwh); - $grid_to_load_kwh = $model->read('grid_to_load_kwh',$grid_to_load_kwh); - $grid_to_battery_kwh = $model->read('grid_to_battery_kwh',$grid_to_battery_kwh); - - if ($model->meta['soc']->npoints) { - $soc = $model->read('soc',$soc); - $soc = $soc*0.01*$p->capacity; - } + $soc_prc = $model->read_at('soc',$prev,NAN); + if (!is_nan($soc_prc)) $soc = $soc_prc*0.01*$p->capacity; + + // Missing values hold the last valid value, including from before this run + $solar = $model->last_valid('solar',$start_time,0); + $use = $model->last_valid('consumption',$start_time,0); - // Reset again $model->seek_to_time($start_time); - - $date->setTimestamp($start_time); + + $date->setTimestamp($prev); $hour = $date->format("H")*1; - $charging_offpeak = false; + $next_hour_time = $start_time; + $charging_offpeak = $this->charging_offpeak_at($model,$p,$date,$prev,$interval); $power_to_kwh = $interval / 3600000.0; // conversion factor from W to kWh for given interval @@ -200,9 +188,14 @@ public function process($p) for ($time=$start_time; $time<$end_time; $time+=$interval) { // $model->seek_to_time_inputs($time); not required as input feed intervals are the same + // Local hour, only worked out when the hour changes $last_hour = $hour; - $date->setTimestamp($time); - $hour = $date->format("H")*1; + if ($time>=$next_hour_time) { + $date->setTimestamp($time); + list($hour,$minute,$second) = explode(":",$date->format("H:i:s")); + $hour = $hour*1; + $next_hour_time = $time - ($minute*60 + $second) + 3600; + } $solar = $model->read('solar',$solar); $use = $model->read('consumption',$use); @@ -385,4 +378,30 @@ public function process($p) $buffersize = $model->save_all(); return array("success"=>true, "message"=>"bytes written: ".($buffersize/1024)." kb"); } + + // Whether off-peak charging is in progress after the datapoint at $prev: the off-peak + // window has started since the start of the data and the state of charge has not + // reached the target at any datapoint since + private function charging_offpeak_at($model, $p, $date, $prev, $interval) + { + if (!$p->has_battery || $p->offpeak_soc_target<=0 || $prev<$model->start_time) return false; + + $date->setTimestamp($prev); + $date->setTime((int) $p->offpeak_start, 0, 0); + if ($date->getTimestamp()>$prev) { + $date->setTimestamp($prev - 86400); + $date->setTime((int) $p->offpeak_start, 0, 0); + } + $window_start = $date->getTimestamp(); + + // First datapoint in the window + $first = $model->start_time + ceil(($window_start - $model->start_time) / $interval) * $interval; + if ($first<$model->start_time || $first>$prev) return false; + + $n = (int) (($prev - $first) / $interval) + 1; + foreach ($model->read_range('soc',$first,$n) as $soc_prc) { + if (!is_nan($soc_prc) && $soc_prc>=$p->offpeak_soc_target) return false; + } + return true; + } } diff --git a/processes/carnot_cop_simulator.php b/processes/carnot_cop_simulator.php index 60a8740..77b9da2 100644 --- a/processes/carnot_cop_simulator.php +++ b/processes/carnot_cop_simulator.php @@ -61,25 +61,21 @@ public function process($p) // Note: implementation only allows for same meta for all output feeds $model->set_output_meta($start_time,$interval); - // Simulation start time - if (!$recalc) $start_time = $model->meta['heatpump_heat_sim']->end_time-$interval; - - if ($start_time==$end_time) { + // Continue from the end of the output feeds + if (!$recalc) $start_time = $model->meta['heatpump_heat_sim']->end_time; + if ($start_time<$model->start_time) $start_time = $model->start_time; + + if ($start_time>=$end_time) { return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - - $power = 0; - $flowT = 0; - $returnT = 0; - $outsideT = 0; - $heat_kwh = 0; - // get cumulative kwh value - $model->seek_to_time($start_time); - if ($model->meta['heatpump_heat_sim']->npoints) { - $heat_kwh = $model->read('heatpump_heat_sim_kwh',$heat_kwh); - } - // reset again + // Cumulative kWh before the start, missing inputs hold the last valid value + $heat_kwh = $model->read_at('heatpump_heat_sim_kwh',$start_time-$interval,0); + $power = $model->last_valid('heatpump_elec',$start_time,0); + $flowT = $model->last_valid('heatpump_flowT',$start_time,0); + $returnT = $model->last_valid('heatpump_returnT',$start_time,0); + $outsideT = $model->last_valid('heatpump_outsideT',$start_time,0); + $model->seek_to_time($start_time); $i=0; diff --git a/processes/constantflow_tokwh.php b/processes/constantflow_tokwh.php index 6509589..bc8e065 100644 --- a/processes/constantflow_tokwh.php +++ b/processes/constantflow_tokwh.php @@ -1,7 +1,10 @@ "Constant flow to kWh", @@ -17,111 +20,17 @@ public function description() { ); } - public function process($processitem) + protected function reset($previous) { - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - - $vhc = (float) $processitem->vhc; - $flow = (float) $processitem->flow; - $tint = $processitem->tint; - $text = $processitem->text; - $out = $processitem->output; - - if (!$tint_meta = getmeta($dir,$tint)) array("success"=>false, "message"=>"could not open input feed"); - if (!$text_meta = getmeta($dir,$text)) array("success"=>false, "message"=>"could not open input feed"); - if (!$out_meta = getmeta($dir,$out)) array("success"=>false, "message"=>"could not open output feed"); - - if (!$tint_fh = @fopen($dir.$tint.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$text_fh = @fopen($dir.$text.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$out_fh = @fopen($dir.$out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $compute_meta=compute_meta($tint_meta,$text_meta); - - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - $kwh = 0; - - if($out_meta->npoints>0) { - print("NOTICE : file not empty \n"); - $pos_last=($out_meta->npoints-1)*4; - print("NOTICE : reading last previously processed value in file position $pos_last \n"); - fseek($out_fh,$pos_last); - $tmp=unpack("f",fread($out_fh,4)); - $kwh = $tmp[1]; - print("NOTICE : accumulation will resume at last previously processed value $kwh \n"); - } else fseek($out_fh,0); - - $buffer=""; - - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - - $pos_tint = floor(($time - $tint_meta->start_time) / $tint_meta->interval); - $pos_text = floor(($time - $text_meta->start_time) / $text_meta->interval); - $value_tint=NAN; - $value_text=NAN; - if ($pos_tint>=0 && $pos_tint<$tint_meta->npoints) { - fseek($tint_fh,$pos_tint*4); - $tint_tmp = unpack("f",fread($tint_fh,4)); - $value_tint = $tint_tmp[1]; - } - if ($pos_text>=0 && $pos_text<$text_meta->npoints) { - fseek($text_fh,$pos_text*4); - $text_tmp = unpack("f",fread($text_fh,4)); - $value_text = $text_tmp[1]; - } - - //if we face one or more NAN value among tint,text, we should not make any calculation and $kwh should remain the same - if(!is_nan($value_tint) && !is_nan($value_text)){ - $kwh+=0.001*$vhc*$flow*max($value_tint-$value_text,0)*$out_meta->interval/3600; - } - //print("$kwh/"); - $buffer.=pack("f",$kwh); - } - - if(!$buffer) { - return array("success"=>false,"message"=>"nothing to write - all is up to date"); - } + $this->kwh = is_nan($previous) ? 0 : $previous; + } - if(!$written_bytes=fwrite($out_fh,$buffer)){ - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - return array("success"=>false,"message"=>"unable to write to the file with id=$out"); - } - $nbdataswritten=$written_bytes/4; - print("NOTICE: constantflow_tokwh() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta($dir,$out,$out_meta); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - print("last time value: $time / $kwh \n"); - - if ($written_bytes>0) { - updatetimevalue($out,$time,$kwh); + // Missing temperatures add no energy + protected function row($tint, $text) + { + if (!is_nan($tint) && !is_nan($text)) { + $this->kwh += 0.001 * $this->params->vhc * $this->params->flow * max($tint - $text, 0) * $this->interval / 3600; } - return array("success"=>true, "message"=>"bytes written: ".$written_bytes.", last time value: ".$time." ".$kwh); + return $this->kwh; } -} \ No newline at end of file +} diff --git a/processes/exportcalc.php b/processes/exportcalc.php index 1291fb1..6af9b77 100644 --- a/processes/exportcalc.php +++ b/processes/exportcalc.php @@ -1,7 +1,9 @@ "Export calculation", @@ -15,107 +17,11 @@ public function description() { ); } - public function process($params) + // Missing consumption is treated as no consumption + protected function row($gen, $use) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->consumption); - $gen_meta = getmeta($this->dir,$params->generation); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->consumption.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->generation.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $exportval = NAN; - if (!is_nan($genval)) $exportval = $genval; - if (!is_nan($genval) && !is_nan($useval)) $exportval = $genval - $useval; - if ($exportval<0) $exportval = 0; - - $buffer .= pack("f",$exportval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$exportval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$exportval); + if (is_nan($gen)) return NAN; + if (!is_nan($use)) $gen -= $use; + return $gen < 0 ? 0 : $gen; } -} \ No newline at end of file +} diff --git a/processes/importcalc.php b/processes/importcalc.php index f0de1ea..c7e3f08 100644 --- a/processes/importcalc.php +++ b/processes/importcalc.php @@ -1,7 +1,10 @@ "Import calculation", @@ -15,107 +18,11 @@ public function description() { ); } - public function process($params) + // Missing generation is treated as no generation + protected function row($use, $gen) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->use); - $gen_meta = getmeta($this->dir,$params->gen); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->out,$out_meta); - $out_meta = getmeta($this->dir,$params->out); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->use.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->gen.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $importval = NAN; - if (!is_nan($useval)) $importval = $useval; - if (!is_nan($genval) && !is_nan($genval)) $importval = $useval - $genval; - if ($importval<0) $importval = 0; - - $buffer .= pack("f",$importval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->out,$time,$importval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$importval); + if (is_nan($use)) return NAN; + if (!is_nan($gen)) $use -= $gen; + return $use < 0 ? 0 : $use; } -} \ No newline at end of file +} diff --git a/processes/limit_morethan_lessthan.php b/processes/limit_morethan_lessthan.php index 57972d3..676cabf 100644 --- a/processes/limit_morethan_lessthan.php +++ b/processes/limit_morethan_lessthan.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - $morethan = $processitem->morethan; - $lessthan = $processitem->lessthan; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - - $fix_count = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - if ($values[$i]>$morethan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$morethan)); - $fix_count ++; - } - else if ($values[$i]<$lessthan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$lessthan)); - $fix_count ++; - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$fix_count." datapoints, ".round(($fix_count/$npoints)*100)."%\n"); + if ($value > $this->params->morethan) return (float) $this->params->morethan; + if ($value < $this->params->lessthan) return (float) $this->params->lessthan; + return $value; } } diff --git a/processes/liquidorairflow_tokwh.php b/processes/liquidorairflow_tokwh.php index fbaa408..6ec0f12 100644 --- a/processes/liquidorairflow_tokwh.php +++ b/processes/liquidorairflow_tokwh.php @@ -1,7 +1,10 @@ "Liquid or airflow to kWh", @@ -17,122 +20,17 @@ public function description() { ); } - public function process($processitem) + protected function reset($previous) { - $dir = $this->dir; - - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $vhc = (float) $processitem->vhc; - $flow = $processitem->flow; - $tint = $processitem->tint; - $text = $processitem->text; - $out = $processitem->output; - - if (!$flow_meta = getmeta($dir,$flow)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$tint_meta = getmeta($dir,$tint)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$text_meta = getmeta($dir,$text)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$out_meta = getmeta($dir,$out)) return array("success"=>false, "message"=>"could not open output feed"); - - if (!$flow_fh = @fopen($dir.$flow.".dat", 'rb')) { - echo "ERROR: could not open $dir $flow.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$tint_fh = @fopen($dir.$tint.".dat", 'rb')) { - echo "ERROR: could not open $dir $tint.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$text_fh = @fopen($dir.$text.".dat", 'rb')) { - echo "ERROR: could not open $dir $text.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$out_fh = @fopen($dir.$out.".dat", 'c+')) { - echo "ERROR: could not open $dir $out.dat\n"; - return array("success"=>false, "message"=>"could not open output feed"); - } - - $compute_meta=compute_meta($flow_meta,$tint_meta,$text_meta); - - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - $kwh = 0; - - if($out_meta->npoints>0) { - print("NOTICE : file not empty \n"); - $pos_last=($out_meta->npoints-1)*4; - print("NOTICE : reading last previously processed value in file position $pos_last \n"); - fseek($out_fh,$pos_last); - $tmp=unpack("f",fread($out_fh,4)); - $kwh = $tmp[1]; - print("NOTICE : accumulation will resume at last previously processed value $kwh \n"); - } else fseek($out_fh,0); - $buffer=""; - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - $pos_flow = floor(($time - $flow_meta->start_time) / $flow_meta->interval); - $pos_tint = floor(($time - $tint_meta->start_time) / $tint_meta->interval); - $pos_text = floor(($time - $text_meta->start_time) / $text_meta->interval); - $value_flow=NAN; - $value_tint=NAN; - $value_text=NAN; - if ($pos_flow>=0 && $pos_flow<$flow_meta->npoints) { - fseek($flow_fh,$pos_flow*4); - $flow_tmp = unpack("f",fread($flow_fh,4)); - $value_flow = $flow_tmp[1]; - } - if ($pos_tint>=0 && $pos_tint<$tint_meta->npoints) { - fseek($tint_fh,$pos_tint*4); - $tint_tmp = unpack("f",fread($tint_fh,4)); - $value_tint = $tint_tmp[1]; - } - if ($pos_text>=0 && $pos_text<$text_meta->npoints) { - fseek($text_fh,$pos_text*4); - $text_tmp = unpack("f",fread($text_fh,4)); - $value_text = $text_tmp[1]; - } - - //if we face one or more NAN value among flow,tint,text, we should not make any calculation and $kwh should remain the same - if(!is_nan($value_flow) && !is_nan($value_tint) && !is_nan($value_text)){ - $kwh+=0.001*$vhc*$value_flow*max($value_tint-$value_text,0)*$out_meta->interval/3600; - } - //print("$kwh/"); - $buffer.=pack("f",$kwh); - } - if(!$buffer) { - return array("success"=>true, "message"=>"nothing to write - all is up to date"); - } + $this->kwh = is_nan($previous) ? 0 : $previous; + } - if(!$written_bytes=fwrite($out_fh,$buffer)){ - fclose($flow_fh); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - return array("success"=>false, "message"=>"unable to write to the file with id=$out"); + // Missing flow or temperatures add no energy + protected function row($flow, $tint, $text) + { + if (!is_nan($flow) && !is_nan($tint) && !is_nan($text)) { + $this->kwh += 0.001 * $this->params->vhc * $flow * max($tint - $text, 0) * $this->interval / 3600; } - $nbdataswritten=$written_bytes/4; - print("NOTICE: liquidorairflow_tokwh() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta($dir,$out,$out_meta); - fclose($flow_fh); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - print("last time value: $time / $kwh \n"); - updatetimevalue($out,$time,$kwh); - return array("success"=>true, "message"=>"bytes written: ".$written_bytes.", last time value: ".$time." ".$kwh); + return $this->kwh; } -} \ No newline at end of file +} diff --git a/processes/mergefeeds.php b/processes/mergefeeds.php index 9620801..c05f440 100644 --- a/processes/mergefeeds.php +++ b/processes/mergefeeds.php @@ -1,7 +1,9 @@ "Merge feeds", @@ -15,106 +17,11 @@ public function description() { ); } - public function process($params) + // Average of both feeds, or whichever feed has data + protected function row($a, $b) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $feedA_meta = getmeta($this->dir,$params->feedA); - $feedB_meta = getmeta($this->dir,$params->feedB); - - // Find longest interval - $interval = $feedA_meta->interval; - if ($feedB_meta->interval>$interval) $interval = $feedB_meta->interval; - // Find latest start time - $start_time = $feedA_meta->start_time; - if ($feedB_meta->start_time>$start_time) $start_time = $feedB_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $feedA_end_time = $feedA_meta->start_time + ($feedA_meta->interval * $feedA_meta->npoints); - $feedB_end_time = $feedB_meta->start_time + ($feedB_meta->interval * $feedB_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $feedA_end_time; - if ($feedB_end_time<$end_time) $end_time = $feedB_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$feedA_fh = @fopen($this->dir.$params->feedA.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedA feed"); - } - if (!$feedB_fh = @fopen($this->dir.$params->feedB.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedB feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_feedA = floor(($time - $feedA_meta->start_time) / $feedA_meta->interval); - $pos_feedB = floor(($time - $feedB_meta->start_time) / $feedB_meta->interval); - - $valueA = NAN; - $valueB = NAN; - - if ($pos_feedA>=0 && $pos_feedA<$feedA_meta->npoints) { - fseek($feedA_fh,$pos_feedA*4); - $tmp = unpack("f",fread($feedA_fh,4)); - $valueA = $tmp[1]; - } - - if ($pos_feedB>=0 && $pos_feedB<$feedB_meta->npoints) { - fseek($feedB_fh,$pos_feedB*4); - $tmp = unpack("f",fread($feedB_fh,4)); - $valueB = $tmp[1]; - } - - $outval = NAN; - if (!is_nan($valueA)) $outval = $valueA; - if (!is_nan($valueB)) $outval = $valueB; - if (!is_nan($valueA) && !is_nan($valueB)) $outval = ($valueB+$valueA)*0.5; - $buffer .= pack("f",$outval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($feedA_fh); - fclose($feedB_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$outval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$outval); + if (is_nan($a)) return $b; + if (is_nan($b)) return $a; + return ($a + $b) * 0.5; } -} \ No newline at end of file +} diff --git a/processes/offsetfeed.php b/processes/offsetfeed.php index f5d88ae..9169930 100644 --- a/processes/offsetfeed.php +++ b/processes/offsetfeed.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - $value += $params->offset; - - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value + $this->params->offset; } -} \ No newline at end of file +} diff --git a/processes/powertokwh.php b/processes/powertokwh.php index 4a3a387..0b339a2 100644 --- a/processes/powertokwh.php +++ b/processes/powertokwh.php @@ -1,7 +1,11 @@ "Power to kWh", @@ -17,96 +21,23 @@ public function description() { ); } - public function process($params) + protected function reset($previous) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $wh = 0; - $power = 0; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } + $this->wh = is_nan($previous) ? 0 : $previous * 1000; + $this->power = 0; + } - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,($start_pos-1)*4); - $tmp = unpack("f",fread($of,4)); - $wh = $tmp[1]*1000.0; - } - - $filtered_count = 0; - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - if (!is_nan($tmp[1])) $power = 1*$tmp[1]; - - // filter spurious power values +-1MW - if ($power>=$params->min_power && $power<$params->max_power) { - $wh += ($power * $input_meta->interval) / 3600.0; - $buffer .= pack("f",$wh*0.001); - } else { - $filtered_count++; + // Missing power values hold the last power value. Power values outside of the + // limits are treated as spurious and add no energy. + protected function row($power) + { + if (!is_nan($power)) { + if ($power < $this->params->min_power || $power >= $this->params->max_power) { + return $this->wh * 0.001; } + $this->power = $power; } - - fwrite($of,$buffer); - - if ($filtered_count>0) { - print "Filtered count: $filtered_count\n"; - } - - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - $value = $wh * 0.001; - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + $this->wh += ($this->power * $this->interval) / 3600.0; + return $this->wh * 0.001; } } diff --git a/processes/remove_morethan_lessthan.php b/processes/remove_morethan_lessthan.php index 185d9b5..067c220 100644 --- a/processes/remove_morethan_lessthan.php +++ b/processes/remove_morethan_lessthan.php @@ -1,8 +1,6 @@ "Limits", "description"=>"Remove values more than or less than a certain value", "settings"=>array( - "feedid"=>array("type"=>"feed", "engine"=>5, "short"=>"Select feed to remove nan values:"), + "feedid"=>array("type"=>"feed", "engine"=>5, "short"=>"Select feed to remove values from:"), "morethan"=>array("type"=>"value", "short"=>"Remove values above this limit:"), - "lessthan"=>array("type"=>"value", "short"=>"Remote values below this limit:") + "lessthan"=>array("type"=>"value", "short"=>"Remove values below this limit:") ) ); } - public function process($processitem) + protected function value($value) { - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - $morethan = $processitem->morethan; - $lessthan = $processitem->lessthan; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - - $fix_count = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - if ($values[$i]>$morethan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",NAN)); - $fix_count ++; - } - else if ($values[$i]<$lessthan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",NAN)); - $fix_count ++; - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$fix_count." datapoints, ".round(($fix_count/$npoints)*100)."%\n"); + if ($value > $this->params->morethan || $value < $this->params->lessthan) return NAN; + return $value; } -} \ No newline at end of file +} diff --git a/processes/removenan_lastvalue.php b/processes/removenan_lastvalue.php index dfb07ae..aae249d 100644 --- a/processes/removenan_lastvalue.php +++ b/processes/removenan_lastvalue.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - $in_nan_period = 0; - $startval = 0; - $startpos = 0; - $nanfix = 0; + private $last_value = NAN; - $value = NAN; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); + protected function reset() + { + $this->last_value = NAN; + } - if (is_nan($values[$i])) { - fseek($fh,($dpos)*4); - fwrite($fh,pack("f",$value)); - $nanfix++; - } else { - $value = $values[$i]; - } - } - $dplefttoread -= $count; - $fpos += $count; + // Replace NAN with the last value, the last value carries over between blocks + protected function values($values) + { + foreach ($values as $i => $value) { + if (is_nan($value)) $values[$i] = $this->last_value; + else $this->last_value = $value; } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$nanfix." datapoints, ".round(($nanfix/$npoints)*100)."%\n"); + return $values; } -} \ No newline at end of file +} diff --git a/processes/scalefeed.php b/processes/scalefeed.php index be1676b..72d335c 100644 --- a/processes/scalefeed.php +++ b/processes/scalefeed.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]*$params->scale; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value * $this->params->scale; } -} \ No newline at end of file +} diff --git a/processes/solarbatterykwh.php b/processes/solarbatterykwh.php index 1255939..1002993 100644 --- a/processes/solarbatterykwh.php +++ b/processes/solarbatterykwh.php @@ -179,11 +179,6 @@ public function process($p) return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - $solar = 0; - $use = 0; - $grid = 0; - $battery = 0; - $solar_to_load_kwh = 0; $solar_to_grid_kwh = 0; $solar_to_battery_kwh = 0; @@ -193,16 +188,28 @@ public function process($p) $grid_to_battery_kwh = 0; // $solar_kwh = 0; // used for testing - // Get starting cumulative kWh values - $model->seek_to_time($start_time); - if ($model->meta['solar_to_load_kwh']->npoints) $solar_to_load_kwh = $model->read('solar_to_load_kwh',$solar_to_load_kwh); - if ($model->meta['solar_to_grid_kwh']->npoints) $solar_to_grid_kwh = $model->read('solar_to_grid_kwh',$solar_to_grid_kwh); - if ($model->meta['solar_to_battery_kwh']->npoints) $solar_to_battery_kwh = $model->read('solar_to_battery_kwh',$solar_to_battery_kwh); - if ($model->meta['battery_to_load_kwh']->npoints) $battery_to_load_kwh = $model->read('battery_to_load_kwh',$battery_to_load_kwh); - if ($model->meta['battery_to_grid_kwh']->npoints) $battery_to_grid_kwh = $model->read('battery_to_grid_kwh',$battery_to_grid_kwh); - if ($model->meta['grid_to_load_kwh']->npoints) $grid_to_load_kwh = $model->read('grid_to_load_kwh',$grid_to_load_kwh); - if ($model->meta['grid_to_battery_kwh']->npoints) $grid_to_battery_kwh = $model->read('grid_to_battery_kwh',$grid_to_battery_kwh); - // if ($model->meta['solar_kwh']->npoints) $solar_kwh = $model->read('solar_kwh',$solar_kwh); + // Get starting cumulative kWh values. The value at the start of the output feed is zero, + // after that the value at each slot includes the datapoint at the start of the slot, in + // which case processing continues from the next datapoint so that it is not counted twice. + $first_time = $start_time; + $start_slot = floor($start_time / $output_interval) * $output_interval; + if ($start_slot > $model->meta['grid_to_load_kwh']->start_time) { + $solar_to_load_kwh = $model->read_at('solar_to_load_kwh',$start_slot,$solar_to_load_kwh); + $solar_to_grid_kwh = $model->read_at('solar_to_grid_kwh',$start_slot,$solar_to_grid_kwh); + $solar_to_battery_kwh = $model->read_at('solar_to_battery_kwh',$start_slot,$solar_to_battery_kwh); + $battery_to_load_kwh = $model->read_at('battery_to_load_kwh',$start_slot,$battery_to_load_kwh); + $battery_to_grid_kwh = $model->read_at('battery_to_grid_kwh',$start_slot,$battery_to_grid_kwh); + $grid_to_load_kwh = $model->read_at('grid_to_load_kwh',$start_slot,$grid_to_load_kwh); + $grid_to_battery_kwh = $model->read_at('grid_to_battery_kwh',$start_slot,$grid_to_battery_kwh); + // $solar_kwh = $model->read_at('solar_kwh',$start_slot,$solar_kwh); + $first_time = $start_time + $interval; + } + + // Missing values hold the last valid value, including from before this run + $solar = $this->valid_power($model->last_valid('solar',$first_time,0), 0); + $use = $this->valid_power($model->last_valid('use',$first_time,0), 0); + $grid = $this->valid_power($model->last_valid('grid',$first_time,0), 0); + $battery = $this->valid_power($model->last_valid('battery',$first_time,0), 0); $starting_values = array( "solar_to_load_kwh"=>number_format($solar_to_load_kwh, 3, '.', '')*1, @@ -214,8 +221,18 @@ public function process($p) "grid_to_battery_kwh"=>number_format($grid_to_battery_kwh, 3, '.', '')*1 ); - // Reset again - $model->seek_to_time($start_time); + // Write the starting values at the start slot, later slots follow + $model->seek_to_time_outputs($start_time); + $model->write('solar_to_load_kwh',$solar_to_load_kwh); + $model->write('solar_to_grid_kwh',$solar_to_grid_kwh); + $model->write('solar_to_battery_kwh',$solar_to_battery_kwh); + $model->write('battery_to_load_kwh',$battery_to_load_kwh); + $model->write('battery_to_grid_kwh',$battery_to_grid_kwh); + $model->write('grid_to_load_kwh',$grid_to_load_kwh); + $model->write('grid_to_battery_kwh',$grid_to_battery_kwh); + // $model->write('solar_kwh',$solar_kwh); // used for testing + + $model->seek_to_time_inputs($first_time); $power_to_kwh = $interval / 3600000.0; // conversion factor from W to kWh for given interval @@ -229,7 +246,7 @@ public function process($p) $dp_per_feed_written = 0; - for ($time=$start_time; $time<$end_time; $time+=$interval) + for ($time=$first_time; $time<$end_time; $time+=$interval) { $solar = $this->valid_power($model->read('solar',$solar), $solar); $use = $this->valid_power($model->read('use',$use), $use); @@ -353,7 +370,6 @@ public function process($p) } echo "\n"; - $model->seek_to_time($start_time+$output_interval); $buffersize = $model->save_all(); $elapsed_time = microtime(true) - $process_start_time; diff --git a/processes/solardirectcalc.php b/processes/solardirectcalc.php index d13a872..fa73745 100644 --- a/processes/solardirectcalc.php +++ b/processes/solardirectcalc.php @@ -1,7 +1,10 @@ "Solar direct calculation", @@ -15,107 +18,9 @@ public function description() { ); } - public function process($params) + protected function row($use, $gen) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->use); - $gen_meta = getmeta($this->dir,$params->gen); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->out,$out_meta); - $out_meta = getmeta($this->dir,$params->out); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->use.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->gen.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $directval = NAN; - if (!is_nan($useval) && !is_nan($genval)) { - if ($genval>$useval) $genval = $useval; - $directval = $genval; - } - $buffer .= pack("f",$directval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$directval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$directval); + if (is_nan($use) || is_nan($gen)) return NAN; + return min($use, $gen); } -} \ No newline at end of file +} diff --git a/processes/to_signed.php b/processes/to_signed.php index dad9c23..e3a5a49 100644 --- a/processes/to_signed.php +++ b/processes/to_signed.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - $dp_modified = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - - // Read in a block of data 100,000 datapoints at a time - // this is quite a bit faster than reading in one datapoint at a time - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - // convert to signed int 16 - if ($values[$i] > 32767) { - $fixed_value = $values[$i] - 65536; - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$fixed_value)); - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - - echo "\n"; - - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"to_signed: ".$dp_modified." datapoints, ".round(($dp_modified/$npoints)*100)."%\n"); + return $value > 32767 ? $value - 65536 : $value; } -} \ No newline at end of file +} diff --git a/readme.md b/readme.md index f4f75e7..a16535d 100644 --- a/readme.md +++ b/readme.md @@ -21,4 +21,18 @@ This is installed by default on the standard emonSD image build. Alternatively to run manually: cd /opt/emoncms/modules/postprocess - sudo php postprocess_run.php \ No newline at end of file + sudo php postprocess_run.php +### Tests + +Processes are tested against small synthetic feeds in a temporary directory, no database or redis is required: + + php tests/run.php # all tests + php tests/run.php scalefeed # tests whose name contains scalefeed + +Most processes extend a base class in `common.php` and only implement the calculation: + +- `MapProcess`: transforms each value of an input feed into an output feed, implement `value($value)`, e.g. `processes/scalefeed.php` +- `InPlaceProcess`: transforms each value of a feed in place, implement `value($value)`, e.g. `processes/to_signed.php` +- `CombineProcess`: combines input feeds on a common timeline into an output feed, implement `row($a, $b, ...)`, e.g. `processes/importcalc.php`, or with `reset($previous)` to accumulate, e.g. `processes/powertokwh.php` + +Simulations that step through time with several inputs and outputs use `ModelHelper`, e.g. `processes/batterysimulator.php`. Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. diff --git a/tests/run.php b/tests/run.php new file mode 100644 index 0000000..19b4448 --- /dev/null +++ b/tests/run.php @@ -0,0 +1,239 @@ +hashes[$key] = $values; return true; } +} +$redis = new TestRedis(); + +$tests = array(); +function test($name, $fn) { global $tests; $tests[$name] = $fn; } + +// -------------------------------------------------------------------------------- +// Feed helpers +// -------------------------------------------------------------------------------- + +$testdir = sys_get_temp_dir()."/postprocess-tests-".getmypid()."/"; + +// Create a feed with the given start time, interval and values +function feed($id, $start_time, $interval, $values = array()) +{ + global $testdir; + $meta = new stdClass(); + $meta->start_time = $start_time; + $meta->interval = $interval; + createmeta($testdir, $id, $meta); + file_put_contents($testdir.$id.".dat", count($values) ? pack("f*", ...$values) : ""); +} + +// Create an empty feed as the feed list does when a new output feed is created +function empty_feed($id, $interval = 10) +{ + feed($id, 0, $interval); +} + +// Append values to an existing feed +function append($id, $values) +{ + global $testdir; + file_put_contents($testdir.$id.".dat", pack("f*", ...$values), FILE_APPEND); +} + +function values($id) +{ + global $testdir; + $bytes = file_get_contents($testdir.$id.".dat"); + return strlen($bytes) ? array_values(unpack("f*", $bytes)) : array(); +} + +function meta($id) +{ + global $testdir; + return getmeta($testdir, $id); +} + +// Run a process, default mode is reprocess all. Output printed by the process is captured +// and only shown if the test fails +function run($process, $params = array()) +{ + global $testdir, $captured; + $params = (object) array_merge(array("process_mode"=>"all", "process_start"=>0), $params); + $class = "PostProcess_".$process; + $p = new $class($testdir); + ob_start(); + try { + $result = $p->process($params); + } finally { + $captured .= ob_get_clean(); + } + return $result; +} + +// -------------------------------------------------------------------------------- +// Assertions +// -------------------------------------------------------------------------------- + +class TestFailure extends Exception {} + +function fmt($value) +{ + if (is_array($value)) return "[".implode(", ", array_map('fmt', $value))."]"; + if (is_float($value) && is_nan($value)) return "NAN"; + return var_export($value, true); +} + +function same_value($expected, $actual) +{ + if (is_float($expected) && is_nan($expected)) return is_float($actual) && is_nan($actual); + if (is_float($actual) && is_nan($actual)) return false; + if (is_numeric($expected) && is_numeric($actual)) { + // Feed values are stored as 32 bit floats + return abs($expected - $actual) <= 1e-4 * max(1, abs($expected)); + } + return $expected === $actual; +} + +function assert_equal($expected, $actual, $message = "") +{ + // Long arrays: report the first difference + if (is_array($expected) && is_array($actual) && max(count($expected), count($actual)) > 20) { + $prefix = ($message ? "$message: " : ""); + foreach ($expected as $i => $value) { + if (!array_key_exists($i, $actual)) { + throw new TestFailure($prefix."expected ".count($expected)." values, got ".count($actual)); + } + if (!same_value($value, $actual[$i])) { + $from = max(0, $i - 2); + throw new TestFailure($prefix."first difference at index $i of ".count($expected).", expected " + .fmt(array_slice($expected, $from, 5))." got ".fmt(array_slice($actual, $from, 5))." from index $from"); + } + } + if (count($actual) != count($expected)) { + throw new TestFailure($prefix."expected ".count($expected)." values, got ".count($actual)); + } + return; + } + $ok = is_array($expected) + ? is_array($actual) && count($expected) == count($actual) + && count(array_filter(array_map('same_value', $expected, $actual))) == count($expected) + : same_value($expected, $actual); + if (!$ok) throw new TestFailure(($message ? "$message: " : "")."expected ".fmt($expected).", got ".fmt($actual)); +} + +function assert_feed($id, $start_time, $interval, $values) +{ + $meta = meta($id); + assert_equal($start_time, $meta->start_time, "feed $id start time"); + assert_equal($interval, $meta->interval, "feed $id interval"); + assert_equal($values, values($id), "feed $id values"); +} + +function assert_success($result) +{ + if (!$result["success"]) throw new TestFailure("process failed: ".$result["message"]); +} + +function assert_error($result, $contains = "") +{ + if ($result["success"]) throw new TestFailure("expected process to fail, got: ".$result["message"]); + if ($contains && strpos($result["message"], $contains) === false) { + throw new TestFailure("expected error containing '$contains', got: ".$result["message"]); + } +} + +function assert_redis($id, $time, $value) +{ + global $redis; + if (!isset($redis->hashes["feed:$id"])) throw new TestFailure("redis feed:$id not set"); + assert_equal($time, $redis->hashes["feed:$id"]["time"], "redis feed:$id time"); + assert_equal($value, $redis->hashes["feed:$id"]["value"], "redis feed:$id value"); +} + +// Compare feeds with a snapshot in tests/snapshots/$name.json. A missing snapshot is +// created from the current output, delete a snapshot to update it after an intended change. +function assert_snapshot($name, $feeds) +{ + $snapshot = array(); + foreach ($feeds as $key => $id) { + $meta = meta($id); + $values = values($id); + $last = count($values) ? end($values) : NAN; + $snapshot[$key] = array( + "start_time"=>$meta->start_time, + "interval"=>$meta->interval, + "npoints"=>count($values), + "md5"=>count($values) ? md5(pack("f*", ...$values)) : "", + "last"=>is_nan($last) ? "NAN" : sprintf("%.6g", $last) + ); + } + $json = json_encode($snapshot, JSON_PRETTY_PRINT)."\n"; + $file = __DIR__."/snapshots/$name.json"; + if (!file_exists($file)) { + if (!is_dir(__DIR__."/snapshots")) mkdir(__DIR__."/snapshots"); + file_put_contents($file, $json); + echo " created snapshot $name\n"; + return; + } + $expected = json_decode(file_get_contents($file), true); + foreach ($expected as $key => $feed) { + if (!isset($snapshot[$key])) throw new TestFailure("snapshot $name: missing feed $key"); + if ($snapshot[$key] != $feed) { + throw new TestFailure("snapshot $name: feed $key expected ".json_encode($feed).", got ".json_encode($snapshot[$key])); + } + } + if (count($snapshot) != count($expected)) throw new TestFailure("snapshot $name: feed count differs"); +} + +// -------------------------------------------------------------------------------- +// Run +// -------------------------------------------------------------------------------- + +foreach (glob(__DIR__."/test_*.php") as $file) require $file; + +$filter = $argv[1] ?? ""; +$passed = 0; +$failed = 0; + +foreach ($tests as $name => $fn) { + if ($filter && strpos($name, $filter) === false) continue; + + // Fresh data directory and redis for every test + if (is_dir($testdir)) array_map('unlink', glob($testdir."*")); + else mkdir($testdir); + $redis = new TestRedis(); + $captured = ""; + + try { + $fn(); + $passed++; + echo " ok $name\n"; + } catch (Throwable $e) { + $failed++; + echo " FAIL $name\n"; + $where = ($e instanceof TestFailure) ? "" : " (".get_class($e)." at ".basename($e->getFile()).":".$e->getLine().")"; + echo " ".$e->getMessage()."$where\n"; + if (trim($captured) !== "") echo " output: ".str_replace("\n", "\n ", trim($captured))."\n"; + } +} + +array_map('unlink', glob($testdir."*")); +@rmdir($testdir); + +echo "\n$passed passed, $failed failed\n"; +exit($failed ? 1 : 0); diff --git a/tests/snapshots/batterysimulator.json b/tests/snapshots/batterysimulator.json new file mode 100644 index 0000000..296b30f --- /dev/null +++ b/tests/snapshots/batterysimulator.json @@ -0,0 +1,128 @@ +{ + "soc": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "cd6a1e4ecc83120ab9a53c5e35227276", + "last": "100" + }, + "power": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "bf322ec81f1dc37f54b67ea394429fae", + "last": "0" + }, + "charge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "4028a6ecf3248fdd2488badffad67d31", + "last": "0" + }, + "discharge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "04ef553a20961d917f816362b98430ea", + "last": "0" + }, + "charge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6c5c1f96e3724a77f882d7b72e0a18cc", + "last": "23.5121" + }, + "discharge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "fb72751c106bf7daa8e1c50343a0df1f", + "last": "19.1296" + }, + "grid": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3357c9c34d706cff77d54e444ef49feb", + "last": "-2423.78" + }, + "import": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "e32572994d19a22c6ffd81b020d351af", + "last": "0" + }, + "export": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "5077770a1155197f29c04fe066dc1655", + "last": "2423.78" + }, + "import_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "c9389f3212b1d6dc49eb135fef4ef616", + "last": "42.0285" + }, + "export_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "aee327d1845e67a971fa1d5c0d46be22", + "last": "190.772" + }, + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6ca9150b4d120e427be3faa614d79eb5", + "last": "54.4157" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "aee327d1845e67a971fa1d5c0d46be22", + "last": "190.772" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6c5c1f96e3724a77f882d7b72e0a18cc", + "last": "23.5121" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "fb72751c106bf7daa8e1c50343a0df1f", + "last": "19.1296" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d948f712fa329203f590e91cf6dd3e3e", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "c9389f3212b1d6dc49eb135fef4ef616", + "last": "42.0285" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d948f712fa329203f590e91cf6dd3e3e", + "last": "0" + } +} diff --git a/tests/snapshots/batterysimulator_offpeak.json b/tests/snapshots/batterysimulator_offpeak.json new file mode 100644 index 0000000..ac39394 --- /dev/null +++ b/tests/snapshots/batterysimulator_offpeak.json @@ -0,0 +1,128 @@ +{ + "soc": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "35e84c7db42afeb7a971cd378e13dca6", + "last": "100" + }, + "power": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "f5caf36755990d82d104d0832f929798", + "last": "0" + }, + "charge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "5de56e2e86f1d8bb7fe3e7bbca454012", + "last": "0" + }, + "discharge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "ab80ed81d948598172e80e78a6c5b339", + "last": "0" + }, + "charge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "605ab622937a327b1202b0e5ebdda366", + "last": "54.4838" + }, + "discharge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "cece123df4c9a0c90cba2c9ee8d1c5af", + "last": "47.0816" + }, + "grid": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "bb1792f735bfecd52fbf87962bb1eefa", + "last": "-2423.78" + }, + "import": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "a0efab22cd8180eed22a9c8c4fc0c613", + "last": "0" + }, + "export": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3ad7796a94488ce085d395c49b115996", + "last": "2423.78" + }, + "import_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "59b2cd325adc18b0160321cfd1d85744", + "last": "101.801" + }, + "export_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d97daf00275d48faf275a8dd7ddb1d0f", + "last": "247.525" + }, + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6ca9150b4d120e427be3faa614d79eb5", + "last": "54.4157" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "50127ab39c65b3780ddaa04e377027bc", + "last": "201.988" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3fb1a75cf58fe7f5c40348d9da704bde", + "last": "12.2963" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8c16a4907b1ad921a64d4d994cfa995b", + "last": "1.54472" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d2f1398575efafe623727270a0165645", + "last": "45.5369" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8018302a2893afee54cb082a07237887", + "last": "59.6134" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8673272af7085e080952e7953ed82a7a", + "last": "42.1875" + } +} diff --git a/tests/snapshots/solarbatterykwh.json b/tests/snapshots/solarbatterykwh.json new file mode 100644 index 0000000..7fa948a --- /dev/null +++ b/tests/snapshots/solarbatterykwh.json @@ -0,0 +1,51 @@ +{ + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "9cd57d52b20f34b231e58ff7f7581bb6", + "last": "54.4076" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40219b1b54dba3bae1131f77937ecc68", + "last": "24.9939" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40cdcfccc2a1d85b4744612c7da077fa", + "last": "189.231" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "c484775c3dacc31ee3741093835be1d4", + "last": "30.381" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "0181a3b838bb4acc9a89b48ef0b842b1", + "last": "30.7772" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + } +} diff --git a/tests/snapshots/solarbatterykwh_derived.json b/tests/snapshots/solarbatterykwh_derived.json new file mode 100644 index 0000000..b1a7028 --- /dev/null +++ b/tests/snapshots/solarbatterykwh_derived.json @@ -0,0 +1,51 @@ +{ + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "9cd57d52b20f34b231e58ff7f7581bb6", + "last": "54.4076" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40219b1b54dba3bae1131f77937ecc68", + "last": "24.9939" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40cdcfccc2a1d85b4744612c7da077fa", + "last": "189.231" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "c484775c3dacc31ee3741093835be1d4", + "last": "30.381" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "279b40ea1a8bab53b8fdf25db84f56af", + "last": "30.7772" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "e22b612ab8c181daf962f0132c4296e7", + "last": "1.34542e-6" + } +} diff --git a/tests/test_combine.php b/tests/test_combine.php new file mode 100644 index 0000000..97face2 --- /dev/null +++ b/tests/test_combine.php @@ -0,0 +1,254 @@ +1, "gen"=>2, "out"=>3])); + // Missing generation is treated as no generation + assert_feed(3, 1000, 10, [300, 0, NAN, 300]); +}); + +test("exportcalc", function() { + feed(1, 1000, 10, [500, 100, NAN, 300]); + feed(2, 1000, 10, [200, 300, 100, NAN]); + empty_feed(3); + assert_success(run("exportcalc", ["generation"=>1, "consumption"=>2, "output"=>3])); + // Missing consumption is treated as no consumption + assert_feed(3, 1000, 10, [300, 0, NAN, 300]); +}); + +test("solardirectcalc", function() { + feed(1, 1000, 10, [500, 100, NAN, 300]); + feed(2, 1000, 10, [200, 300, 100, NAN]); + empty_feed(3); + assert_success(run("solardirectcalc", ["use"=>1, "gen"=>2, "out"=>3])); + assert_feed(3, 1000, 10, [200, 100, NAN, NAN]); +}); + +// -------------------------------------------------------------------------------- +// Two inputs, modes +// -------------------------------------------------------------------------------- + +test("mergefeeds recent", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [3, 4]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + append(1, [5, 6]); + append(2, [7]); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 1000, 10, [2, 3, 6]); +}); + +test("mergefeeds from", function() { + feed(1, 1000, 10, [1, 2, 3]); + feed(2, 1000, 10, [1, 2, 3]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + feed(1, 1000, 10, [10, 20, 30]); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"from", "process_start"=>1010])); + assert_feed(3, 1000, 10, [1, 11, 16.5]); +}); + +test("addfeeds later start time", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1020, 10, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1020, 10, [13, 24]); +}); + +// -------------------------------------------------------------------------------- +// Accumulators +// -------------------------------------------------------------------------------- + +test("accumulator recent continues from last value", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + append(1, [3]); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [1, 3, 6]); +}); + +test("constantflow_tokwh", function() { + // kWh per hour = 0.001 * vhc * flow * dT = 3.6 * dT, only positive dT, NAN adds nothing + feed(1, 3600, 3600, [20, 20, NAN, 20]); + feed(2, 3600, 3600, [10, 25, 10, 15]); + empty_feed(3); + assert_success(run("constantflow_tokwh", ["vhc"=>1000, "flow"=>3.6, "tint"=>1, "text"=>2, "output"=>3])); + assert_feed(3, 3600, 3600, [36, 36, 36, 54]); + append(1, [20]); + append(2, [10]); + assert_success(run("constantflow_tokwh", ["vhc"=>1000, "flow"=>3.6, "tint"=>1, "text"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 3600, 3600, [36, 36, 36, 54, 90]); +}); + +test("liquidorairflow_tokwh", function() { + // kWh per hour = 0.001 * vhc * flow * dT = 10 * flow + feed(1, 3600, 3600, [1, 2, NAN, 1]); + feed(2, 3600, 3600, [20, 20, 20, 20]); + feed(3, 3600, 3600, [10, 10, 10, 10]); + empty_feed(4); + assert_success(run("liquidorairflow_tokwh", ["vhc"=>1000, "flow"=>1, "tint"=>2, "text"=>3, "output"=>4])); + assert_feed(4, 3600, 3600, [10, 30, 30, 40]); +}); + +// -------------------------------------------------------------------------------- +// Basic formula +// -------------------------------------------------------------------------------- + +test("basic_formula sum and scale", function() { + feed(1, 1000, 10, [1, 2, NAN]); + feed(2, 1000, 10, [10, 20, 30]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1+2*f2", "output"=>4])); + assert_feed(4, 1000, 10, [21, 42, NAN]); +}); + +test("basic_formula max", function() { + feed(1, 1000, 10, [5, 1, NAN]); + feed(2, 1000, 10, [2, 3, 1]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"2*max(f1-f2,0)", "output"=>4])); + assert_feed(4, 1000, 10, [6, 0, NAN]); +}); + +test("basic_formula brackets and division", function() { + feed(1, 1000, 10, [3, 5]); + feed(2, 1000, 10, [2, 4]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"10*(f1-1)+f2/2", "output"=>4])); + assert_feed(4, 1000, 10, [21, 42]); +}); + +test("basic_formula recent", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4])); + append(1, [3]); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4, "process_mode"=>"recent"])); + assert_feed(4, 1000, 10, [3, 6, 9]); +}); + +test("basic_formula invalid", function() { + feed(1, 1000, 10, [1]); + feed(2, 1000, 10, [1]); + empty_feed(4); + assert_error(run("basic_formula", ["formula"=>"(f1-f2)*(f1-f2)", "output"=>4]), "could not understand"); +}); + +test("basic_formula feed ids that are prefixes of each other", function() { + feed(1, 1000, 10, [1, 2]); + feed(12, 1000, 10, [10, 20]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1+f12", "output"=>4])); + assert_feed(4, 1000, 10, [11, 22]); +}); + +test("basic_formula from", function() { + feed(1, 1000, 10, [1, 2, 3]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4])); + feed(1, 1000, 10, [10, 20, 30]); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4, "process_mode"=>"from", "process_start"=>1010])); + assert_feed(4, 1000, 10, [3, 60, 90]); +}); + +test("basic_formula multiply by zero", function() { + feed(1, 1000, 10, [1, 2, NAN]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*0+3", "output"=>4])); + assert_feed(4, 1000, 10, [3, 3, NAN]); +}); + +test("basic_formula division by zero is missing", function() { + feed(1, 1000, 10, [6, 6, 6]); + feed(2, 1000, 10, [2, 0, NAN]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1/f2", "output"=>4])); + assert_feed(4, 1000, 10, [3, NAN, NAN]); + assert_success(run("basic_formula", ["formula"=>"f1/0+2", "output"=>4])); + assert_feed(4, 1000, 10, [NAN, NAN, NAN]); +}); + +// Formula value for one datapoint, $values keyed by feed id. Reference for the block +// evaluator in basic_formula, evaluating each datapoint separately +function formula_reference($elements, $values) +{ + // sum of the products of each term + $sum = function($terms) use ($values) { + $s = []; + foreach ($terms as $term) { + $product = []; + foreach ($term as $e) { + $value = ($e[0] == "feed") ? $values[$e[2]] : (float) $e[2]; + if ($e[1] == "/") $value = ($value == 0) ? NAN : 1 / $value; + if ($e[1] == "-") $value = -$value; + $product[] = $value; + } + $s[] = array_product($product); + } + return array_sum($s); + }; + $total = 0; + foreach ($elements as $element) { + $s1 = $sum([$element->scale]); + $s2 = $sum($element->formula); + $s3 = $sum([$element->scale_right]); + if ($element->function == "max") { + $arg2 = is_numeric($element->arg2) ? $element->arg2 : $sum($element->arg2); + if (is_nan($s1) || is_nan($s2) || is_nan($s3) || is_nan($arg2)) return NAN; + $total += $s1 * max($s2, $arg2) * $s3; + } else { + $total += $s1 * $s2 * $s3; + } + } + return $total; +} + +test("basic_formula block evaluation matches per datapoint reference", function() { + global $testdir; + // Random data with missing values and zeros, more than one block + mt_srand(42); + $n = POSTPROCESS_BLOCKSIZE + 1000; + $data = []; + foreach ([1, 2, 3, 12] as $id) { + for ($i=0; $i<$n; $i++) { + $r = mt_rand(0, 99); + $data[$id][$i] = $r < 3 ? NAN : ($r < 8 ? 0.0 : mt_rand(-5000, 5000) / 100); + } + feed($id, 0, 10, $data[$id]); + } + // Feed values as stored + foreach ($data as $id => $values) $data[$id] = values($id); + + $formulas = ["f1+2*f2", "2*max(f1-f2,0)", "10*(f1-1)+f2/2", "f1-0.5*f2+max(f3-21,0)", + "1162.5*f2*(f1-11)", "f1/f2", "-f1*f2/3+4", "max(f1,f2)", "3*max(f1-f2,f3*2)/4", "f1+f12", + "f1*0", "f1/0+2", "-(f1+f2)", "f1-f2-f3*f12/2.5", "2(f1+3)-f2", "f1/f2/f3", "max(f1-f2,f3-f12)+f1"]; + + $p = new PostProcess_basic_formula($testdir); + foreach ($formulas as $formula) { + empty_feed(9); + assert_success(run("basic_formula", ["formula"=>$formula, "output"=>9])); + $parsed = $p->parse_formula($formula); + $expected = []; + for ($i=0; $i<$n; $i++) { + $values = []; + foreach ($parsed["feed_ids"] as $id) $values[$id] = $data[$id][$i]; + $expected[] = formula_reference($parsed["elements"], $values); + } + // Compare as stored, 32 bit floats + $expected = array_values(unpack("f*", pack("f*", ...$expected))); + if (pack("f*", ...$expected) !== pack("f*", ...values(9))) { + assert_equal($expected, values(9), $formula); + throw new TestFailure("$formula: differs from reference by less than the assert_equal tolerance"); + } + } +}); diff --git a/tests/test_common.php b/tests/test_common.php new file mode 100644 index 0000000..98446ec --- /dev/null +++ b/tests/test_common.php @@ -0,0 +1,169 @@ +$start_time, "interval"=>$interval, "npoints"=>$npoints); +} + +test("resolve_start_time", function() { + $meta = meta_of(1000, 10, 5); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"all"], $meta), "all"); + assert_equal(1050, resolve_start_time((object) ["process_mode"=>"recent"], $meta), "recent"); + assert_equal(1050, resolve_start_time((object) [], $meta), "default is recent"); + assert_equal(1020, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>1025], $meta), "from aligns down"); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>0], $meta), "from before start"); + assert_equal(1050, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>9999], $meta), "from after end"); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"recent"], meta_of(1000, 10, 0)), "recent empty"); +}); + +// -------------------------------------------------------------------------------- +// FeedReader +// -------------------------------------------------------------------------------- + +test("FeedReader read", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, NAN, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal(4, $feed->meta->npoints); + assert_equal([1, 2, NAN, 4], $feed->read(0, 4)); + assert_equal([2, NAN], $feed->read(1, 2)); + // Outside of the feed + assert_equal([NAN, NAN, 1, 2], $feed->read(-2, 4)); + assert_equal([4, NAN, NAN], $feed->read(3, 3)); + assert_equal([NAN, NAN], $feed->read(10, 2)); +}); + +test("FeedReader missing feed", function() { + global $testdir; + try { + new FeedReader($testdir, 99); + } catch (PostProcessException $e) { + return; + } + throw new TestFailure("expected PostProcessException"); +}); + +test("FeedReader read_time same interval", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, 3, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal([2, 3], $feed->read_time(1010, 10, 2)); + assert_equal([NAN, 1], $feed->read_time(990, 10, 2)); +}); + +test("FeedReader read_time same interval not aligned", function() { + global $testdir; + // Datapoints at 1005, 1015, ... each interval takes the datapoint within it + feed(1, 1005, 10, [1, 2, 3, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal([1, 2, 3, 4, NAN], $feed->read_time(1000, 10, 5)); +}); + +test("FeedReader read_time downsample averages ignoring NAN", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, 3, NAN, NAN, NAN, 7]); + $feed = new FeedReader($testdir, 1); + assert_equal([1.5, 3, NAN, 7], $feed->read_time(1000, 20, 4)); +}); + +test("FeedReader read_time upsample holds value", function() { + global $testdir; + feed(1, 1000, 30, [1, 2]); + $feed = new FeedReader($testdir, 1); + assert_equal([NAN, 1, 1, 1, 2, 2, 2, NAN], $feed->read_time(990, 10, 8)); +}); + +// -------------------------------------------------------------------------------- +// FeedWriter +// -------------------------------------------------------------------------------- + +test("FeedWriter new feed writes meta with data", function() { + global $testdir; + empty_feed(2); + $out = new FeedWriter($testdir, 2); + $out->init(1000, 10); + $out->seek(1000); + // Meta is not written until there is data + assert_equal(0, meta(2)->start_time); + $out->write(1); + $out->write_block([2, NAN]); + assert_equal(12, $out->close()); + assert_feed(2, 1000, 10, [1, 2, NAN]); + // Last non NAN value + assert_redis(2, 1010, 2); +}); + +test("FeedWriter init on feed with data", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3]); + $out = new FeedWriter($testdir, 2); + $out->init(1000, 10); + try { + $out->init(1000, 20); + throw new TestFailure("expected interval mismatch"); + } catch (PostProcessException $e) { + assert_equal(true, strpos($e->getMessage(), "interval mismatch") !== false); + } + $out->init(2000, 20, true); + $out->seek(2000); + $out->write(5); + $out->close(); + assert_feed(2, 2000, 20, [5]); +}); + +test("FeedWriter overwrite in place", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3, 4]); + $out = new FeedWriter($testdir, 2); + $out->seek(1010); + $out->write_block([20, 30]); + $out->close(); + assert_feed(2, 1000, 10, [1, 20, 30, 4]); +}); + +test("FeedWriter truncate removes stale data", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3, 4]); + $out = new FeedWriter($testdir, 2); + $out->seek(1010); + $out->write(20); + $out->close(true); + assert_feed(2, 1000, 10, [1, 20]); +}); + +test("FeedWriter seek beyond end pads with NAN", function() { + global $testdir; + feed(2, 1000, 10, [1]); + $out = new FeedWriter($testdir, 2); + $out->seek(1030); + $out->write(4); + $out->close(); + assert_feed(2, 1000, 10, [1, NAN, NAN, 4]); +}); + +test("FeedWriter rejects unaligned seek", function() { + global $testdir; + feed(2, 1000, 10, [1]); + $out = new FeedWriter($testdir, 2); + foreach ([1005, 990] as $time) { + try { + $out->seek($time); + throw new TestFailure("expected invalid write time for $time"); + } catch (PostProcessException $e) {} + } +}); + +test("FeedWriter flushes large writes in blocks", function() { + global $testdir; + empty_feed(2); + $n = POSTPROCESS_BLOCKSIZE * 2 + 10; + $out = new FeedWriter($testdir, 2); + $out->init(0, 10); + $out->seek(0); + for ($i=0; $i<$n; $i++) $out->write($i); + $out->close(); + $values = values(2); + assert_equal($n, count($values)); + assert_equal($n - 1, $values[$n - 1]); +}); diff --git a/tests/test_processes.php b/tests/test_processes.php new file mode 100644 index 0000000..4de9c7b --- /dev/null +++ b/tests/test_processes.php @@ -0,0 +1,346 @@ +1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4, NAN, 8]); + // Time of the last datapoint + assert_redis(2, 1030, 8); +}); + +test("scalefeed recent appends new data only", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + // Changes to existing input data are not picked up in recent mode + feed(1, 1000, 10, [100, 100, 100, 100, 5, 6]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [2, 4, NAN, 8, 10, 12]); +}); + +test("scalefeed recent up to date", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [2, 4]); +}); + +test("scalefeed from", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + feed(1, 1000, 10, [10, 20, 30, 40]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"from", "process_start"=>1020])); + assert_feed(2, 1000, 10, [2, 4, 60, 80]); +}); + +test("scalefeed recent rejects interval mismatch", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 20, [1]); + assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"]), "interval mismatch"); + assert_feed(2, 1000, 20, [1]); +}); + +test("scalefeed all replaces output with different interval", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 500, 20, [9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4]); +}); + +test("offsetfeed", function() { + feed(1, 1000, 10, [1, NAN, 3]); + empty_feed(2); + assert_success(run("offsetfeed", ["input"=>1, "offset"=>-1, "output"=>2])); + assert_feed(2, 1000, 10, [0, NAN, 2]); +}); + +test("allowpositive", function() { + feed(1, 1000, 10, [-1, 2, NAN, -3]); + empty_feed(2); + assert_success(run("allowpositive", ["input"=>1, "output"=>2])); + assert_feed(2, 1000, 10, [0, 2, NAN, 0]); +}); + +test("allownegative", function() { + feed(1, 1000, 10, [-1, 2, NAN, -3]); + empty_feed(2); + assert_success(run("allownegative", ["input"=>1, "output"=>2])); + assert_feed(2, 1000, 10, [-1, 0, NAN, -3]); +}); + +test("scalefeed from after end of output continues from output end", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + append(1, [3, 4]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"from", "process_start"=>1030])); + assert_feed(2, 1000, 10, [2, 4, 6, 8]); +}); + +test("scalefeed all removes output beyond end of input", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [9, 9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4]); +}); + +test("scalefeed recent keeps output beyond end of input", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [9, 9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [9, 9, 9, 9]); +}); + +test("scalefeed across block boundaries", function() { + $n = POSTPROCESS_BLOCKSIZE * 2 + 3; + $values = range(0, $n - 1); + feed(1, 0, 10, $values); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 0, 10, array_map(function($v) { return $v * 2; }, $values)); + assert_redis(2, ($n - 1) * 10, ($n - 1) * 2); +}); + +test("scalefeed missing output feed", function() { + feed(1, 1000, 10, [1]); + assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>5]), "does not exist"); +}); + +// -------------------------------------------------------------------------------- +// Accumulators +// -------------------------------------------------------------------------------- + +test("powertokwh all", function() { + // 1 hour interval so that W = Wh per datapoint, NAN holds the last power value + feed(1, 3600, 3600, [1000, 1000, NAN, 2000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + assert_feed(2, 3600, 3600, [1, 2, 3, 5]); + assert_redis(2, 3600 * 4, 5); +}); + +test("powertokwh recent continues from last value", function() { + feed(1, 3600, 3600, [1000, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + append(1, [500, 500]); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000, "process_mode"=>"recent"])); + assert_feed(2, 3600, 3600, [1, 2, 2.5, 3]); +}); + +test("powertokwh from", function() { + feed(1, 3600, 3600, [1000, 1000, 1000, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + feed(1, 3600, 3600, [1000, 1000, 2000, 2000]); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000, "process_mode"=>"from", "process_start"=>3600 * 3])); + assert_feed(2, 3600, 3600, [1, 2, 4, 6]); +}); + +test("powertokwh spurious power adds no energy and keeps alignment", function() { + feed(1, 3600, 3600, [1000, 5000000, NAN, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + // NAN holds the last valid power + assert_feed(2, 3600, 3600, [1, 1, 2, 3]); +}); + +test("accumulator spurious value is not added and keeps alignment", function() { + feed(1, 1000, 10, [1, 5000, 2]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + assert_feed(2, 1000, 10, [1, 1, 3]); +}); + +test("accumulator", function() { + feed(1, 1000, 10, [1, 2, NAN, 3]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + assert_feed(2, 1000, 10, [1, 3, 5, 8]); +}); + +// -------------------------------------------------------------------------------- +// Two inputs +// -------------------------------------------------------------------------------- + +test("addfeeds", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + feed(2, 1000, 10, [10, NAN, NAN, 40]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [11, 2, NAN, 44]); +}); + +test("addfeeds different intervals uses longest, averaging shorter", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1000, 20, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 20, [11.5, 23.5]); +}); + +test("addfeeds stops at end of shortest input", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1000, 10, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [11, 22]); + append(2, [30]); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 1000, 10, [11, 22, 33]); +}); + +test("mergefeeds", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + feed(2, 1000, 10, [10, NAN, NAN, 40]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [5.5, 2, NAN, 22]); +}); + +// -------------------------------------------------------------------------------- +// In place +// -------------------------------------------------------------------------------- + +test("removenan interpolates gaps", function() { + feed(1, 1000, 10, [1, NAN, NAN, 4, NAN]); + assert_success(run("removenan", ["feedid"=>1])); + // Trailing NAN has no end value to interpolate to + assert_feed(1, 1000, 10, [1, 2, 3, 4, NAN]); +}); + +test("limit_morethan_lessthan", function() { + feed(1, 1000, 10, [-50, 5, NAN, 50]); + assert_success(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_feed(1, 1000, 10, [-10, 5, NAN, 10]); +}); + +test("in place processes leave redis alone", function() { + global $redis; + feed(1, 1000, 10, [-50, 5]); + assert_success(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_equal(0, count($redis->hashes)); +}); + +test("in place process on empty feed", function() { + empty_feed(1); + assert_error(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10]), "empty"); +}); + +test("in place process ignores process mode", function() { + feed(1, 1000, 10, [-50, 5, 50]); + $result = run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10, "process_mode"=>"recent"]); + assert_success($result); + assert_equal("modified: 2 datapoints, 67%", $result["message"]); + assert_feed(1, 1000, 10, [-10, 5, 10]); +}); + +test("remove_morethan_lessthan", function() { + feed(1, 1000, 10, [-50, 5, NAN, 50]); + assert_success(run("remove_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_feed(1, 1000, 10, [NAN, 5, NAN, NAN]); +}); + +test("to_signed", function() { + feed(1, 1000, 10, [100, 65535, NAN, 32768, 32767]); + assert_success(run("to_signed", ["feedid"=>1])); + assert_feed(1, 1000, 10, [100, -1, NAN, -32768, 32767]); +}); + +test("removenan_lastvalue", function() { + feed(1, 1000, 10, [NAN, 1, NAN, NAN, 4, NAN]); + assert_success(run("removenan_lastvalue", ["feedid"=>1])); + // Leading NAN has no last value + assert_feed(1, 1000, 10, [NAN, 1, 1, 1, 4, 4]); +}); + +test("removenan_lastvalue carries last value across blocks and resets between runs", function() { + global $testdir; + $n = POSTPROCESS_BLOCKSIZE + 2; + $values = array_fill(0, $n, NAN); + $values[POSTPROCESS_BLOCKSIZE - 1] = 7; + feed(1, 0, 10, $values); + feed(2, 0, 10, [NAN, 3]); + + // The runner reuses process objects between runs + $p = new PostProcess_removenan_lastvalue($testdir); + $params = (object) ["process_mode"=>"all", "process_start"=>0]; + $params->feedid = 1; + ob_start(); + assert_success($p->process($params)); + $params->feedid = 2; + assert_success($p->process($params)); + ob_end_clean(); + + $result = values(1); + assert_equal([7, 7, 7], array_slice($result, POSTPROCESS_BLOCKSIZE - 1)); + assert_equal(true, is_nan($result[0])); + // Not carried over from the previous run + assert_equal([NAN, 3], values(2)); +}); + +// -------------------------------------------------------------------------------- +// Backfill +// -------------------------------------------------------------------------------- + +test("backfillfeed appends averaged source data after target end", function() { + // Target 20s with a gap before its end that must be left alone + feed(1, 1000, 20, [1, 2, NAN, 4]); + // Source 10s, not aligned with the target, ends at 1165 + $source = []; + for ($i=0; $i<16; $i++) $source[] = 100 + $i; + $source[10] = NAN; + $source[11] = NAN; + feed(2, 1005, 10, $source); + + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 2, NAN, 4, 108.5, NAN, 112.5, 114.5]); + assert_redis(1, 1140, 114.5); +}); + +test("backfillfeed waits for complete intervals", function() { + feed(1, 1000, 20, [1]); + feed(2, 1000, 10, [1, 2, 3, 4, 5]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 3.5]); + append(2, [6]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2, "process_mode"=>"recent"])); + assert_feed(1, 1000, 20, [1, 3.5, 5.5]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 3.5, 5.5]); +}); + +test("backfillfeed source with longer interval", function() { + feed(1, 1000, 10, [1]); + feed(2, 1000, 20, [5, 6, 7]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 10, [1, 5, 6, 6, 7, 7]); +}); + +test("backfillfeed rejects same feed", function() { + feed(1, 1000, 10, [1]); + assert_error(run("backfillfeed", ["feed"=>1, "source"=>1]), "different"); +}); + +test("backfillfeed rejects empty target", function() { + empty_feed(1, 20); + feed(2, 1000, 10, [1, 2]); + assert_error(run("backfillfeed", ["feed"=>1, "source"=>2]), "empty"); +}); + +test("backfillfeed missing source fails cleanly", function() { + feed(1, 1000, 10, [1]); + $result = (function() { + try { return run("backfillfeed", ["feed"=>1, "source"=>99]); } + catch (PostProcessException $e) { return array("success"=>false, "message"=>$e->getMessage()); } + })(); + assert_error($result, "does not exist"); +}); diff --git a/tests/test_simulators.php b/tests/test_simulators.php new file mode 100644 index 0000000..f1e640b --- /dev/null +++ b/tests/test_simulators.php @@ -0,0 +1,252 @@ +1, "heatpump_flowT"=>2, "heatpump_returnT"=>3, "heatpump_outsideT"=>4, + "condenser_offset"=>4.0, "refrigerant_offset"=>-6.0, + "practical_efficiency_factor"=>0.5, "running_power_threshold"=>100.0, + "heatpump_cop_sim"=>11, "heatpump_heat_sim"=>12, "heatpump_heat_sim_kwh"=>13, "heatpump_flowrate_sim"=>14 + ), $extra); +} + +test("carnot_cop_simulator", function() { + // Tc = 35+4+273 = 312, Tr = 5-6+273 = 272, COP = 0.5 * 312 / 40 = 3.9 + // Second datapoint is below the running power threshold + feed(1, 3600, 3600, [1000, 50]); + feed(2, 3600, 3600, [35, 35]); + feed(3, 3600, 3600, [30, 30]); + feed(4, 3600, 3600, [5, 5]); + foreach ([11, 12, 13, 14] as $id) empty_feed($id); + + assert_success(run("carnot_cop_simulator", carnot_params())); + assert_feed(11, 3600, 3600, [3.9, 0]); + assert_feed(12, 3600, 3600, [3900, 0]); + assert_feed(13, 3600, 3600, [3.9, 3.9]); + // flow rate m3/h = heat / (4150 * dT) * 3.6 + assert_feed(14, 3600, 3600, [3900 / (4150 * 5) * 3.6, 0]); +}); + +test("carnot_cop_simulator continues kwh", function() { + feed(1, 3600, 3600, [1000]); + feed(2, 3600, 3600, [35]); + feed(3, 3600, 3600, [30]); + feed(4, 3600, 3600, [5]); + foreach ([11, 12, 13, 14] as $id) empty_feed($id); + assert_success(run("carnot_cop_simulator", carnot_params())); + append(1, [1000]); + append(2, [35]); + append(3, [30]); + append(4, [5]); + assert_success(run("carnot_cop_simulator", carnot_params())); + assert_feed(13, 3600, 3600, [3.9, 7.8]); +}); + +// -------------------------------------------------------------------------------- +// Battery simulator +// -------------------------------------------------------------------------------- + +function battery_params($extra = array()) +{ + $params = array( + "solar"=>1, "consumption"=>2, + "has_solar"=>1, "has_battery"=>1, "capacity"=>4.4, "max_charge_rate"=>2500.0, "max_discharge_rate"=>2500.0, + "round_trip_efficiency"=>0.9, "timezone"=>"Europe/London", "offpeak_soc_target"=>0, "offpeak_start"=>3, + "discharge_start"=>0, "discharge_end"=>24, "peak_export_enabled"=>0, "peak_export_start"=>16, "peak_export_end"=>19 + ); + $id = 11; + foreach (battery_outputs() as $key) $params[$key] = $id++; + return array_merge($params, $extra); +} + +function battery_outputs() +{ + return array("soc", "power", "charge", "discharge", "charge_kwh", "discharge_kwh", + "grid", "import", "export", "import_kwh", "export_kwh", + "solar_to_load_kwh", "solar_to_grid_kwh", "solar_to_battery_kwh", "battery_to_load_kwh", + "battery_to_grid_kwh", "grid_to_load_kwh", "grid_to_battery_kwh"); +} + +function battery_output_ids() +{ + $ids = array(); + $id = 11; + foreach (battery_outputs() as $key) $ids[$key] = $id++; + return $ids; +} + +test("batterysimulator without battery", function() { + feed(1, 3600, 3600, [0, 3000, 1000]); + feed(2, 3600, 3600, [500, 1000, 1000]); + foreach (battery_output_ids() as $id) empty_feed($id); + assert_success(run("batterysimulator", battery_params(["has_battery"=>0]))); + $ids = battery_output_ids(); + assert_feed($ids["grid"], 3600, 3600, [500, -2000, 0]); + assert_feed($ids["import_kwh"], 3600, 3600, [0.5, 0.5, 0.5]); + assert_feed($ids["export_kwh"], 3600, 3600, [0, 2, 2]); + assert_feed($ids["solar_to_load_kwh"], 3600, 3600, [0, 1, 2]); +}); + +test("batterysimulator charges from excess solar and discharges to load", function() { + // 1 kWh battery, 100% efficient, starts at 50% + feed(1, 3600, 3600, [1000, 0]); + feed(2, 3600, 3600, [0, 1000]); + foreach (battery_output_ids() as $id) empty_feed($id); + assert_success(run("batterysimulator", battery_params(["capacity"=>1, "round_trip_efficiency"=>1]))); + $ids = battery_output_ids(); + // Charge limited by capacity: 0.5 kWh, then discharge 1 kWh + assert_feed($ids["soc"], 3600, 3600, [100, 0]); + assert_feed($ids["charge"], 3600, 3600, [500, 0]); + assert_feed($ids["discharge"], 3600, 3600, [0, 1000]); + assert_feed($ids["grid"], 3600, 3600, [-500, 0]); +}); + +// -------------------------------------------------------------------------------- +// Solar battery kWh flows +// -------------------------------------------------------------------------------- + +function solarbattery_output_ids() +{ + return array("solar_to_load_kwh"=>11, "solar_to_grid_kwh"=>12, "solar_to_battery_kwh"=>13, + "battery_to_load_kwh"=>14, "battery_to_grid_kwh"=>15, "grid_to_load_kwh"=>16, "grid_to_battery_kwh"=>17); +} + +function solarbattery_params($extra = array()) +{ + return array_merge(array("solar"=>1, "use"=>2, "grid"=>3, "battery"=>4, "strategy"=>"Solar first"), solarbattery_output_ids(), $extra); +} + +// -------------------------------------------------------------------------------- +// Incremental runs give the same output as a single run +// -------------------------------------------------------------------------------- + +// Synthetic input data from 2024-03-25 for $n steps of $interval, includes the change to +// summer time on 2024-03-31 and gaps of missing data +function sim_inputs($n, $interval = 10, $offset = 0) +{ + $start = 1711324800 + $offset; + $data = array("solar"=>[], "use"=>[], "flowT"=>[], "returnT"=>[], "outsideT"=>[], "elec"=>[], "battery"=>[], "grid"=>[]); + for ($i=0; $i<$n; $i++) { + $hour = (($start + $i * $interval) % 86400) / 3600; + $solar = max(0, 3000 * sin(($hour - 6) / 12 * M_PI)); + $use = 250 + (($i * 7919) % 1500) * (($i % 37) < 5 ? 1 : 0.1); + $battery = $solar > $use ? -min(2000, $solar - $use) : min(1500, $use - $solar) * 0.5; + $data["solar"][] = $solar; + $data["use"][] = $use; + $data["battery"][] = $battery; + $data["grid"][] = $use - $solar - $battery; + $data["elec"][] = ($i % 300) < 200 ? 800 + ($i % 50) * 10 : 20; + $data["flowT"][] = 35 + ($i % 120) / 20; + $data["returnT"][] = 31 + ($i % 120) / 25; + $data["outsideT"][] = 5 + 4 * sin($hour / 24 * 2 * M_PI); + } + // Gaps of missing data + foreach ($data as $key => $values) { + for ($i=1000; $i<1050 && $i<$n; $i++) $data[$key][$i] = NAN; + } + return array($start, $interval, $data); +} + +// Write the inputs up to $n datapoints into feeds $ids (key => feed id) +function sim_feeds($inputs, $ids, $n) +{ + list($start, $interval, $data) = $inputs; + foreach ($ids as $key => $id) feed($id, $start, $interval, array_slice($data[$key], 0, $n)); +} + +// Run $process on all data at once and in parts of $parts datapoints, compare the outputs +function assert_incremental($process, $params, $input_ids, $output_ids, $n, $parts, $offset = 0) +{ + $inputs = sim_inputs($n, 10, $offset); + + sim_feeds($inputs, $input_ids, $n); + foreach ($output_ids as $id) empty_feed($id); + assert_success(run($process, $params)); + $single = array(); + foreach ($output_ids as $key => $id) $single[$key] = values($id); + + foreach ($output_ids as $id) empty_feed($id); + foreach ($parts as $part) { + sim_feeds($inputs, $input_ids, $part); + assert_success(run($process, $params)); + } + foreach ($output_ids as $key => $id) { + assert_equal($single[$key], values($id), "$key"); + } +} + +test("carnot_cop_simulator incremental", function() { + assert_incremental("carnot_cop_simulator", carnot_params(), + ["elec"=>1, "flowT"=>2, "returnT"=>3, "outsideT"=>4], + ["cop"=>11, "heat"=>12, "kwh"=>13, "flowrate"=>14], 3000, [1000, 1001, 2500, 3000]); +}); + +test("batterysimulator incremental", function() { + assert_incremental("batterysimulator", battery_params(), + ["solar"=>1, "use"=>2], battery_output_ids(), 3000, [1000, 1001, 2500, 3000]); +}); + +test("batterysimulator incremental with run boundaries during off-peak charging", function() { + // Off-peak window starts at 02:00, datapoint 720 each day, 8640 datapoints per day + assert_incremental("batterysimulator", battery_params([ + "discharge_start"=>7, "discharge_end"=>22, "offpeak_soc_target"=>80, "offpeak_start"=>2, + "peak_export_enabled"=>1, "peak_export_start"=>16, "peak_export_end"=>19 + ]), ["solar"=>1, "use"=>2], battery_output_ids(), 20000, [720, 721, 800, 9400, 9401, 9500, 20000]); +}); + +test("solarbatterykwh incremental", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 20000, [5000, 5001, 15000, 20000]); +}); + +test("solarbatterykwh incremental, inputs not starting on a slot", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 20000, [3000, 15000, 20000], 130); +}); + +test("solarbatterykwh incremental, short runs within the backcast", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 6000, [100, 2000, 2001, 4000, 6000]); +}); + +// -------------------------------------------------------------------------------- +// Snapshots of single runs over 100k datapoints, more than one block, includes the +// change to summer time +// -------------------------------------------------------------------------------- + +function snapshot_run($name, $process, $params, $input_ids, $output_ids) +{ + sim_feeds(sim_inputs(100000), $input_ids, 100000); + foreach ($output_ids as $id) empty_feed($id); + assert_success(run($process, $params)); + assert_snapshot($name, $output_ids); +} + +test("batterysimulator snapshot", function() { + snapshot_run("batterysimulator", "batterysimulator", battery_params(["discharge_start"=>7, "discharge_end"=>22]), + ["solar"=>1, "use"=>2], battery_output_ids()); +}); + +test("batterysimulator snapshot offpeak and peak export", function() { + snapshot_run("batterysimulator_offpeak", "batterysimulator", battery_params([ + "discharge_start"=>7, "discharge_end"=>22, "offpeak_soc_target"=>80, "offpeak_start"=>2, + "peak_export_enabled"=>1, "peak_export_start"=>16, "peak_export_end"=>19 + ]), ["solar"=>1, "use"=>2], battery_output_ids()); +}); + +test("solarbatterykwh snapshot solar first", function() { + snapshot_run("solarbatterykwh", "solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids()); +}); + +test("solarbatterykwh snapshot battery first derived grid", function() { + $params = solarbattery_params(["strategy"=>"Battery first"]); + unset($params["grid"]); + snapshot_run("solarbatterykwh_derived", "solarbatterykwh", $params, + ["solar"=>1, "use"=>2, "battery"=>4], solarbattery_output_ids()); +});