From ca9fbb2e335f7c69389b948450423782aea4a983 Mon Sep 17 00:00:00 2001 From: Trystan Lea Date: Fri, 9 Oct 2026 23:03:13 +0100 Subject: [PATCH 1/5] Add test harness and block based FeedReader/FeedWriter - tests/run.php runs process tests against synthetic feeds, no database or redis required, with tests for existing processes and common code - FeedReader/FeedWriter in common.php read and write feeds in blocks, check writes, resample to a common timeline and update redis - resolve_start_time() implements recent/from/all in one place - Runner reports exceptions thrown by a process as a failed process - Fix interval setting validation using $processitem->key - backfillfeed uses FeedReader/FeedWriter, rejects an empty target Co-Authored-By: Claude Opus 5.5 --- common.php | 293 ++++++++++++++++++++++++++++++++++++- postprocess_run.php | 6 +- processes/backfillfeed.php | 80 ++-------- readme.md | 10 +- tests/run.php | 186 +++++++++++++++++++++++ tests/test_common.php | 169 +++++++++++++++++++++ tests/test_processes.php | 220 ++++++++++++++++++++++++++++ 7 files changed, 890 insertions(+), 74 deletions(-) create mode 100644 tests/run.php create mode 100644 tests/test_common.php create mode 100644 tests/test_processes.php diff --git a/common.php b/common.php index 5f6e79a..7ce6b91 100644 --- a/common.php +++ b/common.php @@ -64,10 +64,10 @@ public function validate($processitem) if ($setting['type']=="interval") { // check if interval is valid - if (is_numeric($processitem->key)) { - $processitem->key = (int) $processitem->key; + if (is_numeric($processitem->{$key})) { + $processitem->{$key} = (int) $processitem->{$key}; } - if (!in_array($processitem->key, array('original',10,30,60,1800,3600))) { + if (!in_array($processitem->{$key}, array('original',10,30,60,1800,3600), true)) { return array("success"=>false,"message"=>"invalid interval"); } } @@ -119,12 +119,289 @@ function getmeta($dir, $id) function createmeta($dir, $id, $meta) { - $metafile = fopen($dir . $id . ".meta", 'wb'); - fwrite($metafile, pack("I", 0)); - fwrite($metafile, pack("I", 0)); - fwrite($metafile, pack("I", $meta->interval)); - fwrite($metafile, pack("I", $meta->start_time)); + if (!$metafile = @fopen($dir . $id . ".meta", 'wb')) return false; + $bytes = pack("IIII", 0, 0, $meta->interval, $meta->start_time); + $written = fwrite($metafile, $bytes); fclose($metafile); + return $written === strlen($bytes); +} + +// -------------------------------------------------------------------------------- +// Block based feed access +// +// FeedReader and FeedWriter read and write feeds in blocks rather than one datapoint +// at a time, which is several times faster in PHP. Errors are thrown as +// PostProcessException, which the runner reports as a failed process. +// -------------------------------------------------------------------------------- + +class PostProcessException extends Exception {} + +// Number of datapoints to read or write at a time +const POSTPROCESS_BLOCKSIZE = 65536; + +// Start time of this run of a process for the output feed meta (start_time, interval, npoints) +// recent: from the end of the output feed +// from: from process_start, aligned to the output interval, no later than the end of the output feed +// all: from the start of the output feed +function resolve_start_time($params, $meta) +{ + $end_time = $meta->start_time + ($meta->npoints * $meta->interval); + $mode = isset($params->process_mode) ? $params->process_mode : 'recent'; + + if ($mode == 'recent') { + return $end_time; + } + if ($mode == 'from') { + $start_time = max((int) $params->process_start, $meta->start_time); + $start_time = $meta->start_time + floor(($start_time - $meta->start_time) / $meta->interval) * $meta->interval; + return min($start_time, $end_time); + } + return $meta->start_time; +} + +class FeedReader +{ + public $id; + public $meta; + private $fh; + + public function __construct($dir, $id) + { + $this->id = (int) $id; + if (!file_exists($dir.$this->id.".meta") || !file_exists($dir.$this->id.".dat")) { + throw new PostProcessException("feed ".$this->id." does not exist"); + } + $this->meta = getmeta($dir, $this->id); + $this->meta->npoints = (int) $this->meta->npoints; + if (!$this->meta->interval) { + throw new PostProcessException("feed ".$this->id." has no interval"); + } + if (!$this->fh = @fopen($dir.$this->id.".dat", 'rb')) { + throw new PostProcessException("could not open feed ".$this->id); + } + } + + public function __destruct() + { + if ($this->fh) fclose($this->fh); + } + + // Read $n datapoints starting at position $pos, positions outside of the feed are NAN + public function read($pos, $n) + { + $from = max($pos, 0); + $to = min($pos + $n, $this->meta->npoints); + + if ($from == $pos && $to == $pos + $n) { + fseek($this->fh, $from * 4); + $values = array_values(unpack("f*", fread($this->fh, $n * 4))); + if (count($values) == $n) return $values; + } + + $values = array_fill(0, $n, NAN); + if ($to > $from) { + fseek($this->fh, $from * 4); + $i = $from - $pos; + foreach (unpack("f*", fread($this->fh, ($to - $from) * 4)) as $value) { + $values[$i++] = $value; + } + } + return $values; + } + + // Read $n values on the timeline $start_time + i * $interval, which may differ from + // the feed's own start time and interval. Each value covers [time, time + interval): + // - feed interval shorter: mean of the datapoints within the interval, ignoring NAN + // - feed interval longer: the datapoint covering the start of the interval + public function read_time($start_time, $interval, $n) + { + $feed_start = $this->meta->start_time; + $feed_interval = $this->meta->interval; + + if ($feed_interval == $interval && ($start_time - $feed_start) % $feed_interval == 0) { + return $this->read(intdiv($start_time - $feed_start, $feed_interval), $n); + } + + $block_start = (int) floor(($start_time - $feed_start) / $feed_interval); + $block_end = (int) ceil(($start_time + $n * $interval - $feed_start) / $feed_interval); + $data = $this->read($block_start, $block_end - $block_start); + + $values = array(); + for ($i=0; $i<$n; $i++) { + $time = $start_time + $i * $interval; + $pos_start = (int) ceil(($time - $feed_start) / $feed_interval); + $pos_end = (int) ceil(($time + $interval - $feed_start) / $feed_interval); + if ($pos_end <= $pos_start) { + $pos_start = (int) floor(($time - $feed_start) / $feed_interval); + $pos_end = $pos_start + 1; + } + $sum = 0; + $count = 0; + for ($pos=$pos_start; $pos<$pos_end; $pos++) { + $value = $data[$pos - $block_start]; + if (!is_nan($value)) { + $sum += $value; + $count++; + } + } + $values[] = $count ? $sum / $count : NAN; + } + return $values; + } +} + +class FeedWriter +{ + public $id; + public $meta; + private $dir; + private $fh; + private $buffer = ""; + private $pos = 0; + private $meta_pending = false; + private $bytes_written = 0; + private $last_time = false; + private $last_value = NAN; + + public function __construct($dir, $id) + { + $this->dir = $dir; + $this->id = (int) $id; + if (!file_exists($dir.$this->id.".meta")) { + throw new PostProcessException("feed ".$this->id." does not exist"); + } + if (!$this->fh = @fopen($dir.$this->id.".dat", 'c+')) { + throw new PostProcessException("could not open feed ".$this->id." for writing"); + } + $this->meta = getmeta($dir, $this->id); + $this->meta->npoints = (int) $this->meta->npoints; + } + + public function __destruct() + { + if ($this->fh) fclose($this->fh); + } + + // Set the start time and interval of an output feed. An empty feed takes them (written + // with the first data), a feed with data must already match unless $reset is set, in + // which case existing data is discarded. + public function init($start_time, $interval, $reset = false) + { + if ($this->meta->npoints > 0 && !$reset) { + if ($this->meta->start_time != $start_time) { + throw new PostProcessException("feed ".$this->id." start time mismatch"); + } + if ($this->meta->interval != $interval) { + throw new PostProcessException("feed ".$this->id." interval mismatch"); + } + return; + } + if ($this->meta->npoints > 0) { + if (!ftruncate($this->fh, 0)) throw new PostProcessException("could not truncate feed ".$this->id); + } + $this->meta->start_time = $start_time; + $this->meta->interval = $interval; + $this->meta->npoints = 0; + $this->meta->end_time = $start_time; + $this->pos = 0; + $this->meta_pending = true; + } + + // Move the write position to $time, a time beyond the end of the feed is padded with NAN + public function seek($time) + { + $offset = $time - $this->meta->start_time; + if (!$this->meta->interval || $offset < 0 || $offset % $this->meta->interval != 0) { + throw new PostProcessException("feed ".$this->id." invalid write time $time"); + } + $this->flush(); + $pos = intdiv($offset, $this->meta->interval); + if ($pos > $this->meta->npoints) { + fseek($this->fh, $this->meta->npoints * 4); + $this->pos = $this->meta->npoints; + $this->write_block(array_fill(0, $pos - $this->meta->npoints, NAN)); + } else { + fseek($this->fh, $pos * 4); + $this->pos = $pos; + } + } + + public function write($value) + { + $this->buffer .= pack("f", $value); + if (!is_nan($value)) { + $this->last_time = $this->meta->start_time + $this->pos * $this->meta->interval; + $this->last_value = $value; + } + $this->pos++; + if (strlen($this->buffer) >= POSTPROCESS_BLOCKSIZE * 4) $this->flush(); + } + + public function write_block($values) + { + $n = count($values); + if (!$n) return; + $this->buffer .= pack("f*", ...$values); + for ($i=$n-1; $i>=0; $i--) { + if (!is_nan($values[$i])) { + $this->last_time = $this->meta->start_time + ($this->pos + $i) * $this->meta->interval; + $this->last_value = $values[$i]; + break; + } + } + $this->pos += $n; + if (strlen($this->buffer) >= POSTPROCESS_BLOCKSIZE * 4) $this->flush(); + } + + // Time of the next datapoint to be written + public function time() + { + return $this->meta->start_time + $this->pos * $this->meta->interval; + } + + public function flush() + { + if ($this->buffer === "") return; + if ($this->meta_pending) { + if (!createmeta($this->dir, $this->id, $this->meta)) { + throw new PostProcessException("could not write meta for feed ".$this->id); + } + $this->meta_pending = false; + } + $length = strlen($this->buffer); + $written = fwrite($this->fh, $this->buffer); + if ($written !== $length) { + throw new PostProcessException("feed ".$this->id.": wrote $written of $length bytes, disk full?"); + } + $this->bytes_written += $length; + $this->meta->npoints = max($this->meta->npoints, $this->pos); + $this->meta->end_time = $this->meta->start_time + $this->meta->npoints * $this->meta->interval; + $this->buffer = ""; + } + + // Flush remaining data and close. $truncate removes any existing data after the last + // datapoint written, use when reprocessing so that no stale data is left at the end. + // Updates the feed's last time and value in redis. Returns the number of bytes written. + public function close($truncate = false) + { + $this->flush(); + if ($truncate && $this->pos < $this->meta->npoints) { + if (!ftruncate($this->fh, $this->pos * 4)) { + throw new PostProcessException("could not truncate feed ".$this->id); + } + $this->meta->npoints = $this->pos; + $this->meta->end_time = $this->time(); + } + fclose($this->fh); + $this->fh = false; + if ($this->last_time !== false) { + updatetimevalue($this->id, $this->last_time, $this->last_value); + } + return $this->bytes_written; + } + + public function last_time() { return $this->last_time; } + public function last_value() { return $this->last_value; } } //compute meta datas of different feeds intended for a preprocessing diff --git a/postprocess_run.php b/postprocess_run.php index adf8182..f1f2cd2 100644 --- a/postprocess_run.php +++ b/postprocess_run.php @@ -24,7 +24,11 @@ if ($process = $postprocess->pop_process_queue()) { print json_encode($process)."\n"; - $result = $process_classes[$process->params->process]->process($process->params); + try { + $result = $process_classes[$process->params->process]->process($process->params); + } catch (Throwable $e) { + $result = array("success"=>false, "message"=>$e->getMessage()); + } if ($result['success']) { print "Success: ".$result['message']."\n"; diff --git a/processes/backfillfeed.php b/processes/backfillfeed.php index aea63d5..500a788 100644 --- a/processes/backfillfeed.php +++ b/processes/backfillfeed.php @@ -23,82 +23,34 @@ public function process($params) return array("success"=>false, "message"=>"feed and source must be different feeds"); } - $target_meta = getmeta($this->dir,$params->feed); - $source_meta = getmeta($this->dir,$params->source); - $interval = $target_meta->interval; + $target = new FeedWriter($this->dir,$params->feed); + $source = new FeedReader($this->dir,$params->source); + $interval = $target->meta->interval; + + if ($target->meta->npoints==0) { + return array("success"=>false, "message"=>"feed to backfill is empty"); + } // Always start from the current end of the target feed, regardless of process mode, // so that existing data in the target feed is never modified - $start_time = $target_meta->end_time; + $start_time = $target->meta->end_time; // Only write target intervals that are fully covered by the source feed, // so that a partially recorded interval is completed on the next run - $end_time = $start_time + floor(($source_meta->end_time - $start_time) / $interval) * $interval; + $end_time = $start_time + floor(($source->meta->end_time - $start_time) / $interval) * $interval; if ($start_time>=$end_time) { return array("success"=>true, "message"=>"no new data to process"); } - if (!$source_fh = @fopen($this->dir.$params->source.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open source feed"); - } - if (!$target_fh = @fopen($this->dir.$params->feed.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open target feed"); + // Source values are averaged into each target interval + $target->seek($start_time); + while (($time = $target->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $target->write_block($source->read_time($time,$interval,$n)); } - fseek($target_fh, $target_meta->npoints * 4); - - $buffer = ""; - $byteswritten = 0; - $last_time = false; - $last_value = NAN; - - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - // Source datapoints that fall within this target interval - $pos_start = ceil(($time - $source_meta->start_time) / $source_meta->interval); - $pos_end = ceil(($time + $interval - $source_meta->start_time) / $source_meta->interval); - // Source interval longer than target interval: use the datapoint covering this interval - if ($pos_end<=$pos_start) { - $pos_start = floor(($time - $source_meta->start_time) / $source_meta->interval); - $pos_end = $pos_start + 1; - } - if ($pos_start<0) $pos_start = 0; - if ($pos_end>$source_meta->npoints) $pos_end = $source_meta->npoints; - - $sum = 0; - $n = 0; - if ($pos_end>$pos_start) { - fseek($source_fh,$pos_start*4); - $values = unpack("f*",fread($source_fh,($pos_end-$pos_start)*4)); - foreach ($values as $value) { - if (!is_nan($value)) { - $sum += $value; - $n++; - } - } - } + $byteswritten = $target->close(); - $outval = NAN; - if ($n>0) { - $outval = $sum / $n; - $last_time = $time; - $last_value = $outval; - } - $buffer .= pack("f",$outval); - - if (strlen($buffer)>=400000) { - $byteswritten += fwrite($target_fh,$buffer); - $buffer = ""; - } - } - - $byteswritten += fwrite($target_fh,$buffer); - fclose($target_fh); - fclose($source_fh); - - if ($last_time!==false) { - updatetimevalue($params->feed,$last_time,$last_value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$last_time." ".$last_value); + return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$target->last_time()." ".$target->last_value()); } } diff --git a/readme.md b/readme.md index f4f75e7..6994f5b 100644 --- a/readme.md +++ b/readme.md @@ -21,4 +21,12 @@ This is installed by default on the standard emonSD image build. Alternatively to run manually: cd /opt/emoncms/modules/postprocess - sudo php postprocess_run.php \ No newline at end of file + sudo php postprocess_run.php +### Tests + +Processes are tested against small synthetic feeds in a temporary directory, no database or redis is required: + + php tests/run.php # all tests + php tests/run.php scalefeed # tests whose name contains scalefeed + +New processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` in `common.php` for block based feed access, see `processes/backfillfeed.php` for an example. diff --git a/tests/run.php b/tests/run.php new file mode 100644 index 0000000..e1856b5 --- /dev/null +++ b/tests/run.php @@ -0,0 +1,186 @@ +hashes[$key] = $values; return true; } +} +$redis = new TestRedis(); + +$tests = array(); +function test($name, $fn) { global $tests; $tests[$name] = $fn; } + +// -------------------------------------------------------------------------------- +// Feed helpers +// -------------------------------------------------------------------------------- + +$testdir = sys_get_temp_dir()."/postprocess-tests-".getmypid()."/"; + +// Create a feed with the given start time, interval and values +function feed($id, $start_time, $interval, $values = array()) +{ + global $testdir; + $meta = new stdClass(); + $meta->start_time = $start_time; + $meta->interval = $interval; + createmeta($testdir, $id, $meta); + file_put_contents($testdir.$id.".dat", count($values) ? pack("f*", ...$values) : ""); +} + +// Create an empty feed as the feed list does when a new output feed is created +function empty_feed($id, $interval = 10) +{ + feed($id, 0, $interval); +} + +// Append values to an existing feed +function append($id, $values) +{ + global $testdir; + file_put_contents($testdir.$id.".dat", pack("f*", ...$values), FILE_APPEND); +} + +function values($id) +{ + global $testdir; + $bytes = file_get_contents($testdir.$id.".dat"); + return strlen($bytes) ? array_values(unpack("f*", $bytes)) : array(); +} + +function meta($id) +{ + global $testdir; + return getmeta($testdir, $id); +} + +// Run a process, default mode is reprocess all. Output printed by the process is captured +// and only shown if the test fails +function run($process, $params = array()) +{ + global $testdir, $captured; + $params = (object) array_merge(array("process_mode"=>"all", "process_start"=>0), $params); + $class = "PostProcess_".$process; + $p = new $class($testdir); + ob_start(); + try { + $result = $p->process($params); + } finally { + $captured .= ob_get_clean(); + } + return $result; +} + +// -------------------------------------------------------------------------------- +// Assertions +// -------------------------------------------------------------------------------- + +class TestFailure extends Exception {} + +function fmt($value) +{ + if (is_array($value)) return "[".implode(", ", array_map('fmt', $value))."]"; + if (is_float($value) && is_nan($value)) return "NAN"; + return var_export($value, true); +} + +function same_value($expected, $actual) +{ + if (is_float($expected) && is_nan($expected)) return is_float($actual) && is_nan($actual); + if (is_float($actual) && is_nan($actual)) return false; + if (is_numeric($expected) && is_numeric($actual)) { + // Feed values are stored as 32 bit floats + return abs($expected - $actual) <= 1e-4 * max(1, abs($expected)); + } + return $expected === $actual; +} + +function assert_equal($expected, $actual, $message = "") +{ + $ok = is_array($expected) + ? is_array($actual) && count($expected) == count($actual) + && count(array_filter(array_map('same_value', $expected, $actual))) == count($expected) + : same_value($expected, $actual); + if (!$ok) throw new TestFailure(($message ? "$message: " : "")."expected ".fmt($expected).", got ".fmt($actual)); +} + +function assert_feed($id, $start_time, $interval, $values) +{ + $meta = meta($id); + assert_equal($start_time, $meta->start_time, "feed $id start time"); + assert_equal($interval, $meta->interval, "feed $id interval"); + assert_equal($values, values($id), "feed $id values"); +} + +function assert_success($result) +{ + if (!$result["success"]) throw new TestFailure("process failed: ".$result["message"]); +} + +function assert_error($result, $contains = "") +{ + if ($result["success"]) throw new TestFailure("expected process to fail, got: ".$result["message"]); + if ($contains && strpos($result["message"], $contains) === false) { + throw new TestFailure("expected error containing '$contains', got: ".$result["message"]); + } +} + +function assert_redis($id, $time, $value) +{ + global $redis; + if (!isset($redis->hashes["feed:$id"])) throw new TestFailure("redis feed:$id not set"); + assert_equal($time, $redis->hashes["feed:$id"]["time"], "redis feed:$id time"); + assert_equal($value, $redis->hashes["feed:$id"]["value"], "redis feed:$id value"); +} + +// -------------------------------------------------------------------------------- +// Run +// -------------------------------------------------------------------------------- + +foreach (glob(__DIR__."/test_*.php") as $file) require $file; + +$filter = $argv[1] ?? ""; +$passed = 0; +$failed = 0; + +foreach ($tests as $name => $fn) { + if ($filter && strpos($name, $filter) === false) continue; + + // Fresh data directory and redis for every test + if (is_dir($testdir)) array_map('unlink', glob($testdir."*")); + else mkdir($testdir); + $redis = new TestRedis(); + $captured = ""; + + try { + $fn(); + $passed++; + echo " ok $name\n"; + } catch (Throwable $e) { + $failed++; + echo " FAIL $name\n"; + $where = ($e instanceof TestFailure) ? "" : " (".get_class($e)." at ".basename($e->getFile()).":".$e->getLine().")"; + echo " ".$e->getMessage()."$where\n"; + if (trim($captured) !== "") echo " output: ".str_replace("\n", "\n ", trim($captured))."\n"; + } +} + +array_map('unlink', glob($testdir."*")); +@rmdir($testdir); + +echo "\n$passed passed, $failed failed\n"; +exit($failed ? 1 : 0); diff --git a/tests/test_common.php b/tests/test_common.php new file mode 100644 index 0000000..98446ec --- /dev/null +++ b/tests/test_common.php @@ -0,0 +1,169 @@ +$start_time, "interval"=>$interval, "npoints"=>$npoints); +} + +test("resolve_start_time", function() { + $meta = meta_of(1000, 10, 5); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"all"], $meta), "all"); + assert_equal(1050, resolve_start_time((object) ["process_mode"=>"recent"], $meta), "recent"); + assert_equal(1050, resolve_start_time((object) [], $meta), "default is recent"); + assert_equal(1020, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>1025], $meta), "from aligns down"); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>0], $meta), "from before start"); + assert_equal(1050, resolve_start_time((object) ["process_mode"=>"from", "process_start"=>9999], $meta), "from after end"); + assert_equal(1000, resolve_start_time((object) ["process_mode"=>"recent"], meta_of(1000, 10, 0)), "recent empty"); +}); + +// -------------------------------------------------------------------------------- +// FeedReader +// -------------------------------------------------------------------------------- + +test("FeedReader read", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, NAN, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal(4, $feed->meta->npoints); + assert_equal([1, 2, NAN, 4], $feed->read(0, 4)); + assert_equal([2, NAN], $feed->read(1, 2)); + // Outside of the feed + assert_equal([NAN, NAN, 1, 2], $feed->read(-2, 4)); + assert_equal([4, NAN, NAN], $feed->read(3, 3)); + assert_equal([NAN, NAN], $feed->read(10, 2)); +}); + +test("FeedReader missing feed", function() { + global $testdir; + try { + new FeedReader($testdir, 99); + } catch (PostProcessException $e) { + return; + } + throw new TestFailure("expected PostProcessException"); +}); + +test("FeedReader read_time same interval", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, 3, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal([2, 3], $feed->read_time(1010, 10, 2)); + assert_equal([NAN, 1], $feed->read_time(990, 10, 2)); +}); + +test("FeedReader read_time same interval not aligned", function() { + global $testdir; + // Datapoints at 1005, 1015, ... each interval takes the datapoint within it + feed(1, 1005, 10, [1, 2, 3, 4]); + $feed = new FeedReader($testdir, 1); + assert_equal([1, 2, 3, 4, NAN], $feed->read_time(1000, 10, 5)); +}); + +test("FeedReader read_time downsample averages ignoring NAN", function() { + global $testdir; + feed(1, 1000, 10, [1, 2, 3, NAN, NAN, NAN, 7]); + $feed = new FeedReader($testdir, 1); + assert_equal([1.5, 3, NAN, 7], $feed->read_time(1000, 20, 4)); +}); + +test("FeedReader read_time upsample holds value", function() { + global $testdir; + feed(1, 1000, 30, [1, 2]); + $feed = new FeedReader($testdir, 1); + assert_equal([NAN, 1, 1, 1, 2, 2, 2, NAN], $feed->read_time(990, 10, 8)); +}); + +// -------------------------------------------------------------------------------- +// FeedWriter +// -------------------------------------------------------------------------------- + +test("FeedWriter new feed writes meta with data", function() { + global $testdir; + empty_feed(2); + $out = new FeedWriter($testdir, 2); + $out->init(1000, 10); + $out->seek(1000); + // Meta is not written until there is data + assert_equal(0, meta(2)->start_time); + $out->write(1); + $out->write_block([2, NAN]); + assert_equal(12, $out->close()); + assert_feed(2, 1000, 10, [1, 2, NAN]); + // Last non NAN value + assert_redis(2, 1010, 2); +}); + +test("FeedWriter init on feed with data", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3]); + $out = new FeedWriter($testdir, 2); + $out->init(1000, 10); + try { + $out->init(1000, 20); + throw new TestFailure("expected interval mismatch"); + } catch (PostProcessException $e) { + assert_equal(true, strpos($e->getMessage(), "interval mismatch") !== false); + } + $out->init(2000, 20, true); + $out->seek(2000); + $out->write(5); + $out->close(); + assert_feed(2, 2000, 20, [5]); +}); + +test("FeedWriter overwrite in place", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3, 4]); + $out = new FeedWriter($testdir, 2); + $out->seek(1010); + $out->write_block([20, 30]); + $out->close(); + assert_feed(2, 1000, 10, [1, 20, 30, 4]); +}); + +test("FeedWriter truncate removes stale data", function() { + global $testdir; + feed(2, 1000, 10, [1, 2, 3, 4]); + $out = new FeedWriter($testdir, 2); + $out->seek(1010); + $out->write(20); + $out->close(true); + assert_feed(2, 1000, 10, [1, 20]); +}); + +test("FeedWriter seek beyond end pads with NAN", function() { + global $testdir; + feed(2, 1000, 10, [1]); + $out = new FeedWriter($testdir, 2); + $out->seek(1030); + $out->write(4); + $out->close(); + assert_feed(2, 1000, 10, [1, NAN, NAN, 4]); +}); + +test("FeedWriter rejects unaligned seek", function() { + global $testdir; + feed(2, 1000, 10, [1]); + $out = new FeedWriter($testdir, 2); + foreach ([1005, 990] as $time) { + try { + $out->seek($time); + throw new TestFailure("expected invalid write time for $time"); + } catch (PostProcessException $e) {} + } +}); + +test("FeedWriter flushes large writes in blocks", function() { + global $testdir; + empty_feed(2); + $n = POSTPROCESS_BLOCKSIZE * 2 + 10; + $out = new FeedWriter($testdir, 2); + $out->init(0, 10); + $out->seek(0); + for ($i=0; $i<$n; $i++) $out->write($i); + $out->close(); + $values = values(2); + assert_equal($n, count($values)); + assert_equal($n - 1, $values[$n - 1]); +}); diff --git a/tests/test_processes.php b/tests/test_processes.php new file mode 100644 index 0000000..997e037 --- /dev/null +++ b/tests/test_processes.php @@ -0,0 +1,220 @@ +1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4, NAN, 8]); + assert_redis(2, 1040, 8); +}); + +test("scalefeed recent appends new data only", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + // Changes to existing input data are not picked up in recent mode + feed(1, 1000, 10, [100, 100, 100, 100, 5, 6]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [2, 4, NAN, 8, 10, 12]); +}); + +test("scalefeed recent up to date", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [2, 4]); +}); + +test("scalefeed from", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + feed(1, 1000, 10, [10, 20, 30, 40]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"from", "process_start"=>1020])); + assert_feed(2, 1000, 10, [2, 4, 60, 80]); +}); + +test("scalefeed rejects interval mismatch", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 20, [1]); + assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2]), "interval mismatch"); +}); + +test("offsetfeed", function() { + feed(1, 1000, 10, [1, NAN, 3]); + empty_feed(2); + assert_success(run("offsetfeed", ["input"=>1, "offset"=>-1, "output"=>2])); + assert_feed(2, 1000, 10, [0, NAN, 2]); +}); + +test("allowpositive", function() { + feed(1, 1000, 10, [-1, 2, NAN, -3]); + empty_feed(2); + assert_success(run("allowpositive", ["input"=>1, "output"=>2])); + assert_feed(2, 1000, 10, [0, 2, NAN, 0]); +}); + +test("allownegative", function() { + feed(1, 1000, 10, [-1, 2, NAN, -3]); + empty_feed(2); + assert_success(run("allownegative", ["input"=>1, "output"=>2])); + assert_feed(2, 1000, 10, [-1, 0, NAN, -3]); +}); + +// -------------------------------------------------------------------------------- +// Accumulators +// -------------------------------------------------------------------------------- + +test("powertokwh all", function() { + // 1 hour interval so that W = Wh per datapoint, NAN holds the last power value + feed(1, 3600, 3600, [1000, 1000, NAN, 2000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + assert_feed(2, 3600, 3600, [1, 2, 3, 5]); + assert_redis(2, 3600 * 5, 5); +}); + +test("powertokwh recent continues from last value", function() { + feed(1, 3600, 3600, [1000, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + append(1, [500, 500]); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000, "process_mode"=>"recent"])); + assert_feed(2, 3600, 3600, [1, 2, 2.5, 3]); +}); + +test("powertokwh from", function() { + feed(1, 3600, 3600, [1000, 1000, 1000, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + feed(1, 3600, 3600, [1000, 1000, 2000, 2000]); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000, "process_mode"=>"from", "process_start"=>3600 * 3])); + assert_feed(2, 3600, 3600, [1, 2, 4, 6]); +}); + +test("accumulator", function() { + feed(1, 1000, 10, [1, 2, NAN, 3]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + assert_feed(2, 1000, 10, [1, 3, 5, 8]); +}); + +// -------------------------------------------------------------------------------- +// Two inputs +// -------------------------------------------------------------------------------- + +test("addfeeds", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + feed(2, 1000, 10, [10, NAN, NAN, 40]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [11, 2, NAN, 44]); +}); + +test("addfeeds different intervals uses longest", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1000, 20, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 20, [11, 23]); +}); + +test("addfeeds stops at end of shortest input", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1000, 10, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [11, 22]); + append(2, [30]); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 1000, 10, [11, 22, 33]); +}); + +test("mergefeeds", function() { + feed(1, 1000, 10, [1, 2, NAN, 4]); + feed(2, 1000, 10, [10, NAN, NAN, 40]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1000, 10, [5.5, 2, NAN, 22]); +}); + +// -------------------------------------------------------------------------------- +// In place +// -------------------------------------------------------------------------------- + +test("removenan interpolates gaps", function() { + feed(1, 1000, 10, [1, NAN, NAN, 4, NAN]); + assert_success(run("removenan", ["feedid"=>1])); + // Trailing NAN has no end value to interpolate to + assert_feed(1, 1000, 10, [1, 2, 3, 4, NAN]); +}); + +test("limit_morethan_lessthan", function() { + feed(1, 1000, 10, [-50, 5, NAN, 50]); + assert_success(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_feed(1, 1000, 10, [-10, 5, NAN, 10]); +}); + +// -------------------------------------------------------------------------------- +// Backfill +// -------------------------------------------------------------------------------- + +test("backfillfeed appends averaged source data after target end", function() { + // Target 20s with a gap before its end that must be left alone + feed(1, 1000, 20, [1, 2, NAN, 4]); + // Source 10s, not aligned with the target, ends at 1165 + $source = []; + for ($i=0; $i<16; $i++) $source[] = 100 + $i; + $source[10] = NAN; + $source[11] = NAN; + feed(2, 1005, 10, $source); + + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 2, NAN, 4, 108.5, NAN, 112.5, 114.5]); + assert_redis(1, 1140, 114.5); +}); + +test("backfillfeed waits for complete intervals", function() { + feed(1, 1000, 20, [1]); + feed(2, 1000, 10, [1, 2, 3, 4, 5]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 3.5]); + append(2, [6]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2, "process_mode"=>"recent"])); + assert_feed(1, 1000, 20, [1, 3.5, 5.5]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 20, [1, 3.5, 5.5]); +}); + +test("backfillfeed source with longer interval", function() { + feed(1, 1000, 10, [1]); + feed(2, 1000, 20, [5, 6, 7]); + assert_success(run("backfillfeed", ["feed"=>1, "source"=>2])); + assert_feed(1, 1000, 10, [1, 5, 6, 6, 7, 7]); +}); + +test("backfillfeed rejects same feed", function() { + feed(1, 1000, 10, [1]); + assert_error(run("backfillfeed", ["feed"=>1, "source"=>1]), "different"); +}); + +test("backfillfeed rejects empty target", function() { + empty_feed(1, 20); + feed(2, 1000, 10, [1, 2]); + assert_error(run("backfillfeed", ["feed"=>1, "source"=>2]), "empty"); +}); + +test("backfillfeed missing source fails cleanly", function() { + feed(1, 1000, 10, [1]); + $result = (function() { + try { return run("backfillfeed", ["feed"=>1, "source"=>99]); } + catch (PostProcessException $e) { return array("success"=>false, "message"=>$e->getMessage()); } + })(); + assert_error($result, "does not exist"); +}); From 763816ee7cb88fc879ad3776f958a7a0a255ef19 Mon Sep 17 00:00:00 2001 From: Trystan Lea Date: Fri, 9 Oct 2026 23:10:19 +0100 Subject: [PATCH 2/5] Add MapProcess and InPlaceProcess base classes Simple value by value processes now only implement value($value): - MapProcess: scalefeed, offsetfeed, allowpositive, allownegative - InPlaceProcess: to_signed, limit_morethan_lessthan, remove_morethan_lessthan, removenan_lastvalue Fixes: - from mode past the end of the output left a 0.0 gap - reprocess all left stale data beyond the end of the input - redis time is the last datapoint, NAN is never set as last value - to_signed always reported 0 datapoints modified - remove_morethan_lessthan setting label typos Co-Authored-By: Claude Opus 5.5 --- common.php | 117 ++++++++++++++++++++++++- processes/allownegative.php | 81 +---------------- processes/allowpositive.php | 81 +---------------- processes/limit_morethan_lessthan.php | 57 ++---------- processes/offsetfeed.php | 82 +---------------- processes/remove_morethan_lessthan.php | 64 ++------------ processes/removenan_lastvalue.php | 63 +++---------- processes/scalefeed.php | 80 +---------------- processes/to_signed.php | 57 ++---------- readme.md | 2 +- tests/test_processes.php | 105 +++++++++++++++++++++- 11 files changed, 268 insertions(+), 521 deletions(-) diff --git a/common.php b/common.php index 7ce6b91..a58c650 100644 --- a/common.php +++ b/common.php @@ -262,6 +262,8 @@ class FeedWriter private $bytes_written = 0; private $last_time = false; private $last_value = NAN; + // Set false when rewriting part of a feed, so that the feed's last time and value are left alone + public $update_redis = true; public function __construct($dir, $id) { @@ -394,7 +396,7 @@ public function close($truncate = false) } fclose($this->fh); $this->fh = false; - if ($this->last_time !== false) { + if ($this->update_redis && $this->last_time !== false) { updatetimevalue($this->id, $this->last_time, $this->last_value); } return $this->bytes_written; @@ -404,6 +406,119 @@ public function last_time() { return $this->last_time; } public function last_value() { return $this->last_value; } } +// -------------------------------------------------------------------------------- +// Base classes for processes that transform each value of a feed +// +// Subclasses implement value($value), called for each datapoint that is not NAN, or +// values($values) to handle a block of datapoints at a time, e.g. to keep state between +// datapoints. reset() is called at the start of each run to reset any such state. +// Process settings are available as $this->params. +// -------------------------------------------------------------------------------- + +abstract class ValueProcess extends PostProcess_common +{ + protected $params; + + protected function reset() {} + + protected function value($value) + { + return $value; + } + + protected function values($values) + { + foreach ($values as $i => $value) { + if (!is_nan($value)) $values[$i] = $this->value($value); + } + return $values; + } +} + +// Writes the transformed values of feed setting "input" to feed setting "output", with the +// same start time and interval. Supports recent, from and all process modes. +abstract class MapProcess extends ValueProcess +{ + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $input = new FeedReader($this->dir, $params->input); + $output = new FeedWriter($this->dir, $params->output); + $interval = $input->meta->interval; + $output->init($input->meta->start_time, $interval); + + $start_time = resolve_start_time($params, $output->meta); + $end_time = $input->meta->end_time; + if ($start_time >= $end_time) { + return array("success"=>true, "message"=>"output feed already up to date"); + } + + $this->params = $params; + $this->reset(); + $output->seek($start_time); + while (($time = $output->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $output->write_block($this->values($input->read_time($time, $interval, $n))); + } + // Reprocessing removes any output data beyond the end of the input + $bytes = $output->close($params->process_mode != 'recent'); + + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); + } + return array("success"=>true, "message"=>"bytes written: ".$bytes.", last time value: ".$output->last_time()." ".$output->last_value()); + } +} + +// Transforms the values of feed setting "feedid" in place. Always processes the whole feed, +// only blocks containing changed values are written. +abstract class InPlaceProcess extends ValueProcess +{ + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $input = new FeedReader($this->dir, $params->feedid); + $output = new FeedWriter($this->dir, $params->feedid); + $output->update_redis = false; + $npoints = $input->meta->npoints; + if ($npoints == 0) { + return array("success"=>false, "message"=>"feed is empty"); + } + + $this->params = $params; + $this->reset(); + $changed = 0; + for ($pos=0; $pos<$npoints; $pos+=POSTPROCESS_BLOCKSIZE) { + $values = $input->read($pos, min(POSTPROCESS_BLOCKSIZE, $npoints - $pos)); + $new_values = $this->values($values); + $block_changed = 0; + foreach ($values as $i => $value) { + $new_value = $new_values[$i]; + if (is_nan($value) ? !is_nan($new_value) : (is_nan($new_value) || $new_value != $value)) { + $block_changed++; + } + } + if ($block_changed) { + $output->seek($input->meta->start_time + $pos * $input->meta->interval); + $output->write_block($new_values); + $changed += $block_changed; + } + } + $output->close(); + + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); + } + return array("success"=>true, "message"=>"modified: ".$changed." datapoints, ".round(($changed / $npoints) * 100)."%"); + } +} + //compute meta datas of different feeds intended for a preprocessing function compute_meta(): stdClass { diff --git a/processes/allownegative.php b/processes/allownegative.php index 603060e..5e8519a 100644 --- a/processes/allownegative.php +++ b/processes/allownegative.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - if ($value>0) $value = 0; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value > 0 ? 0 : $value; } -} \ No newline at end of file +} diff --git a/processes/allowpositive.php b/processes/allowpositive.php index 03200a5..102e50b 100644 --- a/processes/allowpositive.php +++ b/processes/allowpositive.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - if ($value<0) $value = 0; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value < 0 ? 0 : $value; } -} \ No newline at end of file +} diff --git a/processes/limit_morethan_lessthan.php b/processes/limit_morethan_lessthan.php index 57972d3..676cabf 100644 --- a/processes/limit_morethan_lessthan.php +++ b/processes/limit_morethan_lessthan.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - $morethan = $processitem->morethan; - $lessthan = $processitem->lessthan; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - - $fix_count = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - if ($values[$i]>$morethan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$morethan)); - $fix_count ++; - } - else if ($values[$i]<$lessthan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$lessthan)); - $fix_count ++; - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$fix_count." datapoints, ".round(($fix_count/$npoints)*100)."%\n"); + if ($value > $this->params->morethan) return (float) $this->params->morethan; + if ($value < $this->params->lessthan) return (float) $this->params->lessthan; + return $value; } } diff --git a/processes/offsetfeed.php b/processes/offsetfeed.php index f5d88ae..9169930 100644 --- a/processes/offsetfeed.php +++ b/processes/offsetfeed.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]; - $value += $params->offset; - - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value + $this->params->offset; } -} \ No newline at end of file +} diff --git a/processes/remove_morethan_lessthan.php b/processes/remove_morethan_lessthan.php index 185d9b5..067c220 100644 --- a/processes/remove_morethan_lessthan.php +++ b/processes/remove_morethan_lessthan.php @@ -1,8 +1,6 @@ "Limits", "description"=>"Remove values more than or less than a certain value", "settings"=>array( - "feedid"=>array("type"=>"feed", "engine"=>5, "short"=>"Select feed to remove nan values:"), + "feedid"=>array("type"=>"feed", "engine"=>5, "short"=>"Select feed to remove values from:"), "morethan"=>array("type"=>"value", "short"=>"Remove values above this limit:"), - "lessthan"=>array("type"=>"value", "short"=>"Remote values below this limit:") + "lessthan"=>array("type"=>"value", "short"=>"Remove values below this limit:") ) ); } - public function process($processitem) + protected function value($value) { - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - $morethan = $processitem->morethan; - $lessthan = $processitem->lessthan; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - - $fix_count = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - if ($values[$i]>$morethan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",NAN)); - $fix_count ++; - } - else if ($values[$i]<$lessthan) { - fseek($fh,$dpos*4); - fwrite($fh,pack("f",NAN)); - $fix_count ++; - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$fix_count." datapoints, ".round(($fix_count/$npoints)*100)."%\n"); + if ($value > $this->params->morethan || $value < $this->params->lessthan) return NAN; + return $value; } -} \ No newline at end of file +} diff --git a/processes/removenan_lastvalue.php b/processes/removenan_lastvalue.php index dfb07ae..aae249d 100644 --- a/processes/removenan_lastvalue.php +++ b/processes/removenan_lastvalue.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - $in_nan_period = 0; - $startval = 0; - $startpos = 0; - $nanfix = 0; + private $last_value = NAN; - $value = NAN; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); + protected function reset() + { + $this->last_value = NAN; + } - if (is_nan($values[$i])) { - fseek($fh,($dpos)*4); - fwrite($fh,pack("f",$value)); - $nanfix++; - } else { - $value = $values[$i]; - } - } - $dplefttoread -= $count; - $fpos += $count; + // Replace NAN with the last value, the last value carries over between blocks + protected function values($values) + { + foreach ($values as $i => $value) { + if (is_nan($value)) $values[$i] = $this->last_value; + else $this->last_value = $value; } - fclose($fh); - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"nanfix: ".$nanfix." datapoints, ".round(($nanfix/$npoints)*100)."%\n"); + return $values; } -} \ No newline at end of file +} diff --git a/processes/scalefeed.php b/processes/scalefeed.php index be1676b..72d335c 100644 --- a/processes/scalefeed.php +++ b/processes/scalefeed.php @@ -1,6 +1,6 @@ validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } - - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,$start_pos*4); - } - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - $value = NAN; - if (!is_nan($tmp[1])) { - $value = 1*$tmp[1]*$params->scale; - } - $buffer .= pack("f",$value); - } - - fwrite($of,$buffer); - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + return $value * $this->params->scale; } -} \ No newline at end of file +} diff --git a/processes/to_signed.php b/processes/to_signed.php index dad9c23..e3a5a49 100644 --- a/processes/to_signed.php +++ b/processes/to_signed.php @@ -1,6 +1,6 @@ validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - $id = $processitem->feedid; - - if (!$fh = @fopen($dir.$id.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open feed"); - } - if (!$npoints = floor(filesize($dir.$id.".dat") / 4.0)) { - return array("success"=>false, "message"=>"feed is empty"); - } - $fpos = 0; - $dplefttoread = $npoints; - $blocksize = 100000; - $dp_modified = 0; - - $stime = microtime(true); - while ($dplefttoread>0) - { - fseek($fh,$fpos*4); - - // Read in a block of data 100,000 datapoints at a time - // this is quite a bit faster than reading in one datapoint at a time - $values = unpack("f*",fread($fh,4*$blocksize)); - $count = count($values); - if ($count==0) break; - for ($i=1; $i<=$count; $i++) - { - $dpos = $fpos + ($i-1); - - if (!is_nan($values[$i])) { - // convert to signed int 16 - if ($values[$i] > 32767) { - $fixed_value = $values[$i] - 65536; - fseek($fh,$dpos*4); - fwrite($fh,pack("f",$fixed_value)); - } - } - } - $dplefttoread -= $count; - $fpos += $count; - } - fclose($fh); - - echo "\n"; - - echo "time: ".(microtime(true)-$stime)."\n"; - return array("success" => true, "message"=>"to_signed: ".$dp_modified." datapoints, ".round(($dp_modified/$npoints)*100)."%\n"); + return $value > 32767 ? $value - 65536 : $value; } -} \ No newline at end of file +} diff --git a/readme.md b/readme.md index 6994f5b..3c1dc06 100644 --- a/readme.md +++ b/readme.md @@ -29,4 +29,4 @@ Processes are tested against small synthetic feeds in a temporary directory, no php tests/run.php # all tests php tests/run.php scalefeed # tests whose name contains scalefeed -New processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` in `common.php` for block based feed access, see `processes/backfillfeed.php` for an example. +Processes that transform each value of a feed extend `MapProcess` (input feed to output feed, e.g. `processes/scalefeed.php`) or `InPlaceProcess` (modify a feed in place, e.g. `processes/to_signed.php`) in `common.php` and only implement `value($value)`. Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. diff --git a/tests/test_processes.php b/tests/test_processes.php index 997e037..03f755d 100644 --- a/tests/test_processes.php +++ b/tests/test_processes.php @@ -10,7 +10,8 @@ empty_feed(2); assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); assert_feed(2, 1000, 10, [2, 4, NAN, 8]); - assert_redis(2, 1040, 8); + // Time of the last datapoint + assert_redis(2, 1030, 8); }); test("scalefeed recent appends new data only", function() { @@ -67,6 +68,44 @@ assert_feed(2, 1000, 10, [-1, 0, NAN, -3]); }); +test("scalefeed from after end of output continues from output end", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + append(1, [3, 4]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"from", "process_start"=>1030])); + assert_feed(2, 1000, 10, [2, 4, 6, 8]); +}); + +test("scalefeed all removes output beyond end of input", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [9, 9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4]); +}); + +test("scalefeed recent keeps output beyond end of input", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [9, 9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [9, 9, 9, 9]); +}); + +test("scalefeed across block boundaries", function() { + $n = POSTPROCESS_BLOCKSIZE * 2 + 3; + $values = range(0, $n - 1); + feed(1, 0, 10, $values); + empty_feed(2); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 0, 10, array_map(function($v) { return $v * 2; }, $values)); + assert_redis(2, ($n - 1) * 10, ($n - 1) * 2); +}); + +test("scalefeed missing output feed", function() { + feed(1, 1000, 10, [1]); + assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>5]), "does not exist"); +}); + // -------------------------------------------------------------------------------- // Accumulators // -------------------------------------------------------------------------------- @@ -161,6 +200,70 @@ assert_feed(1, 1000, 10, [-10, 5, NAN, 10]); }); +test("in place processes leave redis alone", function() { + global $redis; + feed(1, 1000, 10, [-50, 5]); + assert_success(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_equal(0, count($redis->hashes)); +}); + +test("in place process on empty feed", function() { + empty_feed(1); + assert_error(run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10]), "empty"); +}); + +test("in place process ignores process mode", function() { + feed(1, 1000, 10, [-50, 5, 50]); + $result = run("limit_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10, "process_mode"=>"recent"]); + assert_success($result); + assert_equal("modified: 2 datapoints, 67%", $result["message"]); + assert_feed(1, 1000, 10, [-10, 5, 10]); +}); + +test("remove_morethan_lessthan", function() { + feed(1, 1000, 10, [-50, 5, NAN, 50]); + assert_success(run("remove_morethan_lessthan", ["feedid"=>1, "morethan"=>10, "lessthan"=>-10])); + assert_feed(1, 1000, 10, [NAN, 5, NAN, NAN]); +}); + +test("to_signed", function() { + feed(1, 1000, 10, [100, 65535, NAN, 32768, 32767]); + assert_success(run("to_signed", ["feedid"=>1])); + assert_feed(1, 1000, 10, [100, -1, NAN, -32768, 32767]); +}); + +test("removenan_lastvalue", function() { + feed(1, 1000, 10, [NAN, 1, NAN, NAN, 4, NAN]); + assert_success(run("removenan_lastvalue", ["feedid"=>1])); + // Leading NAN has no last value + assert_feed(1, 1000, 10, [NAN, 1, 1, 1, 4, 4]); +}); + +test("removenan_lastvalue carries last value across blocks and resets between runs", function() { + global $testdir; + $n = POSTPROCESS_BLOCKSIZE + 2; + $values = array_fill(0, $n, NAN); + $values[POSTPROCESS_BLOCKSIZE - 1] = 7; + feed(1, 0, 10, $values); + feed(2, 0, 10, [NAN, 3]); + + // The runner reuses process objects between runs + $p = new PostProcess_removenan_lastvalue($testdir); + $params = (object) ["process_mode"=>"all", "process_start"=>0]; + $params->feedid = 1; + ob_start(); + assert_success($p->process($params)); + $params->feedid = 2; + assert_success($p->process($params)); + ob_end_clean(); + + $result = values(1); + assert_equal([7, 7, 7], array_slice($result, POSTPROCESS_BLOCKSIZE - 1)); + assert_equal(true, is_nan($result[0])); + // Not carried over from the previous run + assert_equal([NAN, 3], values(2)); +}); + // -------------------------------------------------------------------------------- // Backfill // -------------------------------------------------------------------------------- From 95b0d0a6a4e84b07fce85715ede28d8fee1db917 Mon Sep 17 00:00:00 2001 From: Trystan Lea Date: Fri, 9 Oct 2026 23:15:19 +0100 Subject: [PATCH 3/5] Add CombineProcess base class for multi input and accumulating processes Converted: addfeeds, mergefeeds, importcalc, exportcalc, solardirectcalc, powertokwh, accumulator, constantflow_tokwh, liquidorairflow_tokwh and basic_formula. Inputs are read in blocks, 11-17x faster for two and three input processes. Inputs with a shorter interval are averaged into each output interval and only complete output intervals are written. Fixes: - powertokwh/accumulator: filtered values shifted later output data - basic_formula: feed ids that are prefixes of others (f1, f12) not read - basic_formula: from mode appended a duplicate copy of the output - solardirectcalc: redis update used a non existent setting - constantflow/liquidorairflow ignored the process mode - combine processes overwrote output meta in recent mode Reprocess all now replaces an output feed with a different start time or interval. Removes unused compute_meta, bfo takes values rather than file handles. Co-Authored-By: Claude Opus 5.5 --- common.php | 163 +++++++++++++++++++++------- processes/accumulator.php | 108 ++++-------------- processes/addfeeds.php | 111 ++----------------- processes/basic_formula.php | 160 ++++++++------------------- processes/constantflow_tokwh.php | 121 +++------------------ processes/exportcalc.php | 112 ++----------------- processes/importcalc.php | 113 ++----------------- processes/liquidorairflow_tokwh.php | 130 +++------------------- processes/mergefeeds.php | 111 ++----------------- processes/powertokwh.php | 107 ++++-------------- processes/solardirectcalc.php | 111 ++----------------- readme.md | 8 +- tests/test_combine.php | 162 +++++++++++++++++++++++++++ tests/test_processes.php | 33 +++++- 14 files changed, 483 insertions(+), 1067 deletions(-) create mode 100644 tests/test_combine.php diff --git a/common.php b/common.php index a58c650..49446f1 100644 --- a/common.php +++ b/common.php @@ -402,6 +402,20 @@ public function close($truncate = false) return $this->bytes_written; } + // Value of the datapoint at $time, NAN if outside of the feed + public function value_at($time) + { + $this->flush(); + $offset = $time - $this->meta->start_time; + if ($offset < 0 || $offset % $this->meta->interval != 0) return NAN; + $pos = intdiv($offset, $this->meta->interval); + if ($pos >= $this->meta->npoints) return NAN; + fseek($this->fh, $pos * 4); + $value = unpack("f", fread($this->fh, 4))[1]; + fseek($this->fh, $this->pos * 4); + return $value; + } + public function last_time() { return $this->last_time; } public function last_value() { return $this->last_value; } } @@ -436,7 +450,8 @@ protected function values($values) } // Writes the transformed values of feed setting "input" to feed setting "output", with the -// same start time and interval. Supports recent, from and all process modes. +// same start time and interval. Supports recent, from and all process modes, reprocessing +// all replaces the output feed. abstract class MapProcess extends ValueProcess { public function process($params) @@ -448,7 +463,7 @@ public function process($params) $input = new FeedReader($this->dir, $params->input); $output = new FeedWriter($this->dir, $params->output); $interval = $input->meta->interval; - $output->init($input->meta->start_time, $interval); + $output->init($input->meta->start_time, $interval, $params->process_mode == 'all'); $start_time = resolve_start_time($params, $output->meta); $end_time = $input->meta->end_time; @@ -519,28 +534,107 @@ public function process($params) } } -//compute meta datas of different feeds intended for a preprocessing -function compute_meta(): stdClass +// Combines input feeds into an output feed on a common timeline: the longest input interval, +// from the latest input start time to the earliest input end time. Inputs with a shorter +// interval are averaged into each output interval. Only complete output intervals are +// written so that recent mode picks up partially recorded intervals on the next run. +// Supports recent, from and all process modes, reprocessing all replaces the output feed. +// +// Subclasses set $inputs and $output to the feed setting keys and implement +// row($a, $b, ...), called with one value per input in the order of $inputs (including NAN) +// and returning the output value. Settings are available as $this->params and the output +// interval as $this->interval. reset($previous) is called at the start of each run with the +// output value before the start time (NAN if none), e.g. to continue an accumulation. +abstract class CombineProcess extends PostProcess_common { - $numargs = func_num_args(); - $arg_list = func_get_args(); - $all_intervals = []; - $all_start_times = []; - $all_ending_times = []; - $meta = new stdClass(); - for ($i = 0; $i < $numargs; $i++) { - $all_intervals[] = $arg_list[$i]->interval; - $all_start_times[] = $arg_list[$i]->start_time; - $all_ending_times[] = $arg_list[$i]->start_time + $arg_list[$i]->npoints * $arg_list[$i]->interval; - } - $meta->interval = max($all_intervals); - $meta->start_time = floor(max($all_start_times) / $meta->interval) * $meta->interval; - $meta->writing_end_time = min($all_ending_times); - //print("intervals.....");print_r($all_intervals); - //print("start_times....");print_r($all_start_times); - //print("ending_times.....");print_r($all_ending_times); - print("NOTICE : output interval=$meta->interval, start=$meta->start_time, end=$meta->writing_end_time \n"); - return $meta; + protected $inputs = array(); + protected $output = "output"; + protected $params; + protected $interval; + + protected function reset($previous) {} + + // Feed ids of the inputs keyed by name, from the $inputs settings by default + protected function input_feeds($params) + { + $feeds = array(); + foreach ($this->inputs as $key) $feeds[$key] = $params->$key; + return $feeds; + } + + // Output values for a block of input values, one array per input + protected function block($blocks) + { + $out = array(); + $columns = array_values($blocks); + switch (count($columns)) { + case 1: + foreach ($columns[0] as $a) $out[] = $this->row($a); + break; + case 2: + foreach ($columns[0] as $i => $a) $out[] = $this->row($a, $columns[1][$i]); + break; + case 3: + foreach ($columns[0] as $i => $a) $out[] = $this->row($a, $columns[1][$i], $columns[2][$i]); + break; + default: + $n = count($columns[0]); + for ($i=0; $i<$n; $i++) $out[] = $this->row(...array_column($columns, $i)); + } + return $out; + } + + public function process($params) + { + $result = $this->validate($params); + if (!$result["success"]) return $result; + + try { + $this->params = $params; + $inputs = array(); + foreach ($this->input_feeds($params) as $key => $feedid) { + $inputs[$key] = new FeedReader($this->dir, $feedid); + } + if (!count($inputs)) { + return array("success"=>false, "message"=>"no input feeds"); + } + $output = new FeedWriter($this->dir, $params->{$this->output}); + + $interval = 0; + $start_time = 0; + $end_time = PHP_INT_MAX; + foreach ($inputs as $input) { + $interval = max($interval, $input->meta->interval); + $start_time = max($start_time, $input->meta->start_time); + $end_time = min($end_time, $input->meta->end_time); + } + $start_time = (int) floor($start_time / $interval) * $interval; + $end_time = $start_time + (int) floor(($end_time - $start_time) / $interval) * $interval; + $this->interval = $interval; + + $output->init($start_time, $interval, $params->process_mode == 'all'); + $time = resolve_start_time($params, $output->meta); + if ($time >= $end_time) { + return array("success"=>true, "message"=>"output feed already up to date"); + } + + $this->reset($output->value_at($time - $interval)); + $output->seek($time); + while (($time = $output->time()) < $end_time) { + $n = (int) min(POSTPROCESS_BLOCKSIZE, ($end_time - $time) / $interval); + $blocks = array(); + foreach ($inputs as $key => $input) { + $blocks[$key] = $input->read_time($time, $interval, $n); + } + $output->write_block($this->block($blocks)); + } + $bytes = $output->close($params->process_mode != 'recent'); + + } catch (PostProcessException $e) { + return array("success"=>false, "message"=>$e->getMessage()); + } + return array("success"=>true, "message"=>"bytes written: ".$bytes.", last time value: ".$output->last_time()." ".$output->last_value()); + } } /* @@ -573,7 +667,7 @@ function ftoa($b): array } /* -output of a basic formula for a specified time +output of a basic formula for one timestep the formula is described by the $elements array > a third dimensionnal array > datas are on level 2 for a given level 1, we operate multiplication/division within level 2 elements and give a sign to the result the final result is the addition of the whole @@ -581,32 +675,25 @@ function ftoa($b): array 0 -> type of data ie "feed" ou "value" 1 -> operator 2 -> value or feed number -feeds_meta : array of metas such as produced by getmeta +values : feed values for this timestep keyed by feed number */ -function bfo($elements, $feeds_meta, $feeds_dat, $time): float|int +function bfo($elements, $values): float|int { $s = []; foreach ($elements as $element) { - $values = []; + $product = []; foreach ($element as $e) { $value = NAN; - if ($e[0] == "feed") { - $pos = floor(num: ($time - $feeds_meta[$e[2]]->start_time) / $feeds_meta[$e[2]]->interval); - if ($pos >= 0 && $pos < $feeds_meta[$e[2]]->npoints) { - fseek(stream: $feeds_dat[$e[2]], offset: $pos * 4); - $tmp = unpack("f", fread($feeds_dat[$e[2]], 4)); - $value = $tmp[1]; - } - } + if ($e[0] == "feed") $value = $values[$e[2]]; if ($e[0] == "value") $value = $e[2]; if (!is_nan(num: $value) && $value != 0) { if ($e[1] == "/") $value = 1 / $value; if ($e[1] == "-") $value = -$value; } - $values[] = $value; + $product[] = $value; } - if (!in_array(needle: NAN, haystack: $values)) { - $s[] = array_product(array: $values); + if (!in_array(needle: NAN, haystack: $product)) { + $s[] = array_product(array: $product); } else $s[] = NAN; } if (!in_array(needle: NAN, haystack: $s)) { diff --git a/processes/accumulator.php b/processes/accumulator.php index 577212c..4808787 100644 --- a/processes/accumulator.php +++ b/processes/accumulator.php @@ -1,7 +1,11 @@ "Accumulator", @@ -16,95 +20,23 @@ public function description() { ); } - public function process($params) + protected function reset($previous) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $total = 0; - $value = 0; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } + $this->total = is_nan($previous) ? 0 : $previous; + $this->value = 0; + } - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,($start_pos-1)*4); - $tmp = unpack("f",fread($of,4)); - $total = $tmp[1]*1; - } - - $filtered_count = 0; - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - if (!is_nan($tmp[1])) $value = 1*$tmp[1]; - - // filter spurious values +-1M - if ($value>=$params->min_value && $value<$params->max_value) { - $total += $value; - $buffer .= pack("f",$total); - } else { - $filtered_count++; + // Missing values hold the last value. Values outside of the limits are treated + // as spurious and are not added. + protected function row($value) + { + if (!is_nan($value)) { + if ($value < $this->params->min_value || $value >= $this->params->max_value) { + return $this->total; } + $this->value = $value; } - - fwrite($of,$buffer); - - if ($filtered_count>0) { - print "Filtered count: $filtered_count\n"; - } - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$total); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$total); - + $this->total += $this->value; + return $this->total; } -} \ No newline at end of file +} diff --git a/processes/addfeeds.php b/processes/addfeeds.php index 5ee63d8..ff8c453 100644 --- a/processes/addfeeds.php +++ b/processes/addfeeds.php @@ -1,7 +1,9 @@ "Add feeds", @@ -15,106 +17,11 @@ public function description() { ); } - public function process($params) + // Sum of both feeds, or whichever feed has data + protected function row($a, $b) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $feedA_meta = getmeta($this->dir,$params->feedA); - $feedB_meta = getmeta($this->dir,$params->feedB); - - // Find longest interval - $interval = $feedA_meta->interval; - if ($feedB_meta->interval>$interval) $interval = $feedB_meta->interval; - // Find latest start time - $start_time = $feedA_meta->start_time; - if ($feedB_meta->start_time>$start_time) $start_time = $feedB_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $feedA_end_time = $feedA_meta->start_time + ($feedA_meta->interval * $feedA_meta->npoints); - $feedB_end_time = $feedB_meta->start_time + ($feedB_meta->interval * $feedB_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $feedA_end_time; - if ($feedB_end_time<$end_time) $end_time = $feedB_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$feedA_fh = @fopen($this->dir.$params->feedA.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedA feed"); - } - if (!$feedB_fh = @fopen($this->dir.$params->feedB.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedB feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_feedA = floor(($time - $feedA_meta->start_time) / $feedA_meta->interval); - $pos_feedB = floor(($time - $feedB_meta->start_time) / $feedB_meta->interval); - - $valueA = NAN; - $valueB = NAN; - - if ($pos_feedA>=0 && $pos_feedA<$feedA_meta->npoints) { - fseek($feedA_fh,$pos_feedA*4); - $tmp = unpack("f",fread($feedA_fh,4)); - $valueA = $tmp[1]; - } - - if ($pos_feedB>=0 && $pos_feedB<$feedB_meta->npoints) { - fseek($feedB_fh,$pos_feedB*4); - $tmp = unpack("f",fread($feedB_fh,4)); - $valueB = $tmp[1]; - } - - $outval = NAN; - if (!is_nan($valueA)) $outval = $valueA; - if (!is_nan($valueB)) $outval = $valueB; - if (!is_nan($valueA) && !is_nan($valueB)) $outval = $valueB + $valueA; - $buffer .= pack("f",$outval); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($feedA_fh); - fclose($feedB_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$outval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$outval); + if (is_nan($a)) return $b; + if (is_nan($b)) return $a; + return $a + $b; } -} \ No newline at end of file +} diff --git a/processes/basic_formula.php b/processes/basic_formula.php index 9a8ef38..5114a4f 100644 --- a/processes/basic_formula.php +++ b/processes/basic_formula.php @@ -15,8 +15,11 @@ function formula_to_array($formula_string, $reg_exp): array return $fly; } -class PostProcess_basic_formula extends PostProcess_common +class PostProcess_basic_formula extends CombineProcess { + private $elements = []; + private $feed_ids = []; + public function description(): array { $bfdescription="Enter your formula as a symbolic expression - allows brackets and the max function
@@ -39,10 +42,10 @@ public function description(): array { ]; } - public function process($processitem): array + public function process($params): array { $DEBUG = 0; - $result = $this->validate(processitem: $processitem); + $result = $this->validate(processitem: $params); if (!$result["success"]) return $result; $dir = $this->dir; @@ -74,7 +77,7 @@ public function process($processitem): array //retrieving the formula - $formula=$processitem->formula; + $formula=$params->formula; //removing useless brackets $pos=0; $original=$formula; @@ -121,12 +124,8 @@ public function process($processitem): array $original=$formula; //we catch the distinct feed numbers involved in the formula - $feed_ids=[]; - while(preg_match(pattern: "/$Xf/", subject: $formula, matches: $b)){ - //removing the f... - $feed_ids[]=substr(string: $b[0], offset: 1); - $formula=str_replace(search: $b[0], replace: "", subject: $formula); - } + preg_match_all(pattern: "/f(\d+)/", subject: $formula, matches: $b); + $feed_ids=array_values(array_unique(array: $b[1])); $formula=$original; $array= []; @@ -213,21 +212,10 @@ public function process($processitem): array print $original_copy; print "\n\n"; } - if ($original_copy=="") print "formula is OK!!\n"; else { + if ($original_copy!="") { return ["success"=>false, "message"=>"could not understand your formula SORRY...."]; } - //checking the output feed - $fopen_mode='ab'; - if ($processitem->process_mode=='all') { - $fopen_mode='wb'; - } - $out=$processitem->output; - if(!$out_meta = getmeta(dir: $dir,id: $out)) return array("success"=>false, "message"=>"could not get meta for $out"); - if (!$out_fh = @fopen(filename: $dir.$out.".dat", mode: $fopen_mode)) { - return ["success"=>false, "message"=>"could not open $dir $out.dat"]; - } - $elements=[]; foreach ($array as $a){ $element=new stdClass(); @@ -270,100 +258,48 @@ public function process($processitem): array } if ($DEBUG) print_r(value: $elements); - //we retrieve the meta and open the dat files - foreach ($feed_ids as $id){ - if(!$meta = getmeta(dir: $dir, id: $id)) return array("success"=>false, "message"=>"could not get meta for $id"); - $feeds_meta[$id]=$meta; - if (!$fh = @fopen(filename: "$dir$id.dat", mode: 'rb')) { - return ["success"=>false, "message"=>"could not open $dir $id.dat"]; - } - $feeds_dat[$id]=$fh; - } - - $compute_meta= call_user_func_array(callback: "compute_meta", args: $feeds_meta); - - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - - if ($processitem->process_mode=='recent') { - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - } else { - $writing_start_time=$out_meta->start_time; - } - - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - - $buffer=""; + $this->elements=$elements; + $this->feed_ids=$feed_ids; + return parent::process($params); + } - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - $s=[]; - foreach($elements as $element){ - $s1=bfo( - elements: [$element->scale], - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s2=bfo( - elements: $element->formula, - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s3=bfo( - elements: [$element->scale_right], - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - //print("$s1-----$s2-----$s3"); - if (!is_nan(num: $s1) && !is_nan(num: $s2) && !is_nan(num: $s3)) { - if($element->function=="max") { - $arg2 = (is_numeric(value: $element->arg2)) ? $element->arg2 : bfo( - elements: $element->arg2, - feeds_meta: $feeds_meta, - feeds_dat: $feeds_dat, - time: $time - ); - $s[] = (!is_nan(num: $arg2)) ? $s1*max([$s2,$arg2])*$s3 : NAN; - } - if($element->function=="brackets" || $element->function=="none") { - $s[]=$s1*$s2*$s3; - } - } else $s[] = NAN; - } - $sum = (!in_array(needle: NAN, haystack: $s)) ? array_sum(array: $s) : NAN; - $buffer.=pack("f", $sum); - } + protected function input_feeds($params) + { + $feeds=[]; + foreach ($this->feed_ids as $id) $feeds[$id]=$id; + return $feeds; + } - if(!$buffer) { - return ["success"=>false, "message"=>"nothing to write - all is up to date"]; + protected function block($blocks) + { + $out=[]; + $n=count(value: reset($blocks)); + for ($i=0;$i<$n;$i++) { + $values=[]; + foreach ($blocks as $id=>$block) $values[$id]=$block[$i]; + $out[]=$this->evaluate(values: $values); } + return $out; + } - if(!$written_bytes=fwrite(stream: $out_fh, data: $buffer)){ - foreach ($feeds_dat as $f) fclose(stream: $f); - fclose(stream: $out_fh); - return ["success"=>false, "message"=>"unable to write to the file with id=$out"]; + // value of the formula for one timestep, $values are the feed values keyed by feed id + private function evaluate($values) + { + $s=[]; + foreach($this->elements as $element){ + $s1=bfo(elements: [$element->scale], values: $values); + $s2=bfo(elements: $element->formula, values: $values); + $s3=bfo(elements: [$element->scale_right], values: $values); + if (!is_nan(num: $s1) && !is_nan(num: $s2) && !is_nan(num: $s3)) { + if($element->function=="max") { + $arg2 = (is_numeric(value: $element->arg2)) ? $element->arg2 : bfo(elements: $element->arg2, values: $values); + $s[] = (!is_nan(num: $arg2)) ? $s1*max([$s2,$arg2])*$s3 : NAN; + } + if($element->function=="brackets" || $element->function=="none") { + $s[]=$s1*$s2*$s3; + } + } else $s[] = NAN; } - $nbdataswritten=$written_bytes/4; - print("NOTICE: basic_formula() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta(dir: $dir, id: $out, meta: $out_meta); - foreach ($feeds_dat as $f) fclose(stream: $f); - fclose(stream: $out_fh); - print("last time value: $time / $sum \n"); - updatetimevalue(id: $out, time: $time, value: $sum); - return [ - "success"=>true, - "message"=>"bytes written: $written_bytes, last time value: $time, last written value $sum" - ]; + return (!in_array(needle: NAN, haystack: $s)) ? array_sum(array: $s) : NAN; } } diff --git a/processes/constantflow_tokwh.php b/processes/constantflow_tokwh.php index 6509589..bc8e065 100644 --- a/processes/constantflow_tokwh.php +++ b/processes/constantflow_tokwh.php @@ -1,7 +1,10 @@ "Constant flow to kWh", @@ -17,111 +20,17 @@ public function description() { ); } - public function process($processitem) + protected function reset($previous) { - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $dir = $this->dir; - - $vhc = (float) $processitem->vhc; - $flow = (float) $processitem->flow; - $tint = $processitem->tint; - $text = $processitem->text; - $out = $processitem->output; - - if (!$tint_meta = getmeta($dir,$tint)) array("success"=>false, "message"=>"could not open input feed"); - if (!$text_meta = getmeta($dir,$text)) array("success"=>false, "message"=>"could not open input feed"); - if (!$out_meta = getmeta($dir,$out)) array("success"=>false, "message"=>"could not open output feed"); - - if (!$tint_fh = @fopen($dir.$tint.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$text_fh = @fopen($dir.$text.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$out_fh = @fopen($dir.$out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $compute_meta=compute_meta($tint_meta,$text_meta); - - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - $kwh = 0; - - if($out_meta->npoints>0) { - print("NOTICE : file not empty \n"); - $pos_last=($out_meta->npoints-1)*4; - print("NOTICE : reading last previously processed value in file position $pos_last \n"); - fseek($out_fh,$pos_last); - $tmp=unpack("f",fread($out_fh,4)); - $kwh = $tmp[1]; - print("NOTICE : accumulation will resume at last previously processed value $kwh \n"); - } else fseek($out_fh,0); - - $buffer=""; - - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - - $pos_tint = floor(($time - $tint_meta->start_time) / $tint_meta->interval); - $pos_text = floor(($time - $text_meta->start_time) / $text_meta->interval); - $value_tint=NAN; - $value_text=NAN; - if ($pos_tint>=0 && $pos_tint<$tint_meta->npoints) { - fseek($tint_fh,$pos_tint*4); - $tint_tmp = unpack("f",fread($tint_fh,4)); - $value_tint = $tint_tmp[1]; - } - if ($pos_text>=0 && $pos_text<$text_meta->npoints) { - fseek($text_fh,$pos_text*4); - $text_tmp = unpack("f",fread($text_fh,4)); - $value_text = $text_tmp[1]; - } - - //if we face one or more NAN value among tint,text, we should not make any calculation and $kwh should remain the same - if(!is_nan($value_tint) && !is_nan($value_text)){ - $kwh+=0.001*$vhc*$flow*max($value_tint-$value_text,0)*$out_meta->interval/3600; - } - //print("$kwh/"); - $buffer.=pack("f",$kwh); - } - - if(!$buffer) { - return array("success"=>false,"message"=>"nothing to write - all is up to date"); - } + $this->kwh = is_nan($previous) ? 0 : $previous; + } - if(!$written_bytes=fwrite($out_fh,$buffer)){ - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - return array("success"=>false,"message"=>"unable to write to the file with id=$out"); - } - $nbdataswritten=$written_bytes/4; - print("NOTICE: constantflow_tokwh() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta($dir,$out,$out_meta); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - print("last time value: $time / $kwh \n"); - - if ($written_bytes>0) { - updatetimevalue($out,$time,$kwh); + // Missing temperatures add no energy + protected function row($tint, $text) + { + if (!is_nan($tint) && !is_nan($text)) { + $this->kwh += 0.001 * $this->params->vhc * $this->params->flow * max($tint - $text, 0) * $this->interval / 3600; } - return array("success"=>true, "message"=>"bytes written: ".$written_bytes.", last time value: ".$time." ".$kwh); + return $this->kwh; } -} \ No newline at end of file +} diff --git a/processes/exportcalc.php b/processes/exportcalc.php index 1291fb1..6af9b77 100644 --- a/processes/exportcalc.php +++ b/processes/exportcalc.php @@ -1,7 +1,9 @@ "Export calculation", @@ -15,107 +17,11 @@ public function description() { ); } - public function process($params) + // Missing consumption is treated as no consumption + protected function row($gen, $use) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->consumption); - $gen_meta = getmeta($this->dir,$params->generation); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->consumption.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->generation.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $exportval = NAN; - if (!is_nan($genval)) $exportval = $genval; - if (!is_nan($genval) && !is_nan($useval)) $exportval = $genval - $useval; - if ($exportval<0) $exportval = 0; - - $buffer .= pack("f",$exportval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$exportval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$exportval); + if (is_nan($gen)) return NAN; + if (!is_nan($use)) $gen -= $use; + return $gen < 0 ? 0 : $gen; } -} \ No newline at end of file +} diff --git a/processes/importcalc.php b/processes/importcalc.php index f0de1ea..c7e3f08 100644 --- a/processes/importcalc.php +++ b/processes/importcalc.php @@ -1,7 +1,10 @@ "Import calculation", @@ -15,107 +18,11 @@ public function description() { ); } - public function process($params) + // Missing generation is treated as no generation + protected function row($use, $gen) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->use); - $gen_meta = getmeta($this->dir,$params->gen); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->out,$out_meta); - $out_meta = getmeta($this->dir,$params->out); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->use.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->gen.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $importval = NAN; - if (!is_nan($useval)) $importval = $useval; - if (!is_nan($genval) && !is_nan($genval)) $importval = $useval - $genval; - if ($importval<0) $importval = 0; - - $buffer .= pack("f",$importval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->out,$time,$importval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$importval); + if (is_nan($use)) return NAN; + if (!is_nan($gen)) $use -= $gen; + return $use < 0 ? 0 : $use; } -} \ No newline at end of file +} diff --git a/processes/liquidorairflow_tokwh.php b/processes/liquidorairflow_tokwh.php index fbaa408..6ec0f12 100644 --- a/processes/liquidorairflow_tokwh.php +++ b/processes/liquidorairflow_tokwh.php @@ -1,7 +1,10 @@ "Liquid or airflow to kWh", @@ -17,122 +20,17 @@ public function description() { ); } - public function process($processitem) + protected function reset($previous) { - $dir = $this->dir; - - $result = $this->validate($processitem); - if (!$result["success"]) return $result; - - $vhc = (float) $processitem->vhc; - $flow = $processitem->flow; - $tint = $processitem->tint; - $text = $processitem->text; - $out = $processitem->output; - - if (!$flow_meta = getmeta($dir,$flow)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$tint_meta = getmeta($dir,$tint)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$text_meta = getmeta($dir,$text)) return array("success"=>false, "message"=>"could not open input feed"); - if (!$out_meta = getmeta($dir,$out)) return array("success"=>false, "message"=>"could not open output feed"); - - if (!$flow_fh = @fopen($dir.$flow.".dat", 'rb')) { - echo "ERROR: could not open $dir $flow.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$tint_fh = @fopen($dir.$tint.".dat", 'rb')) { - echo "ERROR: could not open $dir $tint.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$text_fh = @fopen($dir.$text.".dat", 'rb')) { - echo "ERROR: could not open $dir $text.dat\n"; - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$out_fh = @fopen($dir.$out.".dat", 'c+')) { - echo "ERROR: could not open $dir $out.dat\n"; - return array("success"=>false, "message"=>"could not open output feed"); - } - - $compute_meta=compute_meta($flow_meta,$tint_meta,$text_meta); - - //reading the output meta and if dat file is empty, we adjust interval and start_time - //we do not report the values in the meta file at this stage. we wait for the dat file to be filled with processed datas - //if dat file is not empty, meta file should already contain correct values - print("NOTICE : ouput is : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - if($out_meta->npoints==0) { - $out_meta->interval=$compute_meta->interval; - $out_meta->start_time=$compute_meta->start_time; - } - print("NOTICE : ouput is now : ($out_meta->npoints,$out_meta->interval,$out_meta->start_time) \n"); - - $writing_start_time=$out_meta->start_time+($out_meta->interval*$out_meta->npoints); - $writing_end_time=$compute_meta->writing_end_time; - $interval=$out_meta->interval; - $kwh = 0; - - if($out_meta->npoints>0) { - print("NOTICE : file not empty \n"); - $pos_last=($out_meta->npoints-1)*4; - print("NOTICE : reading last previously processed value in file position $pos_last \n"); - fseek($out_fh,$pos_last); - $tmp=unpack("f",fread($out_fh,4)); - $kwh = $tmp[1]; - print("NOTICE : accumulation will resume at last previously processed value $kwh \n"); - } else fseek($out_fh,0); - $buffer=""; - for ($time=$writing_start_time;$time<$writing_end_time;$time+=$interval){ - $pos_flow = floor(($time - $flow_meta->start_time) / $flow_meta->interval); - $pos_tint = floor(($time - $tint_meta->start_time) / $tint_meta->interval); - $pos_text = floor(($time - $text_meta->start_time) / $text_meta->interval); - $value_flow=NAN; - $value_tint=NAN; - $value_text=NAN; - if ($pos_flow>=0 && $pos_flow<$flow_meta->npoints) { - fseek($flow_fh,$pos_flow*4); - $flow_tmp = unpack("f",fread($flow_fh,4)); - $value_flow = $flow_tmp[1]; - } - if ($pos_tint>=0 && $pos_tint<$tint_meta->npoints) { - fseek($tint_fh,$pos_tint*4); - $tint_tmp = unpack("f",fread($tint_fh,4)); - $value_tint = $tint_tmp[1]; - } - if ($pos_text>=0 && $pos_text<$text_meta->npoints) { - fseek($text_fh,$pos_text*4); - $text_tmp = unpack("f",fread($text_fh,4)); - $value_text = $text_tmp[1]; - } - - //if we face one or more NAN value among flow,tint,text, we should not make any calculation and $kwh should remain the same - if(!is_nan($value_flow) && !is_nan($value_tint) && !is_nan($value_text)){ - $kwh+=0.001*$vhc*$value_flow*max($value_tint-$value_text,0)*$out_meta->interval/3600; - } - //print("$kwh/"); - $buffer.=pack("f",$kwh); - } - if(!$buffer) { - return array("success"=>true, "message"=>"nothing to write - all is up to date"); - } + $this->kwh = is_nan($previous) ? 0 : $previous; + } - if(!$written_bytes=fwrite($out_fh,$buffer)){ - fclose($flow_fh); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - return array("success"=>false, "message"=>"unable to write to the file with id=$out"); + // Missing flow or temperatures add no energy + protected function row($flow, $tint, $text) + { + if (!is_nan($flow) && !is_nan($tint) && !is_nan($text)) { + $this->kwh += 0.001 * $this->params->vhc * $flow * max($tint - $text, 0) * $this->interval / 3600; } - $nbdataswritten=$written_bytes/4; - print("NOTICE: liquidorairflow_tokwh() wrote $written_bytes bytes ($nbdataswritten float values) \n"); - //we update the meta only as the dat has been filled - createmeta($dir,$out,$out_meta); - fclose($flow_fh); - fclose($tint_fh); - fclose($text_fh); - fclose($out_fh); - print("last time value: $time / $kwh \n"); - updatetimevalue($out,$time,$kwh); - return array("success"=>true, "message"=>"bytes written: ".$written_bytes.", last time value: ".$time." ".$kwh); + return $this->kwh; } -} \ No newline at end of file +} diff --git a/processes/mergefeeds.php b/processes/mergefeeds.php index 9620801..c05f440 100644 --- a/processes/mergefeeds.php +++ b/processes/mergefeeds.php @@ -1,7 +1,9 @@ "Merge feeds", @@ -15,106 +17,11 @@ public function description() { ); } - public function process($params) + // Average of both feeds, or whichever feed has data + protected function row($a, $b) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $feedA_meta = getmeta($this->dir,$params->feedA); - $feedB_meta = getmeta($this->dir,$params->feedB); - - // Find longest interval - $interval = $feedA_meta->interval; - if ($feedB_meta->interval>$interval) $interval = $feedB_meta->interval; - // Find latest start time - $start_time = $feedA_meta->start_time; - if ($feedB_meta->start_time>$start_time) $start_time = $feedB_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->output,$out_meta); - $out_meta = getmeta($this->dir,$params->output); - - // Find end time of input feeds - $feedA_end_time = $feedA_meta->start_time + ($feedA_meta->interval * $feedA_meta->npoints); - $feedB_end_time = $feedB_meta->start_time + ($feedB_meta->interval * $feedB_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $feedA_end_time; - if ($feedB_end_time<$end_time) $end_time = $feedB_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$feedA_fh = @fopen($this->dir.$params->feedA.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedA feed"); - } - if (!$feedB_fh = @fopen($this->dir.$params->feedB.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open feedB feed"); - } - if (!$out_fh = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_feedA = floor(($time - $feedA_meta->start_time) / $feedA_meta->interval); - $pos_feedB = floor(($time - $feedB_meta->start_time) / $feedB_meta->interval); - - $valueA = NAN; - $valueB = NAN; - - if ($pos_feedA>=0 && $pos_feedA<$feedA_meta->npoints) { - fseek($feedA_fh,$pos_feedA*4); - $tmp = unpack("f",fread($feedA_fh,4)); - $valueA = $tmp[1]; - } - - if ($pos_feedB>=0 && $pos_feedB<$feedB_meta->npoints) { - fseek($feedB_fh,$pos_feedB*4); - $tmp = unpack("f",fread($feedB_fh,4)); - $valueB = $tmp[1]; - } - - $outval = NAN; - if (!is_nan($valueA)) $outval = $valueA; - if (!is_nan($valueB)) $outval = $valueB; - if (!is_nan($valueA) && !is_nan($valueB)) $outval = ($valueB+$valueA)*0.5; - $buffer .= pack("f",$outval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($feedA_fh); - fclose($feedB_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$outval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$outval); + if (is_nan($a)) return $b; + if (is_nan($b)) return $a; + return ($a + $b) * 0.5; } -} \ No newline at end of file +} diff --git a/processes/powertokwh.php b/processes/powertokwh.php index 4a3a387..0b339a2 100644 --- a/processes/powertokwh.php +++ b/processes/powertokwh.php @@ -1,7 +1,11 @@ "Power to kWh", @@ -17,96 +21,23 @@ public function description() { ); } - public function process($params) + protected function reset($previous) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $input_meta = getmeta($this->dir,$params->input); - $output_meta = getmeta($this->dir,$params->output); - - // Check that output feed is empty or has same start time and interval - if ($output_meta->npoints>0) { - if ($input_meta->start_time != $output_meta->start_time) { - return array("success"=>false, "message"=>"start time mismatch"); - } - if ($input_meta->interval != $output_meta->interval) { - return array("success"=>false, "message"=>"interval mismatch"); - } - } else { - // Copies over start_time to output meta file - createmeta($this->dir,$params->output,$input_meta); - } - - // If recent mode, check that input feed has more points than output feed - if ($params->process_mode=='recent' && $output_meta->npoints >= $input_meta->npoints) { - return array("success"=>true, "message"=>"output feed already up to date"); - } - - if (!$if = @fopen($this->dir.$params->input.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open input feed"); - } - - if (!$of = @fopen($this->dir.$params->output.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - - $buffer = ""; - $wh = 0; - $power = 0; - $start_pos = 0; - - if ($params->process_mode=='from' && $output_meta->npoints>0) { - $start_pos = floor(($params->process_start - $input_meta->start_time) / $input_meta->interval); - if ($start_pos<0) $start_pos = 0; - if ($start_pos>$input_meta->npoints) { - return array("success"=>false, "message"=>"start time is after end of input feed"); - } - } - - if ($params->process_mode=='recent' && $output_meta->npoints>0) { - $start_pos = $output_meta->npoints; - } + $this->wh = is_nan($previous) ? 0 : $previous * 1000; + $this->power = 0; + } - if ($start_pos>0) { - fseek($if,$start_pos*4); - fseek($of,($start_pos-1)*4); - $tmp = unpack("f",fread($of,4)); - $wh = $tmp[1]*1000.0; - } - - $filtered_count = 0; - - for ($n=$start_pos; $n<$input_meta->npoints; $n++) { - $tmp = unpack("f",fread($if,4)); - - if (!is_nan($tmp[1])) $power = 1*$tmp[1]; - - // filter spurious power values +-1MW - if ($power>=$params->min_power && $power<$params->max_power) { - $wh += ($power * $input_meta->interval) / 3600.0; - $buffer .= pack("f",$wh*0.001); - } else { - $filtered_count++; + // Missing power values hold the last power value. Power values outside of the + // limits are treated as spurious and add no energy. + protected function row($power) + { + if (!is_nan($power)) { + if ($power < $this->params->min_power || $power >= $this->params->max_power) { + return $this->wh * 0.001; } + $this->power = $power; } - - fwrite($of,$buffer); - - if ($filtered_count>0) { - print "Filtered count: $filtered_count\n"; - } - - fclose($of); - fclose($if); - - $time = $input_meta->start_time + ($input_meta->npoints * $input_meta->interval); - $value = $wh * 0.001; - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$value); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$value); + $this->wh += ($this->power * $this->interval) / 3600.0; + return $this->wh * 0.001; } } diff --git a/processes/solardirectcalc.php b/processes/solardirectcalc.php index d13a872..fa73745 100644 --- a/processes/solardirectcalc.php +++ b/processes/solardirectcalc.php @@ -1,7 +1,10 @@ "Solar direct calculation", @@ -15,107 +18,9 @@ public function description() { ); } - public function process($params) + protected function row($use, $gen) { - $result = $this->validate($params); - if (!$result["success"]) return $result; - - $use_meta = getmeta($this->dir,$params->use); - $gen_meta = getmeta($this->dir,$params->gen); - - // Find longest interval - $interval = $use_meta->interval; - if ($gen_meta->interval>$interval) $interval = $gen_meta->interval; - // Find latest start time - $start_time = $use_meta->start_time; - if ($gen_meta->start_time>$start_time) $start_time = $gen_meta->start_time; - // Round start time down to nearest interval - $start_time = floor($start_time / $interval) * $interval; - - // Create output feed meta file - $out_meta = new stdClass(); - $out_meta->start_time = $start_time; - $out_meta->interval = $interval; - createmeta($this->dir,$params->out,$out_meta); - $out_meta = getmeta($this->dir,$params->out); - - // Find end time of input feeds - $use_end_time = $use_meta->start_time + ($use_meta->interval * $use_meta->npoints); - $gen_end_time = $gen_meta->start_time + ($gen_meta->interval * $gen_meta->npoints); - - // Start time of this process - if ($params->process_mode=='recent') { - $start_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - } else if ($params->process_mode=='from') { - $start_time = $params->process_start; - if ($start_time<$out_meta->start_time) $start_time = $out_meta->start_time; - // Align to the output interval and stay within the output, no gap - $start_time = $out_meta->start_time + floor(($start_time - $out_meta->start_time) / $out_meta->interval) * $out_meta->interval; - $out_end_time = $out_meta->start_time + ($out_meta->npoints * $out_meta->interval); - if ($start_time>$out_end_time) $start_time = $out_end_time; - } else { - $start_time = $out_meta->start_time; - } - - // End time of this process - $end_time = $use_end_time; - if ($gen_end_time<$end_time) $end_time = $gen_end_time; - - if ($start_time>=$end_time) { - return array("success"=>true, "message"=>"no new data to process"); - } - - // Open input and output feeds - if (!$use_fh = @fopen($this->dir.$params->use.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open consumption feed"); - } - if (!$gen_fh = @fopen($this->dir.$params->gen.".dat", 'rb')) { - return array("success"=>false, "message"=>"could not open generation feed"); - } - if (!$out_fh = @fopen($this->dir.$params->out.".dat", 'c+')) { - return array("success"=>false, "message"=>"could not open output feed"); - } - // Write from start_time: reprocess and from modes overwrite existing data - fseek($out_fh, (($start_time - $out_meta->start_time) / $out_meta->interval) * 4); - - $buffer = ""; - for ($time=$start_time; $time<$end_time; $time+=$interval) - { - $pos_use = floor(($time - $use_meta->start_time) / $use_meta->interval); - $pos_gen = floor(($time - $gen_meta->start_time) / $gen_meta->interval); - - $useval = NAN; - $genval = NAN; - - if ($pos_use>=0 && $pos_use<$use_meta->npoints) { - fseek($use_fh,$pos_use*4); - $tmp = unpack("f",fread($use_fh,4)); - $useval = $tmp[1]; - } - - if ($pos_gen>=0 && $pos_gen<$gen_meta->npoints) { - fseek($gen_fh,$pos_gen*4); - $tmp = unpack("f",fread($gen_fh,4)); - $genval = $tmp[1]; - } - - $directval = NAN; - if (!is_nan($useval) && !is_nan($genval)) { - if ($genval>$useval) $genval = $useval; - $directval = $genval; - } - $buffer .= pack("f",$directval*1.0); - } - - fwrite($out_fh,$buffer); - fclose($out_fh); - fclose($use_fh); - fclose($gen_fh); - - $byteswritten = strlen($buffer); - if ($byteswritten>0) { - updatetimevalue($params->output,$time,$directval); - } - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$time." ".$directval); + if (is_nan($use) || is_nan($gen)) return NAN; + return min($use, $gen); } -} \ No newline at end of file +} diff --git a/readme.md b/readme.md index 3c1dc06..a31b4a6 100644 --- a/readme.md +++ b/readme.md @@ -29,4 +29,10 @@ Processes are tested against small synthetic feeds in a temporary directory, no php tests/run.php # all tests php tests/run.php scalefeed # tests whose name contains scalefeed -Processes that transform each value of a feed extend `MapProcess` (input feed to output feed, e.g. `processes/scalefeed.php`) or `InPlaceProcess` (modify a feed in place, e.g. `processes/to_signed.php`) in `common.php` and only implement `value($value)`. Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. +Most processes extend a base class in `common.php` and only implement the calculation: + +- `MapProcess`: transforms each value of an input feed into an output feed, implement `value($value)`, e.g. `processes/scalefeed.php` +- `InPlaceProcess`: transforms each value of a feed in place, implement `value($value)`, e.g. `processes/to_signed.php` +- `CombineProcess`: combines input feeds on a common timeline into an output feed, implement `row($a, $b, ...)`, e.g. `processes/importcalc.php`, or with `reset($previous)` to accumulate, e.g. `processes/powertokwh.php` + +Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. diff --git a/tests/test_combine.php b/tests/test_combine.php new file mode 100644 index 0000000..bd709ed --- /dev/null +++ b/tests/test_combine.php @@ -0,0 +1,162 @@ +1, "gen"=>2, "out"=>3])); + // Missing generation is treated as no generation + assert_feed(3, 1000, 10, [300, 0, NAN, 300]); +}); + +test("exportcalc", function() { + feed(1, 1000, 10, [500, 100, NAN, 300]); + feed(2, 1000, 10, [200, 300, 100, NAN]); + empty_feed(3); + assert_success(run("exportcalc", ["generation"=>1, "consumption"=>2, "output"=>3])); + // Missing consumption is treated as no consumption + assert_feed(3, 1000, 10, [300, 0, NAN, 300]); +}); + +test("solardirectcalc", function() { + feed(1, 1000, 10, [500, 100, NAN, 300]); + feed(2, 1000, 10, [200, 300, 100, NAN]); + empty_feed(3); + assert_success(run("solardirectcalc", ["use"=>1, "gen"=>2, "out"=>3])); + assert_feed(3, 1000, 10, [200, 100, NAN, NAN]); +}); + +// -------------------------------------------------------------------------------- +// Two inputs, modes +// -------------------------------------------------------------------------------- + +test("mergefeeds recent", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 1000, 10, [3, 4]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + append(1, [5, 6]); + append(2, [7]); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 1000, 10, [2, 3, 6]); +}); + +test("mergefeeds from", function() { + feed(1, 1000, 10, [1, 2, 3]); + feed(2, 1000, 10, [1, 2, 3]); + empty_feed(3); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + feed(1, 1000, 10, [10, 20, 30]); + assert_success(run("mergefeeds", ["feedA"=>1, "feedB"=>2, "output"=>3, "process_mode"=>"from", "process_start"=>1010])); + assert_feed(3, 1000, 10, [1, 11, 16.5]); +}); + +test("addfeeds later start time", function() { + feed(1, 1000, 10, [1, 2, 3, 4]); + feed(2, 1020, 10, [10, 20]); + empty_feed(3); + assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); + assert_feed(3, 1020, 10, [13, 24]); +}); + +// -------------------------------------------------------------------------------- +// Accumulators +// -------------------------------------------------------------------------------- + +test("accumulator recent continues from last value", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + append(1, [3]); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000, "process_mode"=>"recent"])); + assert_feed(2, 1000, 10, [1, 3, 6]); +}); + +test("constantflow_tokwh", function() { + // kWh per hour = 0.001 * vhc * flow * dT = 3.6 * dT, only positive dT, NAN adds nothing + feed(1, 3600, 3600, [20, 20, NAN, 20]); + feed(2, 3600, 3600, [10, 25, 10, 15]); + empty_feed(3); + assert_success(run("constantflow_tokwh", ["vhc"=>1000, "flow"=>3.6, "tint"=>1, "text"=>2, "output"=>3])); + assert_feed(3, 3600, 3600, [36, 36, 36, 54]); + append(1, [20]); + append(2, [10]); + assert_success(run("constantflow_tokwh", ["vhc"=>1000, "flow"=>3.6, "tint"=>1, "text"=>2, "output"=>3, "process_mode"=>"recent"])); + assert_feed(3, 3600, 3600, [36, 36, 36, 54, 90]); +}); + +test("liquidorairflow_tokwh", function() { + // kWh per hour = 0.001 * vhc * flow * dT = 10 * flow + feed(1, 3600, 3600, [1, 2, NAN, 1]); + feed(2, 3600, 3600, [20, 20, 20, 20]); + feed(3, 3600, 3600, [10, 10, 10, 10]); + empty_feed(4); + assert_success(run("liquidorairflow_tokwh", ["vhc"=>1000, "flow"=>1, "tint"=>2, "text"=>3, "output"=>4])); + assert_feed(4, 3600, 3600, [10, 30, 30, 40]); +}); + +// -------------------------------------------------------------------------------- +// Basic formula +// -------------------------------------------------------------------------------- + +test("basic_formula sum and scale", function() { + feed(1, 1000, 10, [1, 2, NAN]); + feed(2, 1000, 10, [10, 20, 30]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1+2*f2", "output"=>4])); + assert_feed(4, 1000, 10, [21, 42, NAN]); +}); + +test("basic_formula max", function() { + feed(1, 1000, 10, [5, 1, NAN]); + feed(2, 1000, 10, [2, 3, 1]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"2*max(f1-f2,0)", "output"=>4])); + assert_feed(4, 1000, 10, [6, 0, NAN]); +}); + +test("basic_formula brackets and division", function() { + feed(1, 1000, 10, [3, 5]); + feed(2, 1000, 10, [2, 4]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"10*(f1-1)+f2/2", "output"=>4])); + assert_feed(4, 1000, 10, [21, 42]); +}); + +test("basic_formula recent", function() { + feed(1, 1000, 10, [1, 2]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4])); + append(1, [3]); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4, "process_mode"=>"recent"])); + assert_feed(4, 1000, 10, [3, 6, 9]); +}); + +test("basic_formula invalid", function() { + feed(1, 1000, 10, [1]); + feed(2, 1000, 10, [1]); + empty_feed(4); + assert_error(run("basic_formula", ["formula"=>"(f1-f2)*(f1-f2)", "output"=>4]), "could not understand"); +}); + +test("basic_formula feed ids that are prefixes of each other", function() { + feed(1, 1000, 10, [1, 2]); + feed(12, 1000, 10, [10, 20]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1+f12", "output"=>4])); + assert_feed(4, 1000, 10, [11, 22]); +}); + +test("basic_formula from", function() { + feed(1, 1000, 10, [1, 2, 3]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4])); + feed(1, 1000, 10, [10, 20, 30]); + assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4, "process_mode"=>"from", "process_start"=>1010])); + assert_feed(4, 1000, 10, [3, 60, 90]); +}); diff --git a/tests/test_processes.php b/tests/test_processes.php index 03f755d..4de9c7b 100644 --- a/tests/test_processes.php +++ b/tests/test_processes.php @@ -41,10 +41,18 @@ assert_feed(2, 1000, 10, [2, 4, 60, 80]); }); -test("scalefeed rejects interval mismatch", function() { +test("scalefeed recent rejects interval mismatch", function() { feed(1, 1000, 10, [1, 2]); feed(2, 1000, 20, [1]); - assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2]), "interval mismatch"); + assert_error(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2, "process_mode"=>"recent"]), "interval mismatch"); + assert_feed(2, 1000, 20, [1]); +}); + +test("scalefeed all replaces output with different interval", function() { + feed(1, 1000, 10, [1, 2]); + feed(2, 500, 20, [9, 9, 9]); + assert_success(run("scalefeed", ["input"=>1, "scale"=>2, "output"=>2])); + assert_feed(2, 1000, 10, [2, 4]); }); test("offsetfeed", function() { @@ -116,7 +124,7 @@ empty_feed(2); assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); assert_feed(2, 3600, 3600, [1, 2, 3, 5]); - assert_redis(2, 3600 * 5, 5); + assert_redis(2, 3600 * 4, 5); }); test("powertokwh recent continues from last value", function() { @@ -137,6 +145,21 @@ assert_feed(2, 3600, 3600, [1, 2, 4, 6]); }); +test("powertokwh spurious power adds no energy and keeps alignment", function() { + feed(1, 3600, 3600, [1000, 5000000, NAN, 1000]); + empty_feed(2); + assert_success(run("powertokwh", ["input"=>1, "output"=>2, "max_power"=>1000000, "min_power"=>-1000000])); + // NAN holds the last valid power + assert_feed(2, 3600, 3600, [1, 1, 2, 3]); +}); + +test("accumulator spurious value is not added and keeps alignment", function() { + feed(1, 1000, 10, [1, 5000, 2]); + empty_feed(2); + assert_success(run("accumulator", ["input"=>1, "output"=>2, "max_value"=>1000, "min_value"=>-1000])); + assert_feed(2, 1000, 10, [1, 1, 3]); +}); + test("accumulator", function() { feed(1, 1000, 10, [1, 2, NAN, 3]); empty_feed(2); @@ -156,12 +179,12 @@ assert_feed(3, 1000, 10, [11, 2, NAN, 44]); }); -test("addfeeds different intervals uses longest", function() { +test("addfeeds different intervals uses longest, averaging shorter", function() { feed(1, 1000, 10, [1, 2, 3, 4]); feed(2, 1000, 20, [10, 20]); empty_feed(3); assert_success(run("addfeeds", ["feedA"=>1, "feedB"=>2, "output"=>3])); - assert_feed(3, 1000, 20, [11, 23]); + assert_feed(3, 1000, 20, [11.5, 23.5]); }); test("addfeeds stops at end of shortest input", function() { From 2cf4f69727a8dcf70c9d0570e8facc6addfda5cd Mon Sep 17 00:00:00 2001 From: Trystan Lea Date: Fri, 9 Oct 2026 23:37:11 +0100 Subject: [PATCH 4/5] Rebuild ModelHelper on FeedReader/FeedWriter, fix simulator resume bugs ModelHelper reads inputs and writes outputs in 8192 datapoint blocks rather than one at a time and keeping all output in memory. On a year of 10s data: carnot_cop_simulator 7.8s to 2.6s, 51MB to 3MB; batterysimulator 6.2s to 4.5s, 228MB to 8MB; solarbatterykwh unchanged at 3MB and slightly faster. Fixes, tested by processing in several runs giving the same output as a single run: - carnot_cop_simulator: first run shifted outputs one interval later, each run counted the step it continued from twice - batterysimulator: each run counted the step it continued from twice in state of charge and kWh totals, off-peak charging stopped at run boundaries (now restored from the stored state of charge) - solarbatterykwh: the 12 hour backcast counted one step twice - runs starting in a gap hold the last valid value from before the run batterysimulator works out the local hour only when it changes. Tests add snapshots of single runs from the previous code, which the new code matches exactly. Co-Authored-By: Claude Opus 5.5 --- common.php | 245 ++++++++++++----- processes/batterysimulator.php | 103 ++++--- processes/carnot_cop_simulator.php | 28 +- processes/solarbatterykwh.php | 54 ++-- readme.md | 2 +- tests/run.php | 53 ++++ tests/snapshots/batterysimulator.json | 128 +++++++++ tests/snapshots/batterysimulator_offpeak.json | 128 +++++++++ tests/snapshots/solarbatterykwh.json | 51 ++++ tests/snapshots/solarbatterykwh_derived.json | 51 ++++ tests/test_simulators.php | 252 ++++++++++++++++++ 11 files changed, 946 insertions(+), 149 deletions(-) create mode 100644 tests/snapshots/batterysimulator.json create mode 100644 tests/snapshots/batterysimulator_offpeak.json create mode 100644 tests/snapshots/solarbatterykwh.json create mode 100644 tests/snapshots/solarbatterykwh_derived.json create mode 100644 tests/test_simulators.php diff --git a/common.php b/common.php index 49446f1..0e3affe 100644 --- a/common.php +++ b/common.php @@ -703,15 +703,31 @@ function bfo($elements, $values): float|int return $sum; } +// Helper for simulation processes that step through time one datapoint at a time with +// several input and output feeds. Inputs are read and outputs written in blocks, each feed +// has a position like a file pointer: read() and write() move it on, seek_to_time() sets it. class ModelHelper { + // Datapoints per input block and per output write. Smaller than POSTPROCESS_BLOCKSIZE as + // values are held as PHP arrays for many feeds at once, 8192 is also slightly faster. + const BLOCKSIZE = 8192; + private $dir; private $params; - private $fh = array(); - private $buffer = array(); - public $meta = array(); - public $value = array(); + private $readers = array(); + private $writers = array(); + // Next position to read or write for each feed + private $pos = array(); + // Block of input values starting at $cache_start + private $cache = array(); + private $cache_start = array(); + private $cache_length = array(); + // Output values not yet passed to the writer, and whether the writer is at $pos + private $pending = array(); + private $pending_length = array(); + private $positioned = array(); + public $meta = array(); public $start_time = false; public $end_time = false; @@ -721,114 +737,201 @@ public function __construct($dir, $params) $this->params = $params; } - private function load($key, $mode) + // Open feed setting $key as an input. The model start and end times are the latest + // start and earliest end of the inputs. Returns false if the setting or feed is missing. + public function input($key) { - $this->meta[$key] = false; - $this->fh[$key] = false; - $this->value[$key] = false; - - // check for valid key if (!isset($this->params->$key)) return false; - $feedid = $this->params->$key; - // load meta - if (!$meta = getmeta($this->dir, $feedid)) return false; - $this->meta[$key] = $meta; - - if (!$this->fh[$key] = @fopen($this->dir . $feedid . ".dat", $mode)) { - echo "ERROR: could not open " . $this->dir . $feedid . ".dat\n"; + try { + $reader = new FeedReader($this->dir, $this->params->$key); + } catch (PostProcessException $e) { return false; } + $this->readers[$key] = $reader; + $this->meta[$key] = $reader->meta; + $this->pos[$key] = 0; + $this->cache[$key] = array(); + $this->cache_start[$key] = 0; + $this->cache_length[$key] = 0; - // Fetch last value - $this->value[$key] = 0.0; - if ($meta->npoints > 0) { - fseek($this->fh[$key], ($meta->npoints - 1) * 4); - $tmp = unpack("f", fread($this->fh[$key], 4)); - if (!is_nan($tmp[1])) $this->value[$key] = 1 * $tmp[1]; - } + if ($this->start_time === false || $reader->meta->start_time > $this->start_time) $this->start_time = $reader->meta->start_time; + if ($this->end_time === false || $reader->meta->end_time < $this->end_time) $this->end_time = $reader->meta->end_time; return true; } - public function input($key) + // Open feed setting $key as an output. Returns false if the setting or feed is missing, + // writes to outputs that are not open are ignored. + public function output($key) { - if (!$result = $this->load($key, 'rb')) return false; - if ($this->start_time === false) $this->start_time = $this->meta[$key]->start_time; - else if ($this->meta[$key]->start_time > $this->start_time) $this->start_time = $this->meta[$key]->start_time; - if ($this->end_time === false) $this->end_time = $this->meta[$key]->end_time; - else if ($this->meta[$key]->end_time < $this->end_time) $this->end_time = $this->meta[$key]->end_time; + if (!isset($this->params->$key)) return false; + try { + $writer = new FeedWriter($this->dir, $this->params->$key); + } catch (PostProcessException $e) { + return false; + } + $this->writers[$key] = $writer; + $this->meta[$key] = $writer->meta; + $this->pos[$key] = 0; + $this->pending[$key] = array(); + $this->pending_length[$key] = 0; + $this->positioned[$key] = false; + $this->cache_start[$key] = 0; + $this->cache_length[$key] = 0; return true; } - public function output($key) + // Set the start time and interval of all outputs, outputs with data must already match + public function set_output_meta($start_time, $interval) { - if (!$this->load($key, 'c+')) return false; - $this->buffer[$key] = ""; - return true; + foreach ($this->writers as $key => $writer) { + $writer->init($start_time, $interval); + $this->meta[$key] = $writer->meta; + } } + // Read the value at the position of $key and move on, $value is returned if missing public function read($key, $value) { - if (!$this->fh[$key]) return $value; - $bytes = fread($this->fh[$key], 4); - if (strlen($bytes) != 4) return $value; - $tmp = unpack("f", $bytes); - if ($tmp[1] !== null && !is_nan($tmp[1])) $value = $tmp[1]; + // Fast path: input value in the current block + if (!isset($this->pos[$key])) return $value; + $i = $this->pos[$key] - $this->cache_start[$key]; + if ($i >= 0 && $i < $this->cache_length[$key]) { + $this->pos[$key]++; + $v = $this->cache[$key][$i]; + return is_nan($v) ? $value : $v; + } + + if (isset($this->writers[$key])) { + $v = $this->read_at($key, $this->time_of($key, $this->pos[$key]), NAN); + $this->pos[$key]++; + $this->positioned[$key] = false; + return is_nan($v) ? $value : $v; + } + if (!isset($this->readers[$key])) return $value; + + // Read the next block + $pos = $this->pos[$key]++; + $npoints = $this->meta[$key]->npoints; + if ($pos < 0 || $pos >= $npoints) return $value; + $this->cache[$key] = $this->readers[$key]->read($pos, min(self::BLOCKSIZE, $npoints - $pos)); + $this->cache_start[$key] = $pos; + $this->cache_length[$key] = count($this->cache[$key]); + $v = $this->cache[$key][0]; + return is_nan($v) ? $value : $v; + } + + // Value of $key at $time without moving its position, $value is returned if missing + public function read_at($key, $time, $value) + { + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $v = $this->writers[$key]->value_at($time); + } else if (isset($this->readers[$key])) { + $meta = $this->meta[$key]; + $pos = floor(($time - $meta->start_time) / $meta->interval); + $v = $this->readers[$key]->read((int) $pos, 1)[0]; + } else { + return $value; + } + return is_nan($v) ? $value : $v; + } + + // Values of $key for $n datapoints from $time without moving its position + public function read_range($key, $time, $n) + { + if (!isset($this->meta[$key])) return array_fill(0, $n, NAN); + $meta = $this->meta[$key]; + $pos = (int) floor(($time - $meta->start_time) / $meta->interval); + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $this->writers[$key]->flush(); + $reader = new FeedReader($this->dir, $this->params->$key); + return $reader->read($pos, $n); + } + return $this->readers[$key]->read($pos, $n); + } + + // Last value of input $key before $time that is not missing, looking back up to one + // block, used to continue holding the last value at the start of a run + public function last_valid($key, $time, $value) + { + if (!isset($this->readers[$key])) return $value; + $meta = $this->meta[$key]; + $end = (int) ceil(($time - $meta->start_time) / $meta->interval); + $start = max(0, $end - POSTPROCESS_BLOCKSIZE); + if ($end <= $start) return $value; + $values = $this->readers[$key]->read($start, $end - $start); + for ($i=count($values)-1; $i>=0; $i--) { + if (!is_nan($values[$i])) return $values[$i]; + } return $value; } public function seek_to_time($time) { - foreach (array_keys($this->fh) as $key) { - if (!$this->meta[$key]) continue; - $pos = floor(($time - $this->meta[$key]->start_time) / $this->meta[$key]->interval); - fseek($this->fh[$key], $pos * 4); - } + foreach (array_keys($this->pos) as $key) $this->seek($key, $time); } public function seek_to_time_inputs($time) { - foreach (array_keys($this->fh) as $key) { - if (!isset($this->buffer[$key])) { - $pos = floor(($time - $this->meta[$key]->start_time) / $this->meta[$key]->interval); - fseek($this->fh[$key], $pos * 4); - } + foreach (array_keys($this->readers) as $key) $this->seek($key, $time); + } + + public function seek_to_time_outputs($time) + { + foreach (array_keys($this->writers) as $key) $this->seek($key, $time); + } + + private function seek($key, $time) + { + $meta = $this->meta[$key]; + if (isset($this->writers[$key])) { + $this->flush_pending($key); + $this->positioned[$key] = false; } + $this->pos[$key] = (int) floor(($time - $meta->start_time) / $meta->interval); } + // Write $value at the position of output $key and move on public function write($key, $value) { - // Skip outputs that are not loaded - if (empty($this->fh[$key])) return; - $this->buffer[$key] .= pack("f", $value); - $this->value[$key] = $value; + // Fast path: output positioned with room in the block + if (isset($this->positioned[$key]) && $this->positioned[$key]) { + $this->pending[$key][] = $value; + $this->pos[$key]++; + if (++$this->pending_length[$key] >= self::BLOCKSIZE) $this->flush_pending($key); + return; + } + if (!isset($this->writers[$key])) return; + $this->writers[$key]->seek($this->time_of($key, $this->pos[$key])); + $this->positioned[$key] = true; + $this->write($key, $value); } - public function set_output_meta($start_time, $interval) + private function flush_pending($key) { - foreach (array_keys($this->buffer) as $key) { - $this->meta[$key]->start_time = $start_time; - $this->meta[$key]->interval = $interval; - if ($this->meta[$key]->end_time == 0) $this->meta[$key]->end_time = $start_time; + if ($this->pending_length[$key]) { + $this->writers[$key]->write_block($this->pending[$key]); + $this->pending[$key] = array(); + $this->pending_length[$key] = 0; } } + private function time_of($key, $pos) + { + return $this->meta[$key]->start_time + $pos * $this->meta[$key]->interval; + } + + // Write remaining data, close outputs and update their last time and value. + // Returns the number of bytes written. public function save_all() { $total_size = 0; - foreach (array_keys($this->buffer) as $key) { - $size = strlen($this->buffer[$key]); - if ($size > 0) { - $feedid = $this->params->$key; - // Write meta data - createmeta($this->dir, $feedid, $this->meta[$key]); - // Write data - fwrite($this->fh[$key], $this->buffer[$key]); - fclose($this->fh[$key]); - $total_size += $size; - // Update feed last time and value - updatetimevalue($feedid, time(), $this->value[$key]); - } + foreach ($this->writers as $key => $writer) { + $this->flush_pending($key); + $total_size += $writer->close(); } + $this->writers = array(); return $total_size; } } diff --git a/processes/batterysimulator.php b/processes/batterysimulator.php index 26edcea..753de6b 100644 --- a/processes/batterysimulator.php +++ b/processes/batterysimulator.php @@ -145,54 +145,42 @@ public function process($p) // Note: implementation only allows for same meta for all output feeds $model->set_output_meta($start_time,$interval); - // Process new data since last run - if (!$recalc) $start_time = $model->meta['grid']->end_time-$interval; + // Continue from the end of the output feeds + if (!$recalc) $start_time = $model->meta['grid']->end_time; if ($start_time<$model->start_time) $start_time = $model->start_time; - if ($start_time==$end_time) { + if ($start_time>=$end_time) { return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - - $solar = 0; - $use = 0; + + // Starting values from the datapoint before the start + $prev = $start_time - $interval; + $charge_kwh = $model->read_at('charge_kwh',$prev,0); + $discharge_kwh = $model->read_at('discharge_kwh',$prev,0); + $import_kwh = $model->read_at('import_kwh',$prev,0); + $export_kwh = $model->read_at('export_kwh',$prev,0); + $solar_to_load_kwh = $model->read_at('solar_to_load_kwh',$prev,0); + $solar_to_grid_kwh = $model->read_at('solar_to_grid_kwh',$prev,0); + $solar_to_battery_kwh = $model->read_at('solar_to_battery_kwh',$prev,0); + $battery_to_load_kwh = $model->read_at('battery_to_load_kwh',$prev,0); + $battery_to_grid_kwh = $model->read_at('battery_to_grid_kwh',$prev,0); + $grid_to_load_kwh = $model->read_at('grid_to_load_kwh',$prev,0); + $grid_to_battery_kwh = $model->read_at('grid_to_battery_kwh',$prev,0); + $soc = $p->capacity * 0.5; - $charge_kwh = 0; - $discharge_kwh = 0; - $import_kwh = 0; - $export_kwh = 0; - $solar_to_load_kwh = 0; - $solar_to_grid_kwh = 0; - $solar_to_battery_kwh = 0; - $battery_to_load_kwh = 0; - $battery_to_grid_kwh = 0; - $grid_to_load_kwh = 0; - $grid_to_battery_kwh = 0; - - // Get starting values - $model->seek_to_time($start_time); - $charge_kwh = $model->read('charge_kwh',$charge_kwh); - $discharge_kwh = $model->read('discharge_kwh',$discharge_kwh); - $import_kwh = $model->read('import_kwh',$import_kwh); - $export_kwh = $model->read('export_kwh',$export_kwh); - $solar_to_load_kwh = $model->read('solar_to_load_kwh',$solar_to_load_kwh); - $solar_to_grid_kwh = $model->read('solar_to_grid_kwh',$solar_to_grid_kwh); - $solar_to_battery_kwh = $model->read('solar_to_battery_kwh',$solar_to_battery_kwh); - $battery_to_load_kwh = $model->read('battery_to_load_kwh',$battery_to_load_kwh); - $battery_to_grid_kwh = $model->read('battery_to_grid_kwh',$battery_to_grid_kwh); - $grid_to_load_kwh = $model->read('grid_to_load_kwh',$grid_to_load_kwh); - $grid_to_battery_kwh = $model->read('grid_to_battery_kwh',$grid_to_battery_kwh); - - if ($model->meta['soc']->npoints) { - $soc = $model->read('soc',$soc); - $soc = $soc*0.01*$p->capacity; - } + $soc_prc = $model->read_at('soc',$prev,NAN); + if (!is_nan($soc_prc)) $soc = $soc_prc*0.01*$p->capacity; + + // Missing values hold the last valid value, including from before this run + $solar = $model->last_valid('solar',$start_time,0); + $use = $model->last_valid('consumption',$start_time,0); - // Reset again $model->seek_to_time($start_time); - - $date->setTimestamp($start_time); + + $date->setTimestamp($prev); $hour = $date->format("H")*1; - $charging_offpeak = false; + $next_hour_time = $start_time; + $charging_offpeak = $this->charging_offpeak_at($model,$p,$date,$prev,$interval); $power_to_kwh = $interval / 3600000.0; // conversion factor from W to kWh for given interval @@ -200,9 +188,14 @@ public function process($p) for ($time=$start_time; $time<$end_time; $time+=$interval) { // $model->seek_to_time_inputs($time); not required as input feed intervals are the same + // Local hour, only worked out when the hour changes $last_hour = $hour; - $date->setTimestamp($time); - $hour = $date->format("H")*1; + if ($time>=$next_hour_time) { + $date->setTimestamp($time); + list($hour,$minute,$second) = explode(":",$date->format("H:i:s")); + $hour = $hour*1; + $next_hour_time = $time - ($minute*60 + $second) + 3600; + } $solar = $model->read('solar',$solar); $use = $model->read('consumption',$use); @@ -385,4 +378,30 @@ public function process($p) $buffersize = $model->save_all(); return array("success"=>true, "message"=>"bytes written: ".($buffersize/1024)." kb"); } + + // Whether off-peak charging is in progress after the datapoint at $prev: the off-peak + // window has started since the start of the data and the state of charge has not + // reached the target at any datapoint since + private function charging_offpeak_at($model, $p, $date, $prev, $interval) + { + if (!$p->has_battery || $p->offpeak_soc_target<=0 || $prev<$model->start_time) return false; + + $date->setTimestamp($prev); + $date->setTime((int) $p->offpeak_start, 0, 0); + if ($date->getTimestamp()>$prev) { + $date->setTimestamp($prev - 86400); + $date->setTime((int) $p->offpeak_start, 0, 0); + } + $window_start = $date->getTimestamp(); + + // First datapoint in the window + $first = $model->start_time + ceil(($window_start - $model->start_time) / $interval) * $interval; + if ($first<$model->start_time || $first>$prev) return false; + + $n = (int) (($prev - $first) / $interval) + 1; + foreach ($model->read_range('soc',$first,$n) as $soc_prc) { + if (!is_nan($soc_prc) && $soc_prc>=$p->offpeak_soc_target) return false; + } + return true; + } } diff --git a/processes/carnot_cop_simulator.php b/processes/carnot_cop_simulator.php index 60a8740..77b9da2 100644 --- a/processes/carnot_cop_simulator.php +++ b/processes/carnot_cop_simulator.php @@ -61,25 +61,21 @@ public function process($p) // Note: implementation only allows for same meta for all output feeds $model->set_output_meta($start_time,$interval); - // Simulation start time - if (!$recalc) $start_time = $model->meta['heatpump_heat_sim']->end_time-$interval; - - if ($start_time==$end_time) { + // Continue from the end of the output feeds + if (!$recalc) $start_time = $model->meta['heatpump_heat_sim']->end_time; + if ($start_time<$model->start_time) $start_time = $model->start_time; + + if ($start_time>=$end_time) { return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - - $power = 0; - $flowT = 0; - $returnT = 0; - $outsideT = 0; - $heat_kwh = 0; - // get cumulative kwh value - $model->seek_to_time($start_time); - if ($model->meta['heatpump_heat_sim']->npoints) { - $heat_kwh = $model->read('heatpump_heat_sim_kwh',$heat_kwh); - } - // reset again + // Cumulative kWh before the start, missing inputs hold the last valid value + $heat_kwh = $model->read_at('heatpump_heat_sim_kwh',$start_time-$interval,0); + $power = $model->last_valid('heatpump_elec',$start_time,0); + $flowT = $model->last_valid('heatpump_flowT',$start_time,0); + $returnT = $model->last_valid('heatpump_returnT',$start_time,0); + $outsideT = $model->last_valid('heatpump_outsideT',$start_time,0); + $model->seek_to_time($start_time); $i=0; diff --git a/processes/solarbatterykwh.php b/processes/solarbatterykwh.php index 1255939..1002993 100644 --- a/processes/solarbatterykwh.php +++ b/processes/solarbatterykwh.php @@ -179,11 +179,6 @@ public function process($p) return array("success"=>true,"message"=>"Nothing to do, data already up to date"); } - $solar = 0; - $use = 0; - $grid = 0; - $battery = 0; - $solar_to_load_kwh = 0; $solar_to_grid_kwh = 0; $solar_to_battery_kwh = 0; @@ -193,16 +188,28 @@ public function process($p) $grid_to_battery_kwh = 0; // $solar_kwh = 0; // used for testing - // Get starting cumulative kWh values - $model->seek_to_time($start_time); - if ($model->meta['solar_to_load_kwh']->npoints) $solar_to_load_kwh = $model->read('solar_to_load_kwh',$solar_to_load_kwh); - if ($model->meta['solar_to_grid_kwh']->npoints) $solar_to_grid_kwh = $model->read('solar_to_grid_kwh',$solar_to_grid_kwh); - if ($model->meta['solar_to_battery_kwh']->npoints) $solar_to_battery_kwh = $model->read('solar_to_battery_kwh',$solar_to_battery_kwh); - if ($model->meta['battery_to_load_kwh']->npoints) $battery_to_load_kwh = $model->read('battery_to_load_kwh',$battery_to_load_kwh); - if ($model->meta['battery_to_grid_kwh']->npoints) $battery_to_grid_kwh = $model->read('battery_to_grid_kwh',$battery_to_grid_kwh); - if ($model->meta['grid_to_load_kwh']->npoints) $grid_to_load_kwh = $model->read('grid_to_load_kwh',$grid_to_load_kwh); - if ($model->meta['grid_to_battery_kwh']->npoints) $grid_to_battery_kwh = $model->read('grid_to_battery_kwh',$grid_to_battery_kwh); - // if ($model->meta['solar_kwh']->npoints) $solar_kwh = $model->read('solar_kwh',$solar_kwh); + // Get starting cumulative kWh values. The value at the start of the output feed is zero, + // after that the value at each slot includes the datapoint at the start of the slot, in + // which case processing continues from the next datapoint so that it is not counted twice. + $first_time = $start_time; + $start_slot = floor($start_time / $output_interval) * $output_interval; + if ($start_slot > $model->meta['grid_to_load_kwh']->start_time) { + $solar_to_load_kwh = $model->read_at('solar_to_load_kwh',$start_slot,$solar_to_load_kwh); + $solar_to_grid_kwh = $model->read_at('solar_to_grid_kwh',$start_slot,$solar_to_grid_kwh); + $solar_to_battery_kwh = $model->read_at('solar_to_battery_kwh',$start_slot,$solar_to_battery_kwh); + $battery_to_load_kwh = $model->read_at('battery_to_load_kwh',$start_slot,$battery_to_load_kwh); + $battery_to_grid_kwh = $model->read_at('battery_to_grid_kwh',$start_slot,$battery_to_grid_kwh); + $grid_to_load_kwh = $model->read_at('grid_to_load_kwh',$start_slot,$grid_to_load_kwh); + $grid_to_battery_kwh = $model->read_at('grid_to_battery_kwh',$start_slot,$grid_to_battery_kwh); + // $solar_kwh = $model->read_at('solar_kwh',$start_slot,$solar_kwh); + $first_time = $start_time + $interval; + } + + // Missing values hold the last valid value, including from before this run + $solar = $this->valid_power($model->last_valid('solar',$first_time,0), 0); + $use = $this->valid_power($model->last_valid('use',$first_time,0), 0); + $grid = $this->valid_power($model->last_valid('grid',$first_time,0), 0); + $battery = $this->valid_power($model->last_valid('battery',$first_time,0), 0); $starting_values = array( "solar_to_load_kwh"=>number_format($solar_to_load_kwh, 3, '.', '')*1, @@ -214,8 +221,18 @@ public function process($p) "grid_to_battery_kwh"=>number_format($grid_to_battery_kwh, 3, '.', '')*1 ); - // Reset again - $model->seek_to_time($start_time); + // Write the starting values at the start slot, later slots follow + $model->seek_to_time_outputs($start_time); + $model->write('solar_to_load_kwh',$solar_to_load_kwh); + $model->write('solar_to_grid_kwh',$solar_to_grid_kwh); + $model->write('solar_to_battery_kwh',$solar_to_battery_kwh); + $model->write('battery_to_load_kwh',$battery_to_load_kwh); + $model->write('battery_to_grid_kwh',$battery_to_grid_kwh); + $model->write('grid_to_load_kwh',$grid_to_load_kwh); + $model->write('grid_to_battery_kwh',$grid_to_battery_kwh); + // $model->write('solar_kwh',$solar_kwh); // used for testing + + $model->seek_to_time_inputs($first_time); $power_to_kwh = $interval / 3600000.0; // conversion factor from W to kWh for given interval @@ -229,7 +246,7 @@ public function process($p) $dp_per_feed_written = 0; - for ($time=$start_time; $time<$end_time; $time+=$interval) + for ($time=$first_time; $time<$end_time; $time+=$interval) { $solar = $this->valid_power($model->read('solar',$solar), $solar); $use = $this->valid_power($model->read('use',$use), $use); @@ -353,7 +370,6 @@ public function process($p) } echo "\n"; - $model->seek_to_time($start_time+$output_interval); $buffersize = $model->save_all(); $elapsed_time = microtime(true) - $process_start_time; diff --git a/readme.md b/readme.md index a31b4a6..a16535d 100644 --- a/readme.md +++ b/readme.md @@ -35,4 +35,4 @@ Most processes extend a base class in `common.php` and only implement the calcul - `InPlaceProcess`: transforms each value of a feed in place, implement `value($value)`, e.g. `processes/to_signed.php` - `CombineProcess`: combines input feeds on a common timeline into an output feed, implement `row($a, $b, ...)`, e.g. `processes/importcalc.php`, or with `reset($previous)` to accumulate, e.g. `processes/powertokwh.php` -Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. +Simulations that step through time with several inputs and outputs use `ModelHelper`, e.g. `processes/batterysimulator.php`. Other processes can use `FeedReader`, `FeedWriter` and `resolve_start_time()` for block based feed access, see `processes/backfillfeed.php` for an example. diff --git a/tests/run.php b/tests/run.php index e1856b5..19b4448 100644 --- a/tests/run.php +++ b/tests/run.php @@ -111,6 +111,24 @@ function same_value($expected, $actual) function assert_equal($expected, $actual, $message = "") { + // Long arrays: report the first difference + if (is_array($expected) && is_array($actual) && max(count($expected), count($actual)) > 20) { + $prefix = ($message ? "$message: " : ""); + foreach ($expected as $i => $value) { + if (!array_key_exists($i, $actual)) { + throw new TestFailure($prefix."expected ".count($expected)." values, got ".count($actual)); + } + if (!same_value($value, $actual[$i])) { + $from = max(0, $i - 2); + throw new TestFailure($prefix."first difference at index $i of ".count($expected).", expected " + .fmt(array_slice($expected, $from, 5))." got ".fmt(array_slice($actual, $from, 5))." from index $from"); + } + } + if (count($actual) != count($expected)) { + throw new TestFailure($prefix."expected ".count($expected)." values, got ".count($actual)); + } + return; + } $ok = is_array($expected) ? is_array($actual) && count($expected) == count($actual) && count(array_filter(array_map('same_value', $expected, $actual))) == count($expected) @@ -147,6 +165,41 @@ function assert_redis($id, $time, $value) assert_equal($value, $redis->hashes["feed:$id"]["value"], "redis feed:$id value"); } +// Compare feeds with a snapshot in tests/snapshots/$name.json. A missing snapshot is +// created from the current output, delete a snapshot to update it after an intended change. +function assert_snapshot($name, $feeds) +{ + $snapshot = array(); + foreach ($feeds as $key => $id) { + $meta = meta($id); + $values = values($id); + $last = count($values) ? end($values) : NAN; + $snapshot[$key] = array( + "start_time"=>$meta->start_time, + "interval"=>$meta->interval, + "npoints"=>count($values), + "md5"=>count($values) ? md5(pack("f*", ...$values)) : "", + "last"=>is_nan($last) ? "NAN" : sprintf("%.6g", $last) + ); + } + $json = json_encode($snapshot, JSON_PRETTY_PRINT)."\n"; + $file = __DIR__."/snapshots/$name.json"; + if (!file_exists($file)) { + if (!is_dir(__DIR__."/snapshots")) mkdir(__DIR__."/snapshots"); + file_put_contents($file, $json); + echo " created snapshot $name\n"; + return; + } + $expected = json_decode(file_get_contents($file), true); + foreach ($expected as $key => $feed) { + if (!isset($snapshot[$key])) throw new TestFailure("snapshot $name: missing feed $key"); + if ($snapshot[$key] != $feed) { + throw new TestFailure("snapshot $name: feed $key expected ".json_encode($feed).", got ".json_encode($snapshot[$key])); + } + } + if (count($snapshot) != count($expected)) throw new TestFailure("snapshot $name: feed count differs"); +} + // -------------------------------------------------------------------------------- // Run // -------------------------------------------------------------------------------- diff --git a/tests/snapshots/batterysimulator.json b/tests/snapshots/batterysimulator.json new file mode 100644 index 0000000..296b30f --- /dev/null +++ b/tests/snapshots/batterysimulator.json @@ -0,0 +1,128 @@ +{ + "soc": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "cd6a1e4ecc83120ab9a53c5e35227276", + "last": "100" + }, + "power": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "bf322ec81f1dc37f54b67ea394429fae", + "last": "0" + }, + "charge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "4028a6ecf3248fdd2488badffad67d31", + "last": "0" + }, + "discharge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "04ef553a20961d917f816362b98430ea", + "last": "0" + }, + "charge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6c5c1f96e3724a77f882d7b72e0a18cc", + "last": "23.5121" + }, + "discharge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "fb72751c106bf7daa8e1c50343a0df1f", + "last": "19.1296" + }, + "grid": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3357c9c34d706cff77d54e444ef49feb", + "last": "-2423.78" + }, + "import": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "e32572994d19a22c6ffd81b020d351af", + "last": "0" + }, + "export": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "5077770a1155197f29c04fe066dc1655", + "last": "2423.78" + }, + "import_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "c9389f3212b1d6dc49eb135fef4ef616", + "last": "42.0285" + }, + "export_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "aee327d1845e67a971fa1d5c0d46be22", + "last": "190.772" + }, + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6ca9150b4d120e427be3faa614d79eb5", + "last": "54.4157" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "aee327d1845e67a971fa1d5c0d46be22", + "last": "190.772" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6c5c1f96e3724a77f882d7b72e0a18cc", + "last": "23.5121" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "fb72751c106bf7daa8e1c50343a0df1f", + "last": "19.1296" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d948f712fa329203f590e91cf6dd3e3e", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "c9389f3212b1d6dc49eb135fef4ef616", + "last": "42.0285" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d948f712fa329203f590e91cf6dd3e3e", + "last": "0" + } +} diff --git a/tests/snapshots/batterysimulator_offpeak.json b/tests/snapshots/batterysimulator_offpeak.json new file mode 100644 index 0000000..ac39394 --- /dev/null +++ b/tests/snapshots/batterysimulator_offpeak.json @@ -0,0 +1,128 @@ +{ + "soc": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "35e84c7db42afeb7a971cd378e13dca6", + "last": "100" + }, + "power": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "f5caf36755990d82d104d0832f929798", + "last": "0" + }, + "charge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "5de56e2e86f1d8bb7fe3e7bbca454012", + "last": "0" + }, + "discharge": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "ab80ed81d948598172e80e78a6c5b339", + "last": "0" + }, + "charge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "605ab622937a327b1202b0e5ebdda366", + "last": "54.4838" + }, + "discharge_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "cece123df4c9a0c90cba2c9ee8d1c5af", + "last": "47.0816" + }, + "grid": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "bb1792f735bfecd52fbf87962bb1eefa", + "last": "-2423.78" + }, + "import": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "a0efab22cd8180eed22a9c8c4fc0c613", + "last": "0" + }, + "export": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3ad7796a94488ce085d395c49b115996", + "last": "2423.78" + }, + "import_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "59b2cd325adc18b0160321cfd1d85744", + "last": "101.801" + }, + "export_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d97daf00275d48faf275a8dd7ddb1d0f", + "last": "247.525" + }, + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "6ca9150b4d120e427be3faa614d79eb5", + "last": "54.4157" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "50127ab39c65b3780ddaa04e377027bc", + "last": "201.988" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "3fb1a75cf58fe7f5c40348d9da704bde", + "last": "12.2963" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8c16a4907b1ad921a64d4d994cfa995b", + "last": "1.54472" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "d2f1398575efafe623727270a0165645", + "last": "45.5369" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8018302a2893afee54cb082a07237887", + "last": "59.6134" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 10, + "npoints": 100000, + "md5": "8673272af7085e080952e7953ed82a7a", + "last": "42.1875" + } +} diff --git a/tests/snapshots/solarbatterykwh.json b/tests/snapshots/solarbatterykwh.json new file mode 100644 index 0000000..7fa948a --- /dev/null +++ b/tests/snapshots/solarbatterykwh.json @@ -0,0 +1,51 @@ +{ + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "9cd57d52b20f34b231e58ff7f7581bb6", + "last": "54.4076" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40219b1b54dba3bae1131f77937ecc68", + "last": "24.9939" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40cdcfccc2a1d85b4744612c7da077fa", + "last": "189.231" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "c484775c3dacc31ee3741093835be1d4", + "last": "30.381" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "0181a3b838bb4acc9a89b48ef0b842b1", + "last": "30.7772" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + } +} diff --git a/tests/snapshots/solarbatterykwh_derived.json b/tests/snapshots/solarbatterykwh_derived.json new file mode 100644 index 0000000..b1a7028 --- /dev/null +++ b/tests/snapshots/solarbatterykwh_derived.json @@ -0,0 +1,51 @@ +{ + "solar_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "9cd57d52b20f34b231e58ff7f7581bb6", + "last": "54.4076" + }, + "solar_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40219b1b54dba3bae1131f77937ecc68", + "last": "24.9939" + }, + "solar_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "40cdcfccc2a1d85b4744612c7da077fa", + "last": "189.231" + }, + "battery_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "c484775c3dacc31ee3741093835be1d4", + "last": "30.381" + }, + "battery_to_grid_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "b4873e51585912e01ad8ec2379310553", + "last": "0" + }, + "grid_to_load_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "279b40ea1a8bab53b8fdf25db84f56af", + "last": "30.7772" + }, + "grid_to_battery_kwh": { + "start_time": 1711324800, + "interval": 900, + "npoints": 1112, + "md5": "e22b612ab8c181daf962f0132c4296e7", + "last": "1.34542e-6" + } +} diff --git a/tests/test_simulators.php b/tests/test_simulators.php new file mode 100644 index 0000000..f1e640b --- /dev/null +++ b/tests/test_simulators.php @@ -0,0 +1,252 @@ +1, "heatpump_flowT"=>2, "heatpump_returnT"=>3, "heatpump_outsideT"=>4, + "condenser_offset"=>4.0, "refrigerant_offset"=>-6.0, + "practical_efficiency_factor"=>0.5, "running_power_threshold"=>100.0, + "heatpump_cop_sim"=>11, "heatpump_heat_sim"=>12, "heatpump_heat_sim_kwh"=>13, "heatpump_flowrate_sim"=>14 + ), $extra); +} + +test("carnot_cop_simulator", function() { + // Tc = 35+4+273 = 312, Tr = 5-6+273 = 272, COP = 0.5 * 312 / 40 = 3.9 + // Second datapoint is below the running power threshold + feed(1, 3600, 3600, [1000, 50]); + feed(2, 3600, 3600, [35, 35]); + feed(3, 3600, 3600, [30, 30]); + feed(4, 3600, 3600, [5, 5]); + foreach ([11, 12, 13, 14] as $id) empty_feed($id); + + assert_success(run("carnot_cop_simulator", carnot_params())); + assert_feed(11, 3600, 3600, [3.9, 0]); + assert_feed(12, 3600, 3600, [3900, 0]); + assert_feed(13, 3600, 3600, [3.9, 3.9]); + // flow rate m3/h = heat / (4150 * dT) * 3.6 + assert_feed(14, 3600, 3600, [3900 / (4150 * 5) * 3.6, 0]); +}); + +test("carnot_cop_simulator continues kwh", function() { + feed(1, 3600, 3600, [1000]); + feed(2, 3600, 3600, [35]); + feed(3, 3600, 3600, [30]); + feed(4, 3600, 3600, [5]); + foreach ([11, 12, 13, 14] as $id) empty_feed($id); + assert_success(run("carnot_cop_simulator", carnot_params())); + append(1, [1000]); + append(2, [35]); + append(3, [30]); + append(4, [5]); + assert_success(run("carnot_cop_simulator", carnot_params())); + assert_feed(13, 3600, 3600, [3.9, 7.8]); +}); + +// -------------------------------------------------------------------------------- +// Battery simulator +// -------------------------------------------------------------------------------- + +function battery_params($extra = array()) +{ + $params = array( + "solar"=>1, "consumption"=>2, + "has_solar"=>1, "has_battery"=>1, "capacity"=>4.4, "max_charge_rate"=>2500.0, "max_discharge_rate"=>2500.0, + "round_trip_efficiency"=>0.9, "timezone"=>"Europe/London", "offpeak_soc_target"=>0, "offpeak_start"=>3, + "discharge_start"=>0, "discharge_end"=>24, "peak_export_enabled"=>0, "peak_export_start"=>16, "peak_export_end"=>19 + ); + $id = 11; + foreach (battery_outputs() as $key) $params[$key] = $id++; + return array_merge($params, $extra); +} + +function battery_outputs() +{ + return array("soc", "power", "charge", "discharge", "charge_kwh", "discharge_kwh", + "grid", "import", "export", "import_kwh", "export_kwh", + "solar_to_load_kwh", "solar_to_grid_kwh", "solar_to_battery_kwh", "battery_to_load_kwh", + "battery_to_grid_kwh", "grid_to_load_kwh", "grid_to_battery_kwh"); +} + +function battery_output_ids() +{ + $ids = array(); + $id = 11; + foreach (battery_outputs() as $key) $ids[$key] = $id++; + return $ids; +} + +test("batterysimulator without battery", function() { + feed(1, 3600, 3600, [0, 3000, 1000]); + feed(2, 3600, 3600, [500, 1000, 1000]); + foreach (battery_output_ids() as $id) empty_feed($id); + assert_success(run("batterysimulator", battery_params(["has_battery"=>0]))); + $ids = battery_output_ids(); + assert_feed($ids["grid"], 3600, 3600, [500, -2000, 0]); + assert_feed($ids["import_kwh"], 3600, 3600, [0.5, 0.5, 0.5]); + assert_feed($ids["export_kwh"], 3600, 3600, [0, 2, 2]); + assert_feed($ids["solar_to_load_kwh"], 3600, 3600, [0, 1, 2]); +}); + +test("batterysimulator charges from excess solar and discharges to load", function() { + // 1 kWh battery, 100% efficient, starts at 50% + feed(1, 3600, 3600, [1000, 0]); + feed(2, 3600, 3600, [0, 1000]); + foreach (battery_output_ids() as $id) empty_feed($id); + assert_success(run("batterysimulator", battery_params(["capacity"=>1, "round_trip_efficiency"=>1]))); + $ids = battery_output_ids(); + // Charge limited by capacity: 0.5 kWh, then discharge 1 kWh + assert_feed($ids["soc"], 3600, 3600, [100, 0]); + assert_feed($ids["charge"], 3600, 3600, [500, 0]); + assert_feed($ids["discharge"], 3600, 3600, [0, 1000]); + assert_feed($ids["grid"], 3600, 3600, [-500, 0]); +}); + +// -------------------------------------------------------------------------------- +// Solar battery kWh flows +// -------------------------------------------------------------------------------- + +function solarbattery_output_ids() +{ + return array("solar_to_load_kwh"=>11, "solar_to_grid_kwh"=>12, "solar_to_battery_kwh"=>13, + "battery_to_load_kwh"=>14, "battery_to_grid_kwh"=>15, "grid_to_load_kwh"=>16, "grid_to_battery_kwh"=>17); +} + +function solarbattery_params($extra = array()) +{ + return array_merge(array("solar"=>1, "use"=>2, "grid"=>3, "battery"=>4, "strategy"=>"Solar first"), solarbattery_output_ids(), $extra); +} + +// -------------------------------------------------------------------------------- +// Incremental runs give the same output as a single run +// -------------------------------------------------------------------------------- + +// Synthetic input data from 2024-03-25 for $n steps of $interval, includes the change to +// summer time on 2024-03-31 and gaps of missing data +function sim_inputs($n, $interval = 10, $offset = 0) +{ + $start = 1711324800 + $offset; + $data = array("solar"=>[], "use"=>[], "flowT"=>[], "returnT"=>[], "outsideT"=>[], "elec"=>[], "battery"=>[], "grid"=>[]); + for ($i=0; $i<$n; $i++) { + $hour = (($start + $i * $interval) % 86400) / 3600; + $solar = max(0, 3000 * sin(($hour - 6) / 12 * M_PI)); + $use = 250 + (($i * 7919) % 1500) * (($i % 37) < 5 ? 1 : 0.1); + $battery = $solar > $use ? -min(2000, $solar - $use) : min(1500, $use - $solar) * 0.5; + $data["solar"][] = $solar; + $data["use"][] = $use; + $data["battery"][] = $battery; + $data["grid"][] = $use - $solar - $battery; + $data["elec"][] = ($i % 300) < 200 ? 800 + ($i % 50) * 10 : 20; + $data["flowT"][] = 35 + ($i % 120) / 20; + $data["returnT"][] = 31 + ($i % 120) / 25; + $data["outsideT"][] = 5 + 4 * sin($hour / 24 * 2 * M_PI); + } + // Gaps of missing data + foreach ($data as $key => $values) { + for ($i=1000; $i<1050 && $i<$n; $i++) $data[$key][$i] = NAN; + } + return array($start, $interval, $data); +} + +// Write the inputs up to $n datapoints into feeds $ids (key => feed id) +function sim_feeds($inputs, $ids, $n) +{ + list($start, $interval, $data) = $inputs; + foreach ($ids as $key => $id) feed($id, $start, $interval, array_slice($data[$key], 0, $n)); +} + +// Run $process on all data at once and in parts of $parts datapoints, compare the outputs +function assert_incremental($process, $params, $input_ids, $output_ids, $n, $parts, $offset = 0) +{ + $inputs = sim_inputs($n, 10, $offset); + + sim_feeds($inputs, $input_ids, $n); + foreach ($output_ids as $id) empty_feed($id); + assert_success(run($process, $params)); + $single = array(); + foreach ($output_ids as $key => $id) $single[$key] = values($id); + + foreach ($output_ids as $id) empty_feed($id); + foreach ($parts as $part) { + sim_feeds($inputs, $input_ids, $part); + assert_success(run($process, $params)); + } + foreach ($output_ids as $key => $id) { + assert_equal($single[$key], values($id), "$key"); + } +} + +test("carnot_cop_simulator incremental", function() { + assert_incremental("carnot_cop_simulator", carnot_params(), + ["elec"=>1, "flowT"=>2, "returnT"=>3, "outsideT"=>4], + ["cop"=>11, "heat"=>12, "kwh"=>13, "flowrate"=>14], 3000, [1000, 1001, 2500, 3000]); +}); + +test("batterysimulator incremental", function() { + assert_incremental("batterysimulator", battery_params(), + ["solar"=>1, "use"=>2], battery_output_ids(), 3000, [1000, 1001, 2500, 3000]); +}); + +test("batterysimulator incremental with run boundaries during off-peak charging", function() { + // Off-peak window starts at 02:00, datapoint 720 each day, 8640 datapoints per day + assert_incremental("batterysimulator", battery_params([ + "discharge_start"=>7, "discharge_end"=>22, "offpeak_soc_target"=>80, "offpeak_start"=>2, + "peak_export_enabled"=>1, "peak_export_start"=>16, "peak_export_end"=>19 + ]), ["solar"=>1, "use"=>2], battery_output_ids(), 20000, [720, 721, 800, 9400, 9401, 9500, 20000]); +}); + +test("solarbatterykwh incremental", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 20000, [5000, 5001, 15000, 20000]); +}); + +test("solarbatterykwh incremental, inputs not starting on a slot", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 20000, [3000, 15000, 20000], 130); +}); + +test("solarbatterykwh incremental, short runs within the backcast", function() { + assert_incremental("solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids(), 6000, [100, 2000, 2001, 4000, 6000]); +}); + +// -------------------------------------------------------------------------------- +// Snapshots of single runs over 100k datapoints, more than one block, includes the +// change to summer time +// -------------------------------------------------------------------------------- + +function snapshot_run($name, $process, $params, $input_ids, $output_ids) +{ + sim_feeds(sim_inputs(100000), $input_ids, 100000); + foreach ($output_ids as $id) empty_feed($id); + assert_success(run($process, $params)); + assert_snapshot($name, $output_ids); +} + +test("batterysimulator snapshot", function() { + snapshot_run("batterysimulator", "batterysimulator", battery_params(["discharge_start"=>7, "discharge_end"=>22]), + ["solar"=>1, "use"=>2], battery_output_ids()); +}); + +test("batterysimulator snapshot offpeak and peak export", function() { + snapshot_run("batterysimulator_offpeak", "batterysimulator", battery_params([ + "discharge_start"=>7, "discharge_end"=>22, "offpeak_soc_target"=>80, "offpeak_start"=>2, + "peak_export_enabled"=>1, "peak_export_start"=>16, "peak_export_end"=>19 + ]), ["solar"=>1, "use"=>2], battery_output_ids()); +}); + +test("solarbatterykwh snapshot solar first", function() { + snapshot_run("solarbatterykwh", "solarbatterykwh", solarbattery_params(), + ["solar"=>1, "use"=>2, "grid"=>3, "battery"=>4], solarbattery_output_ids()); +}); + +test("solarbatterykwh snapshot battery first derived grid", function() { + $params = solarbattery_params(["strategy"=>"Battery first"]); + unset($params["grid"]); + snapshot_run("solarbatterykwh_derived", "solarbatterykwh", $params, + ["solar"=>1, "use"=>2, "battery"=>4], solarbattery_output_ids()); +}); From 9a839e121068ae63a83af0144a10eca9952466e3 Mon Sep 17 00:00:00 2001 From: Trystan Lea Date: Fri, 9 Oct 2026 23:48:38 +0100 Subject: [PATCH 5/5] basic_formula: evaluate formulas a block at a time, fix quirks The parsed formula is evaluated one operation at a time over a block of datapoints rather than for each datapoint separately, 5-7x faster on 1M datapoints. A test compares the output byte for byte with a per datapoint reference evaluation on random data with gaps and zeros. Fixes: - division by zero gives a missing value rather than 0 - a literal 0 is kept, f1*0 was evaluated as f1*1 - result message no longer converts a NAN last value to a string Parsing is split out into parse_formula(), ftoa() moves from common.php to basic_formula.php and bfo() is replaced by the block evaluator. Co-Authored-By: Claude Opus 5.5 --- common.php | 80 ++++-------------------- processes/backfillfeed.php | 2 +- processes/basic_formula.php | 120 ++++++++++++++++++++++++++---------- tests/test_combine.php | 92 +++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 103 deletions(-) diff --git a/common.php b/common.php index 0e3affe..f398f63 100644 --- a/common.php +++ b/common.php @@ -250,6 +250,16 @@ public function read_time($start_time, $interval, $n) } } +// Result message for a process that has written to $writer +function written_message($bytes, $writer) +{ + $message = "bytes written: ".$bytes; + if ($writer->last_time() !== false) { + $message .= ", last time value: ".$writer->last_time()." ".$writer->last_value(); + } + return $message; +} + class FeedWriter { public $id; @@ -484,7 +494,7 @@ public function process($params) } catch (PostProcessException $e) { return array("success"=>false, "message"=>$e->getMessage()); } - return array("success"=>true, "message"=>"bytes written: ".$bytes.", last time value: ".$output->last_time()." ".$output->last_value()); + return array("success"=>true, "message"=>written_message($bytes, $output)); } } @@ -633,74 +643,8 @@ public function process($params) } catch (PostProcessException $e) { return array("success"=>false, "message"=>$e->getMessage()); } - return array("success"=>true, "message"=>"bytes written: ".$bytes.", last time value: ".$output->last_time()." ".$output->last_value()); - } -} - -/* -format an array for the formula engine -$b is the result of a preg_match on a formula chunk with the regexp /($Xop)?($Xnbr)?($Xf)?/ -$Xop : regexp for operator -$Xnbr : regexp for float or int value -$Xf : regexp for a feed -cf process basic_formula for more details on $Xop,$Xnbr,$Xf -ouputs a formula element as a 3 elements vector : -0 -> type of data ie "feed" ou "value" -1 -> operator -2 -> value or feed number -*/ -function ftoa($b): array -{ - $c = []; - //print_r($b); - if (sizeof(value: $b) == 4) { - $c[0] = "feed"; - $c[2] = intval(value: substr(string: $b[3], offset: 1)); - } else { - $c[0] = "value"; - $c[2] = $b[2]; - } - $c[1] = $b[1]; - if (!$c[2]) $c[2] = 1; - if (!$c[1]) $c[1] = '+'; - return $c; -} - -/* -output of a basic formula for one timestep -the formula is described by the $elements array > a third dimensionnal array > datas are on level 2 -for a given level 1, we operate multiplication/division within level 2 elements and give a sign to the result -the final result is the addition of the whole -a formula element is a 3 elements vector : -0 -> type of data ie "feed" ou "value" -1 -> operator -2 -> value or feed number -values : feed values for this timestep keyed by feed number -*/ -function bfo($elements, $values): float|int -{ - $s = []; - foreach ($elements as $element) { - $product = []; - foreach ($element as $e) { - $value = NAN; - if ($e[0] == "feed") $value = $values[$e[2]]; - if ($e[0] == "value") $value = $e[2]; - if (!is_nan(num: $value) && $value != 0) { - if ($e[1] == "/") $value = 1 / $value; - if ($e[1] == "-") $value = -$value; - } - $product[] = $value; - } - if (!in_array(needle: NAN, haystack: $product)) { - $s[] = array_product(array: $product); - } else $s[] = NAN; + return array("success"=>true, "message"=>written_message($bytes, $output)); } - if (!in_array(needle: NAN, haystack: $s)) { - $sum = array_sum(array: $s); - } else $sum = NAN; - - return $sum; } // Helper for simulation processes that step through time one datapoint at a time with diff --git a/processes/backfillfeed.php b/processes/backfillfeed.php index 500a788..887dddd 100644 --- a/processes/backfillfeed.php +++ b/processes/backfillfeed.php @@ -51,6 +51,6 @@ public function process($params) } $byteswritten = $target->close(); - return array("success"=>true, "message"=>"bytes written: ".$byteswritten.", last time value: ".$target->last_time()." ".$target->last_value()); + return array("success"=>true, "message"=>written_message($byteswritten, $target)); } } diff --git a/processes/basic_formula.php b/processes/basic_formula.php index 5114a4f..80f2f98 100644 --- a/processes/basic_formula.php +++ b/processes/basic_formula.php @@ -1,5 +1,33 @@ type of data ie "feed" ou "value" +1 -> operator +2 -> value or feed number +a missing value is 1, e.g. a lone operator +*/ +function ftoa($b): array +{ + $c = []; + if (sizeof(value: $b) == 4) { + $c[0] = "feed"; + $c[2] = intval(value: substr(string: $b[3], offset: 1)); + } else { + $c[0] = "value"; + $c[2] = $b[2]; + if ($c[2] === "") $c[2] = "1"; + } + $c[1] = $b[1]; + if (!$c[1]) $c[1] = '+'; + return $c; +} + function formula_to_array($formula_string, $reg_exp): array { //process a formula string to an array @@ -7,8 +35,7 @@ function formula_to_array($formula_string, $reg_exp): array foreach (preg_split(pattern: "@(?=(\*|\/))@",subject: $formula_string) as $piece) { if (preg_match(pattern: $reg_exp, subject: $piece, matches: $b)){ if (count(value: $b)>2){ - $c=ftoa(b: $b); - if($c[2]) $fly[]=$c; + $fly[]=ftoa(b: $b); } } } @@ -44,11 +71,21 @@ public function description(): array { public function process($params): array { - $DEBUG = 0; $result = $this->validate(processitem: $params); if (!$result["success"]) return $result; - $dir = $this->dir; + if (!$parsed = $this->parse_formula(formula: $params->formula)) { + return ["success"=>false, "message"=>"could not understand your formula SORRY...."]; + } + $this->elements=$parsed["elements"]; + $this->feed_ids=$parsed["feed_ids"]; + return parent::process($params); + } + + // parse a formula into elements and the feed ids it uses, false if not understood + public function parse_formula($formula) + { + $DEBUG = 0; // regular expression to recognize a float or int value // use of ?: not to perturb things in creating useless references $Xnbr="(?:[0-9]+\.[0-9]+|[0-9]+)"; @@ -76,8 +113,6 @@ public function process($params): array $nbf=count(value: $functions); - //retrieving the formula - $formula=$params->formula; //removing useless brackets $pos=0; $original=$formula; @@ -212,9 +247,7 @@ public function process($params): array print $original_copy; print "\n\n"; } - if ($original_copy!="") { - return ["success"=>false, "message"=>"could not understand your formula SORRY...."]; - } + if ($original_copy!="") return false; $elements=[]; foreach ($array as $a){ @@ -258,9 +291,7 @@ public function process($params): array } if ($DEBUG) print_r(value: $elements); - $this->elements=$elements; - $this->feed_ids=$feed_ids; - return parent::process($params); + return ["elements"=>$elements, "feed_ids"=>$feed_ids]; } protected function input_feeds($params) @@ -270,36 +301,57 @@ protected function input_feeds($params) return $feeds; } + // formula values for a block of datapoints, the formula is evaluated one operation at a time + // over the whole block. missing values and division by zero give NAN protected function block($blocks) { - $out=[]; $n=count(value: reset($blocks)); - for ($i=0;$i<$n;$i++) { - $values=[]; - foreach ($blocks as $id=>$block) $values[$id]=$block[$i]; - $out[]=$this->evaluate(values: $values); + $total=array_fill(0,$n,0); + foreach ($this->elements as $element) { + $s1=$this->sum_block(terms: [$element->scale], blocks: $blocks, n: $n); + $s2=$this->sum_block(terms: $element->formula, blocks: $blocks, n: $n); + $s3=$this->sum_block(terms: [$element->scale_right], blocks: $blocks, n: $n); + if ($element->function=="max") { + $arg2_constant=is_numeric(value: $element->arg2); + $arg2=$arg2_constant ? null : $this->sum_block(terms: $element->arg2, blocks: $blocks, n: $n); + for ($i=0;$i<$n;$i++) { + $a2=$arg2_constant ? $element->arg2 : $arg2[$i]; + if (is_nan($s1[$i]) || is_nan($s2[$i]) || is_nan($s3[$i]) || is_nan($a2)) $total[$i]=NAN; + else $total[$i]+=$s1[$i]*max($s2[$i],$a2)*$s3[$i]; + } + } else { + for ($i=0;$i<$n;$i++) $total[$i]+=$s1[$i]*$s2[$i]*$s3[$i]; + } } - return $out; + return $total; } - // value of the formula for one timestep, $values are the feed values keyed by feed id - private function evaluate($values) + // for each datapoint of the block, the sum of the products of each term + // a term is a list of factors [type "feed" or "value", operator, feed id or value] + private function sum_block($terms,$blocks,$n) { - $s=[]; - foreach($this->elements as $element){ - $s1=bfo(elements: [$element->scale], values: $values); - $s2=bfo(elements: $element->formula, values: $values); - $s3=bfo(elements: [$element->scale_right], values: $values); - if (!is_nan(num: $s1) && !is_nan(num: $s2) && !is_nan(num: $s3)) { - if($element->function=="max") { - $arg2 = (is_numeric(value: $element->arg2)) ? $element->arg2 : bfo(elements: $element->arg2, values: $values); - $s[] = (!is_nan(num: $arg2)) ? $s1*max([$s2,$arg2])*$s3 : NAN; - } - if($element->function=="brackets" || $element->function=="none") { - $s[]=$s1*$s2*$s3; + $sum=array_fill(0,$n,0); + foreach ($terms as $term) { + $product=array_fill(0,$n,1); + foreach ($term as $e) { + if ($e[0]=="value") { + $value=(float) $e[2]; + if ($e[1]=="/") $value=($value==0) ? NAN : 1/$value; + if ($e[1]=="-") $value=-$value; + for ($i=0;$i<$n;$i++) $product[$i]*=$value; + } else { + $values=$blocks[$e[2]]; + if ($e[1]=="/") { + foreach ($values as $i=>$value) $product[$i]*=($value==0) ? NAN : 1/$value; + } else if ($e[1]=="-") { + foreach ($values as $i=>$value) $product[$i]*=-$value; + } else { + foreach ($values as $i=>$value) $product[$i]*=$value; + } } - } else $s[] = NAN; + } + for ($i=0;$i<$n;$i++) $sum[$i]+=$product[$i]; } - return (!in_array(needle: NAN, haystack: $s)) ? array_sum(array: $s) : NAN; + return $sum; } } diff --git a/tests/test_combine.php b/tests/test_combine.php index bd709ed..97face2 100644 --- a/tests/test_combine.php +++ b/tests/test_combine.php @@ -160,3 +160,95 @@ assert_success(run("basic_formula", ["formula"=>"f1*3", "output"=>4, "process_mode"=>"from", "process_start"=>1010])); assert_feed(4, 1000, 10, [3, 60, 90]); }); + +test("basic_formula multiply by zero", function() { + feed(1, 1000, 10, [1, 2, NAN]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1*0+3", "output"=>4])); + assert_feed(4, 1000, 10, [3, 3, NAN]); +}); + +test("basic_formula division by zero is missing", function() { + feed(1, 1000, 10, [6, 6, 6]); + feed(2, 1000, 10, [2, 0, NAN]); + empty_feed(4); + assert_success(run("basic_formula", ["formula"=>"f1/f2", "output"=>4])); + assert_feed(4, 1000, 10, [3, NAN, NAN]); + assert_success(run("basic_formula", ["formula"=>"f1/0+2", "output"=>4])); + assert_feed(4, 1000, 10, [NAN, NAN, NAN]); +}); + +// Formula value for one datapoint, $values keyed by feed id. Reference for the block +// evaluator in basic_formula, evaluating each datapoint separately +function formula_reference($elements, $values) +{ + // sum of the products of each term + $sum = function($terms) use ($values) { + $s = []; + foreach ($terms as $term) { + $product = []; + foreach ($term as $e) { + $value = ($e[0] == "feed") ? $values[$e[2]] : (float) $e[2]; + if ($e[1] == "/") $value = ($value == 0) ? NAN : 1 / $value; + if ($e[1] == "-") $value = -$value; + $product[] = $value; + } + $s[] = array_product($product); + } + return array_sum($s); + }; + $total = 0; + foreach ($elements as $element) { + $s1 = $sum([$element->scale]); + $s2 = $sum($element->formula); + $s3 = $sum([$element->scale_right]); + if ($element->function == "max") { + $arg2 = is_numeric($element->arg2) ? $element->arg2 : $sum($element->arg2); + if (is_nan($s1) || is_nan($s2) || is_nan($s3) || is_nan($arg2)) return NAN; + $total += $s1 * max($s2, $arg2) * $s3; + } else { + $total += $s1 * $s2 * $s3; + } + } + return $total; +} + +test("basic_formula block evaluation matches per datapoint reference", function() { + global $testdir; + // Random data with missing values and zeros, more than one block + mt_srand(42); + $n = POSTPROCESS_BLOCKSIZE + 1000; + $data = []; + foreach ([1, 2, 3, 12] as $id) { + for ($i=0; $i<$n; $i++) { + $r = mt_rand(0, 99); + $data[$id][$i] = $r < 3 ? NAN : ($r < 8 ? 0.0 : mt_rand(-5000, 5000) / 100); + } + feed($id, 0, 10, $data[$id]); + } + // Feed values as stored + foreach ($data as $id => $values) $data[$id] = values($id); + + $formulas = ["f1+2*f2", "2*max(f1-f2,0)", "10*(f1-1)+f2/2", "f1-0.5*f2+max(f3-21,0)", + "1162.5*f2*(f1-11)", "f1/f2", "-f1*f2/3+4", "max(f1,f2)", "3*max(f1-f2,f3*2)/4", "f1+f12", + "f1*0", "f1/0+2", "-(f1+f2)", "f1-f2-f3*f12/2.5", "2(f1+3)-f2", "f1/f2/f3", "max(f1-f2,f3-f12)+f1"]; + + $p = new PostProcess_basic_formula($testdir); + foreach ($formulas as $formula) { + empty_feed(9); + assert_success(run("basic_formula", ["formula"=>$formula, "output"=>9])); + $parsed = $p->parse_formula($formula); + $expected = []; + for ($i=0; $i<$n; $i++) { + $values = []; + foreach ($parsed["feed_ids"] as $id) $values[$id] = $data[$id][$i]; + $expected[] = formula_reference($parsed["elements"], $values); + } + // Compare as stored, 32 bit floats + $expected = array_values(unpack("f*", pack("f*", ...$expected))); + if (pack("f*", ...$expected) !== pack("f*", ...values(9))) { + assert_equal($expected, values(9), $formula); + throw new TestFailure("$formula: differs from reference by less than the assert_equal tolerance"); + } + } +});