From ef768a10194a6e19df72e81a160be05dccb8b179 Mon Sep 17 00:00:00 2001 From: theferry Date: Mon, 26 Oct 2026 14:10:29 +0800 Subject: [PATCH] Keep GPU occupancy monitoring recoverable across models and disconnects --- SOURCE-SHA256.json | 24 +++-- docs/OPERATIONS.md | 10 ++ gpuqueue/platform.py | 130 +++++++++++++++----------- gpuqueue/serverbox.py | 1 + gpuqueue/service.py | 2 +- gpuqueue/telemetry.py | 58 ++++++++++++ tests/resources.test.js | 20 ++++ tests/test_monitoring.py | 146 ++++++++++++++++++++++++++++++ tests/test_request_diagnostics.py | 4 +- tests/test_serverbox.py | 3 +- web/app.js | 33 ++++++- web/index.html | 1 + web/model.js | 14 +++ web/style.css | 2 + 14 files changed, 379 insertions(+), 69 deletions(-) create mode 100644 gpuqueue/telemetry.py create mode 100644 tests/resources.test.js create mode 100644 tests/test_monitoring.py diff --git a/SOURCE-SHA256.json b/SOURCE-SHA256.json index e33c04a..07791e8 100644 --- a/SOURCE-SHA256.json +++ b/SOURCE-SHA256.json @@ -2,7 +2,7 @@ ".gitignore": "1364e59c28568273da17a19c4b4fb78dd86aaed90f6b7e71d7823585bba84707", ".openai/hosting.json": "8e1049aefe1f5e5853c8bd13a3e4e1b01f9251a452bfd8b5e384a260f4a551b7", "README.md": "f6cc6f56f7ca81b7a80cf5359a5849216d620fa5719910aee6a8e11289d289be", - "docs/OPERATIONS.md": "f3bf36698f26eb470016f5ed7b64fa174e8b9422a56375a0ccd9efae6b01ab29", + "docs/OPERATIONS.md": "985dbb23e3e1c84204be09cc51b786f6a5914267a3deed94ef1f7eb26399303b", "docs/typography-review.md": "57fc36226bcf1b0cff78acc9ce25c63a57e89ca739161f3c2e959f6ec35b4aa7", "downloads/GPU-Queue-Widget-1.1.2-macOS.zip": "77b1652f46675cf7f0b57616411df8d66124fbd2da499a270e1ad246c7902bff", "downloads/GPU-Queue-Widget-1.1.3-macOS.zip": "25d8a432737623a950a65115b1bb6b0f72bbfbbd86c783842a93a245a2ea4be1", @@ -16,10 +16,10 @@ "gpuqueue/cli.py": "d757987571b6552edc3c28fd11ffedb40116b2b7d2d5f6e118e0330314e4204e", "gpuqueue/control.py": "1412129e7f0f7c4f67db0c4c11f3e66f24cc1353f7bf67ad13b2ef86ae29de28", "gpuqueue/core.py": "a726498140ffc19076a461b9254ef3cb7ce7085119bd11997650f2068fef4c24", - "gpuqueue/platform.py": "1c4aad5ea1c2f41937ebaeafec03a5cc99ffee7d2dda706446d9919e49277f9d", + "gpuqueue/platform.py": "2e940049c5d58a03df556032ff1f11d433b0fce81c7288a854910539d4b22817", "gpuqueue/runner.py": "300c18ee4360bdf320f1a02c9a1eae21cca33d39347f4157b43e705f5177c304", - "gpuqueue/serverbox.py": "204470fb517ccd27609f5b2d1d68fa30d1a53f541a18d8e4412761fa8023817c", - "gpuqueue/service.py": "df60d5d835a3baef689dba4a80362088c07eea1e711ecc7244fda4ed504d9972", + "gpuqueue/serverbox.py": "247ca4fac527def799b4224fdf99506be4033467c27f413d139ab6b47b2afd1e", + "gpuqueue/service.py": "889854da765ca66a9487ae9fb7c01155b9f479f11321a88067531fba051b4677", "gpuqueue/stop_control.py": "10d2241a2882dbb97605f4fcb364c6f6fbe01451b8d90137d3aed1ff859433f7", "gpuqueue/web.py": "ae58b8225450409196de233a4e6901dc5811c5728d8ffcd9e3fdcce000345b06", "launchers/macos/Open-GPU-Queue.command": "b3a725d7435fe75c307f63bcb9a5c04ac578893c2ec67ab2f47f5e15f3611c6d", @@ -50,16 +50,20 @@ "tests/test_gpuqueue.py": "614df82d4652e8d529d5c030df852aabc766f9b03427bf2acf8d286a4e0cc316", "tests/test_launcher.py": "c336ce79267510501294ee1f73600b5b57ab8143469d75a8cf5d827e650e941d", "tests/test_local.py": "5721b2cf858b1fbaa4b56f479c0ee1490c1df63f1b8a27da4b632c92e908481d", - "tests/test_serverbox.py": "0716194806731cf4a11f33fb6fa8a163c6558404f336f8855c14188e98bb6cb3", + "tests/test_serverbox.py": "c8a7c0912a25263ab0b8e95545d689a38eba9fb70ffd81ff7c71eb5d76fea81d", "tests/test_stop.py": "39717a7e08eb2bf7c1e2b9b5416ed335bc71c0760210091fd3c2d70ffbf977cd", "tests/test_widget_autostart.py": "7819f157ca95931554a4aba9e4b0c195f99b10e21499b70e8eb9b0aa987bef40", "tests/test_widget_install.py": "bb0c80f1e2d8bb2fa685a51d8d5d9bebd05b4effc485e00d971618facda02b5c", "tests/widget-update.test.cjs": "7a3a3f5c9e7dd76c0e54968f09bbe48f9cdbf4ad959d46b46eddb3b8e2d48cb5", "tests/widget.test.cjs": "21dde14aa712b8f3c814354af8a1323e6c7f92af866169e78dd9b4095f501faf", "updates/gpu-queue-widget.json": "4ff3b46935ab67689c962cc22ceb3155960ff6de7572ff3435f15c481580a475", - "web/app.js": "9401da1be8229a99dc311c9e83df76d760a8eb553ac87269e25d87e2c3903ad0", - "web/index.html": "ad5cf456e1824c8ea9827c9c2db3e8741529779e68995d5074a766714c37f718", - "web/model.js": "55d7b7162a64eb2b142e6be6b5b4174c463a94b49d8525947e0563ba62896d4c", - "web/style.css": "e0e3d3335e69def00fe4ab990794bc2a8bf99d42767a37263a0a8f8a2a07c886", - "worker/api.js": "65ba1c1959a75f29eb03d4eaa0b4e1d8a776d3c6003985692d11417f8680dede" + "web/app.js": "58d19ef97f1d53c92e1c443bd9fbb6a9d6f9eb5f88fcd3ee5924e9841a9d86db", + "web/index.html": "1f50b7bdeadd7a9120480fe72dae3063ef4505c0ed19f8d907311e1f42b8b4cf", + "web/model.js": "aeec69ec79276ecd0c159f9f1c5f508ecfa90e5f3a227bd7bd269dccad5e6506", + "web/style.css": "7f73d024a00868bfb8839dee225ad0df7c33fdebe2ee672074332c90bf6f1957", + "worker/api.js": "65ba1c1959a75f29eb03d4eaa0b4e1d8a776d3c6003985692d11417f8680dede", + "tests/test_request_diagnostics.py": "8486cf3443efd0a15d6341c4e65378ed547b31b4a49a173df0a863f8f31dd2c4", + "gpuqueue/telemetry.py": "b142ffac5ebfbe5e98074595c94a67cf1f1dda597d90d87e09879bc891854f0c", + "tests/resources.test.js": "5c4150a62e891f2d59f6bb2d4c7f16a9da7b0f004655224a53f0d66d3beb5e94", + "tests/test_monitoring.py": "5bbca3ba9ec28269d4dadba01ff77c95ffd26d75a5ab53f8d1f2650318c1f676" } diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 713d282..33f3e74 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -129,3 +129,13 @@ Mac 的 `serverbox-gpu-status` helper 需要支持此注册表协议。注册表 - 停止任务不等于释放实例，也不能撤回已在途的 GPU 申请。进程退出证据与资源释放回执分开；平台主动释放接口仍未开放。 测试使用隔离目录和 CPU 假进程验证进程树终止、无关进程保留、产物保留、版本/来源校验、队列竞争与停止确认，不用真实 GPU 任务做取消测试。 + +## 持续 GPU 占用监测 + +服务按申请 UUID 自动发现本人的资源，收到就绪通知后通过已有 `mf-gpu` 通道运行只读 `nvidia-smi` 采样。型号与容量取实际读数，支持 H100、A100、4090 等 NVIDIA 显卡及多卡混合显示，不按型号写死指标；不支持 AMD/Intel 专用驱动采集。 + +成功采样后至少间隔 30 秒发出下一次探测；实际刷新还受平台队列与通知查询耗时影响。计算任务结束后继续监测。已知资源的探测失败按 30、60、120、240、300 秒退避，成功后清除错误。超时的未完成回执保留，迟到结果可恢复，不重复投放；提交回执不明时先按唯一探测标记核查本地规范文件。平台确认回收或失败后停止探测，未确认就绪的申请不运行 GPU 命令。 + +网页每 5 秒刷新，逐卡显示型号、GPU UUID、利用率、显存（GiB）、温度与采样时间。超过 90 秒或资源断开时标记最后读数并排除实时汇总；服务器断线也不会把旧读数显示为实时。硬件不支持的指标显示“—”。进程占用查询不可用时仍展示硬件读数，但暂停复用该资源；更换 GPU UUID 后重新积累低占用历史，避免沿用旧卡记录判断并行。 + +采样继续发布到 `~/.local/state/serverbox-gpu/registry.json`，保持 schema 1 与现有桌面客户端兼容。更新服务器端无需重新安装 Mac 客户端。新增回归见 `tests/test_monitoring.py` 和 `tests/resources.test.js`。 diff --git a/gpuqueue/platform.py b/gpuqueue/platform.py index 062cb35..c69841b 100644 --- a/gpuqueue/platform.py +++ b/gpuqueue/platform.py @@ -1,8 +1,6 @@ """Personal Markflow adapter. No action is called until browser approval exists.""" -import csv import datetime as dt import json -import math import os from pathlib import Path import re @@ -24,20 +22,7 @@ def reserve_mb(gpu, parallel=False): # margin applies only when admitting another task onto an occupied GPU. return max(2048, gpu['total_mb'] * .1) if parallel else 2048 -PROBE_SCRIPT = '''import subprocess,json,time,socket,sys -def query(fields,kind='gpu'): - p=subprocess.run(['nvidia-smi','--query-'+kind+'='+fields,'--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=20) - if p.returncode:raise RuntimeError('nvidia-smi query failed') - return p.stdout.strip() -try: - gpus=query('index,uuid,name,memory.total,memory.used,utilization.gpu,temperature.gpu') - at=time.time() - apps=query('gpu_uuid,pid','compute-apps') -except subprocess.TimeoutExpired: - print('GPUQ_PROBE_TIMEOUT=1',flush=True) - sys.exit(124) -print('GPUQ_TELEMETRY='+json.dumps({'gpus':gpus,'apps':apps,'at':at,'host':socket.gethostname()})) -''' +from .telemetry import PROBE_SCRIPT, parse_sample class PlatformError(RuntimeError): pass @@ -151,54 +136,88 @@ def submit_job(self, request_id, argv, cwd): raise PlatformError('GPU 任务回执不完整，已停止自动重试') return value['job'] + def probe_failure(self, resource, message, timed_out=False): + failures = resource.get('probe_failures', 0)+1 + resource.update(status='unknown', error=message, probe_failures=failures, + probe_retry_at=time.time()+min(300, 30*2**min(failures-1, 4))) + if timed_out: + resource['probe_timeout'] = {'at': now(), 'job': (resource.get('probe') or {}).get('job')} + def refresh_probe(self, resource): - if resource['status'] not in ['ready', 'online']: + if resource['status'] in ['pending', 'closed', 'failed']: + return + # Retry only known allocations, never ambiguous application receipts. + if resource['status'] == 'unknown' and not (resource.get('telemetry') or + resource.get('ready_message') or resource.get('ready_notice_consumed') or resource.get('probe') or resource.get('probe_submission')): return + submission = resource.get('probe_submission') + if submission: + for specification in self.job_folder(resource['id']).glob('*.json'): + if read_json(specification, {}).get('argv') == submission['argv']: + resource['probe'] = {'job': specification.stem, 'at': submission['at']} + resource.pop('probe_submission', None) + break + if resource.get('probe_submission'): + return # Unknown submit outcome: reconcile, never duplicate it. probe = resource.get('probe') if probe: folder = self.job_folder(resource['id']) / probe['job'] status = read_json(folder/'status.json', {}) if status.get('state') == 'finished' and status.get('exit_code') == 0: - output = (folder/'output.log').read_text(errors='replace') - line = next((line for line in output.splitlines() if line.startswith('GPUQ_TELEMETRY=')), None) - if not line: - raise PlatformError('GPU 检查已结束但没有有效遥测') - payload = json.loads(line.split('=', 1)[1]) - gpus = [] - for row in csv.reader(payload['gpus'].splitlines(), skipinitialspace=True): - if len(row) not in [6, 7]: - raise PlatformError('GPU 遥测字段不完整') - temperature = row[6].strip() if len(row) == 7 else 'N/A' - gpus.append({'index': int(row[0]), 'uuid': row[1], 'name': row[2], 'total_mb': float(row[3]), 'used_mb': float(row[4]), 'utilization': float(row[5]), 'temperature_c': None if temperature in ['N/A', '[N/A]', 'Not Supported', '[Not Supported]'] else float(temperature)}) - apps = [row for row in csv.reader(payload['apps'].splitlines(), skipinitialspace=True) if len(row) == 2] - stamp = float(payload.get('at', status.get('at', (folder/'output.log').stat().st_mtime))) - if not math.isfinite(stamp) or stamp <= 0 or stamp > time.time()+5: - raise PlatformError('GPU 遥测采样时间无效') - resource.update(status='online', telemetry={'gpus': gpus, 'apps': apps, 'at': stamp, 'host': payload.get('host', '')}, probe=None, probe_checked=time.time()) - resource.setdefault('samples', []).append({'at': stamp, 'utilization': max((g['utilization'] for g in gpus), default=100)}) - resource['samples'] = [sample for sample in resource['samples'] if stamp-sample['at'] < 420] + try: + output = (folder/'output.log').read_text(errors='replace') + line = next((line for line in output.splitlines() if line.startswith('GPUQ_TELEMETRY=')), None) + if not line: + raise ValueError('GPU 检查已结束但没有有效遥测') + payload = json.loads(line.split('=', 1)[1]) + telemetry = parse_sample(payload, (folder/'output.log').stat().st_mtime) + if telemetry['at'] <= resource.get('telemetry', {}).get('at', 0): + raise ValueError('GPU 遥测采样时间未前进') + except (ValueError, KeyError, TypeError, OSError) as error: + self.probe_failure(resource, 'GPU 遥测无效：'+str(error)) + resource['probe'] = None + return + previous = {g['uuid'] for g in resource.get('telemetry', {}).get('gpus', [])} + if previous != {g['uuid'] for g in telemetry['gpus']}: + resource['samples'] = [] + resource.update(status='online', telemetry=telemetry, probe=None, + probe_checked=time.time(), probe_failures=0) + for key in ['error', 'probe_timeout', 'probe_retry_at']: + resource.pop(key, None) + utilization = max((g['utilization'] if g['utilization'] is not None else 100 + for g in telemetry['gpus']), default=100) + resource.setdefault('samples', []).append({'at': telemetry['at'], 'utilization': utilization}) + resource['samples'] = [s for s in resource['samples'] if telemetry['at']-s['at'] < 420] return if status.get('state') in ['failed', 'finished']: output = folder/'output.log' timed_out = (status.get('exit_code') == 124 and output.is_file() and 'GPUQ_PROBE_TIMEOUT=1' in output.read_text(errors='replace').splitlines()) - resource.update(status='unknown', probe=None, - error='GPU 检查超时；旧实例是否回收尚未确认' if timed_out else 'GPU 只读检查失败，无法确认资源可用') - if timed_out: - resource['probe_timeout'] = {'at': now(), 'job': probe['job']} + self.probe_failure(resource, 'GPU 检查超时；旧实例是否回收尚未确认' if timed_out + else 'GPU 只读检查失败，稍后重试', timed_out) + resource['probe'] = None return - if time.time()-probe['at'] > 100: - # Only remove a diagnostic that has not been claimed. - specification = self.job_folder(resource['id'])/(probe['job']+'.json') - if not status and specification.exists(): - specification.unlink() - resource.update(status='unknown', probe=None, - probe_timeout={'at': now(), 'job': probe['job']}, - error='GPU 检查超时；旧实例是否回收尚未确认') + if time.time()-probe['at'] > 100 and resource['status'] != 'unknown': + # Keep the receipt: a slow or disconnected worker can complete + # it later. Never queue duplicate probes behind an unclaimed one. + self.probe_failure(resource, 'GPU 检查超时；旧实例是否回收尚未确认', True) + return + if time.time() < resource.get('probe_retry_at', 0): return if time.time()-resource.get('probe_checked', 0) >= 30: - job = self.submit_job(resource['id'], ['/usr/bin/python3', '-c', PROBE_SCRIPT], self.home) + argv = ['/usr/bin/python3', '-c', PROBE_SCRIPT+'\n# gpuq-probe '+str(uuid.uuid4())] + resource['probe_submission'] = {'argv': argv, 'at': time.time()} + self.persist() + try: + job = self.submit_job(resource['id'], argv, self.home) + except (PlatformError, OSError, subprocess.SubprocessError) as error: + if isinstance(error, OSError): + resource.pop('probe_submission', None) # Child never started. + self.probe_failure(resource, 'GPU 只读检查暂不可用：'+str(error)) + self.persist() + raise resource['probe'] = {'job': job, 'at': time.time()} + resource.pop('probe_submission', None) self.persist() def refresh(self, active=False): @@ -211,13 +230,20 @@ def refresh(self, active=False): for task in self.store.tasks(): replaceable.update(r['id'] for r in self.replacement_candidates(task)) replaceable.update(task['internal'].get('timeout_replacement', {}).get('prior_requests', [])) + # Collect all outstanding receipts before potentially slow notice I/O. + for resource in list(self.resources.values()): + if resource.get('probe'): + try: + self.refresh_probe(resource) + except (PlatformError, OSError, subprocess.SubprocessError, ValueError) as error: + errors.append(error) for resource in list(self.resources.values()): # A failed notification query must not prevent reading a probe # already completed in shared HOME, or skip other allocations. for refresh in (self.refresh_notice, self.refresh_probe): try: refresh(resource) - except (PlatformError, OSError, subprocess.SubprocessError) as error: + except (PlatformError, OSError, subprocess.SubprocessError, ValueError) as error: if refresh == self.refresh_notice and resource['id'] in replaceable and self.probe_timed_out(resource): # An unreachable historical notice must not veto the # user's timeout policy or block a newer allocation. @@ -243,7 +269,7 @@ def request(self, task): required = task['doc']['resources'] # A known smaller allocation may be insufficient. Unknown or busy usable # allocations must be reconciled rather than treated as absent. - if any(r['status'] != 'online' or time.time()-r.get('telemetry', {}).get('at', 0)>90 or len([g for g in r.get('telemetry', {}).get('gpus', []) if g['total_mb'] >= memory_mb(required)+reserve_mb(g)]) >= required['gpus'] and r.get('type') == required['type'] for r in unresolved): + if any(r['status'] != 'online' or time.time()-r.get('telemetry', {}).get('at', 0)>90 or len([g for g in r.get('telemetry', {}).get('gpus', []) if g.get('total_mb') is not None and g['total_mb'] >= memory_mb(required)+reserve_mb(g)]) >= required['gpus'] and r.get('type') == required['type'] for r in unresolved): return None if read_json(self.store.root/'tasks'/task['id']/'run/stop.json',{}).get('digest') == task['digest']: return None @@ -305,7 +331,7 @@ def select(self, task, active): return None, '默认顺序执行；未满足安全并行条件' for resource in self.resources.values(): telemetry = resource.get('telemetry', {}) - if resource['status'] != 'online' or time.time()-telemetry.get('at', 0) > 90 or resource.get('type') != required['type']: + if resource['status'] != 'online' or time.time()-telemetry.get('at', 0) > 90 or resource.get('type') != required['type'] or telemetry.get('apps_known') is False: continue if active: samples = [s for s in resource.get('samples', []) if time.time()-s['at'] < 330] @@ -321,7 +347,7 @@ def select(self, task, active): managed_pids.update(str(s['pid']) for s in status.get('steps', []) if s.get('state') == 'running' and s.get('pid')) occupied = {row[0] for row in telemetry.get('apps', []) if row[1] not in managed_pids} required_mb = memory_mb(required) - available = [g for g in telemetry.get('gpus', []) if g['uuid'] not in occupied and g['total_mb']-g['used_mb'] >= required_mb+reserve_mb(g, parallel=bool(active))] + available = [g for g in telemetry.get('gpus', []) if g.get('total_mb') is not None and g.get('used_mb') is not None and g['uuid'] not in occupied and g['total_mb']-g['used_mb'] >= required_mb+reserve_mb(g, parallel=bool(active))] available.sort(key=lambda gpu: gpu['total_mb']-gpu['used_mb'], reverse=True) if len(available) >= required['gpus']: return (resource, [g['uuid'] for g in available[:required['gpus']]]), '' diff --git a/gpuqueue/serverbox.py b/gpuqueue/serverbox.py index 32a3504..1e8e13e 100644 --- a/gpuqueue/serverbox.py +++ b/gpuqueue/serverbox.py @@ -30,6 +30,7 @@ def publish(resources, home): temperature_c=metric(gpu.get('temperature_c')))) records.append(dict(request=request, status=resource['status'], collected_at=stamp, host=telemetry.get('host', ''), gpus=gpus, + error=resource.get('error'), retry_at=resource.get('probe_retry_at'), release_pending=resource.get('release', {}).get('state') == 'pending_interface')) path = Path(home)/'.local/state/serverbox-gpu/registry.json' atomic_json(path, dict(schema=1, source='gpuq', updated_at=time.time(), resources=records)) diff --git a/gpuqueue/service.py b/gpuqueue/service.py index 42312ec..152951e 100644 --- a/gpuqueue/service.py +++ b/gpuqueue/service.py @@ -51,7 +51,7 @@ def sync(self): changed.append({'id': task['id'], 'digest': task['digest'], 'payload': task['public']}) resources = [] for resource in self.platform.resources.values(): - resources.append({key: resource.get(key) for key in ['id', 'status', 'type', 'gpus', 'telemetry', 'error', 'release']}) + resources.append({key: resource.get(key) for key in ['id', 'status', 'type', 'gpus', 'telemetry', 'error', 'release', 'probe_retry_at']}) response = self.remote('/api/worker/sync', {'tasks': changed, 'heartbeat': {'service': 'online', 'error': self.error, 'resources': resources, 'resource_policy': self.store.metadata('resource_policy'), 'updated_at': now()}}) self.synced.update(fingerprints) self.remote_revision = response['revision'] diff --git a/gpuqueue/telemetry.py b/gpuqueue/telemetry.py new file mode 100644 index 0000000..687a829 --- /dev/null +++ b/gpuqueue/telemetry.py @@ -0,0 +1,58 @@ +"""Model-independent NVIDIA samples; missing counters remain unknown.""" +import csv +import math +import time + +PROBE_SCRIPT = '''import subprocess,json,time,socket,sys +def query(fields,kind='gpu'): + p=subprocess.run(['nvidia-smi','--query-'+kind+'='+fields,'--format=csv,noheader,nounits'],capture_output=True,text=True,timeout=20) + if p.returncode:raise RuntimeError('nvidia-smi query failed') + return p.stdout.strip() +try: + try:gpus=query('index,uuid,name,memory.total,memory.used,utilization.gpu,temperature.gpu') + except RuntimeError:gpus=query('index,uuid,name,memory.total,memory.used,utilization.gpu') + at=time.time() + try:apps=query('gpu_uuid,pid','compute-apps');apps_known=True + except RuntimeError:apps='';apps_known=False +except subprocess.TimeoutExpired: + print('GPUQ_PROBE_TIMEOUT=1',flush=True) + sys.exit(124) +print('GPUQ_TELEMETRY='+json.dumps({'gpus':gpus,'apps':apps,'apps_known':apps_known,'at':at,'host':socket.gethostname()})) +''' + + +def counter(value, maximum=None): + value = value.strip() + if value in ('N/A', '[N/A]', 'Not Supported', '[Not Supported]', ''): + return None + number = float(value) + if not math.isfinite(number) or number < 0 or maximum is not None and number > maximum: + raise ValueError('GPU 遥测数值无效') + return number + + +def parse_sample(payload, fallback_at): + gpus, seen = [], set() + for row in csv.reader(payload['gpus'].splitlines(), skipinitialspace=True): + if len(row) not in (6, 7): + raise ValueError('GPU 遥测字段不完整') + index, uid = int(row[0]), row[1].strip() + if index < 0 or not uid or uid in seen: + raise ValueError('GPU 遥测标识无效') + seen.add(uid) + total, used = counter(row[3]), counter(row[4]) + if total == 0 or total is not None and used is not None and used > total: + raise ValueError('GPU 遥测显存无效') + gpus.append(dict(index=index, uuid=uid, name=row[2].strip(), total_mb=total, + used_mb=used, utilization=counter(row[5], 100), + temperature_c=counter(row[6]) if len(row) == 7 else None)) + if not gpus: + raise ValueError('GPU 检查没有返回任何显卡') + apps = list(csv.reader(payload.get('apps', '').splitlines(), skipinitialspace=True)) + apps_known = payload.get('apps_known', True) is True and all( + len(row) == 2 and row[0] in seen and row[1].strip().isdigit() for row in apps) + stamp = float(payload.get('at', fallback_at)) + if not math.isfinite(stamp) or stamp <= 0 or stamp > time.time()+5: + raise ValueError('GPU 遥测采样时间无效') + return dict(gpus=gpus, apps=apps if apps_known else [], apps_known=apps_known, + at=stamp, host=payload.get('host', '')) diff --git a/tests/resources.test.js b/tests/resources.test.js new file mode 100644 index 0000000..00f9c87 --- /dev/null +++ b/tests/resources.test.js @@ -0,0 +1,20 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {resourceView} from '../web/model.js'; +const gpu=(uuid='GPU-a',utilization=60)=>({uuid,utilization,used_mb:1024,total_mb:8192}); +const resource=(gpus,at=995,status='online')=>({id:'request',status,telemetry:{at,gpus}}); +test('mixed cards aggregate measured values and deduplicate GPU UUIDs',()=>{ + const view=resourceView([resource([gpu()]),resource([gpu(),gpu('GPU-b',80)])],1000); + assert.equal(view.gpus.length,2);assert.equal(view.utilization,70);assert.equal(view.used,2);assert.equal(view.total,16); +}); +test('stale, closed and offline resources retain rows without entering live totals',()=>{ + for(const r of [resource([gpu()],900),resource([gpu()],995,'closed'),resource([gpu()],995,'unknown'),resource([gpu()],1100)]){ + const view=resourceView([r],1000);assert.equal(view.rows.length,1);assert.equal(view.rows[0].fresh,false);assert.equal(view.utilization,null); + } + assert.equal(resourceView([resource([gpu()])],1000,false).gpus.length,0); +}); +test('unsupported counters remain unknown instead of displaying idle zeroes',()=>{ + const view=resourceView([resource([{uuid:'GPU-na',utilization:null,used_mb:null,total_mb:8192}])],1000); + assert.equal(view.utilization,null);assert.equal(view.used,null);assert.equal(view.partial,true); + assert.equal(resourceView([resource([gpu('GPU-idle',0)])],1000).utilization,0); +}); diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py new file mode 100644 index 0000000..acd1819 --- /dev/null +++ b/tests/test_monitoring.py @@ -0,0 +1,146 @@ +import json +from pathlib import Path +import subprocess +import tempfile +import time +import unittest +from unittest.mock import Mock, patch +from gpuqueue.core import Store, atomic_json, read_json +from gpuqueue.platform import Markflow, PlatformError +from gpuqueue.telemetry import parse_sample, PROBE_SCRIPT + +REQUEST = '11111111-1111-4111-8111-111111111111' +JOB = '22222222-2222-4222-8222-222222222222' + +class MonitoringTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.home = Path(self.temp.name) + self.store = Store(self.home/'queue') + self.adapter = Markflow(self.store, dict(home=str(self.home), username='linlishuo')) + self.resource = dict(id=REQUEST, status='unknown', ready_notice_consumed=True) + self.adapter.resources = {REQUEST: self.resource} + self.adapter.submit_job = Mock(return_value=JOB) + self.adapter.lark = Mock(return_value=dict(messages=[])) + + def tearDown(self): + self.temp.cleanup() + + def sample(self, name='NVIDIA GeForce RTX 4090', uid='GPU-4090', used=1000): + return dict(at=time.time(), gpus=f'0, {uid}, {name}, 49140, {used}, 75, N/A', apps='') + + def complete(self, payload): + self.resource['probe'] = dict(job=JOB, at=time.time()) + folder = self.adapter.job_folder(REQUEST)/JOB + atomic_json(folder/'status.json', dict(state='finished', exit_code=0)) + (folder/'output.log').write_text('GPUQ_TELEMETRY='+json.dumps(payload)) + self.adapter.refresh_probe(self.resource) + + def test_unknown_allocation_recovers_after_idle_and_clears_old_error(self): + self.resource.update(error='GPU 检查超时', probe_timeout={'at':'old'}) + self.adapter.refresh(active=False) + self.adapter.submit_job.assert_called_once() + self.complete(self.sample()) + self.assertEqual(self.resource['status'], 'online') + self.assertNotIn('error', self.resource) + self.assertNotIn('probe_timeout', self.resource) + self.assertEqual(self.resource['telemetry']['gpus'][0]['name'], 'NVIDIA GeForce RTX 4090') + + def test_failure_backs_off_without_new_gpu_application(self): + self.resource['probe'] = dict(job=JOB, at=time.time()) + atomic_json(self.adapter.job_folder(REQUEST)/JOB/'status.json', dict(state='failed', exit_code=7)) + self.adapter.refresh_probe(self.resource) + self.assertGreater(self.resource['probe_retry_at'], time.time()+25) + self.adapter.refresh_probe(self.resource) + self.adapter.submit_job.assert_not_called() + self.resource['probe_retry_at'] = 0 + self.adapter.refresh_probe(self.resource) + self.adapter.submit_job.assert_called_once() + self.assertEqual(self.resource['status'], 'unknown') + + def test_late_probe_is_retained_and_consumed_without_duplicate(self): + self.resource.update(status='online', probe=dict(job=JOB, at=time.time()-120)) + self.adapter.refresh_probe(self.resource) + self.assertEqual(self.resource['status'], 'unknown') + self.assertEqual(self.resource['probe']['job'], JOB) + self.adapter.refresh_probe(self.resource) + self.adapter.submit_job.assert_not_called() + self.complete(self.sample()) + self.assertEqual(self.resource['status'], 'online') + + def test_closed_failed_pending_and_unconfirmed_requests_never_probe(self): + for state in ['closed', 'failed', 'pending', 'unknown']: + resource = dict(id=REQUEST, status=state) + self.adapter.refresh_probe(resource) + self.adapter.submit_job.assert_not_called() + + def test_bad_sample_does_not_poison_registry_or_other_allocation(self): + self.resource['telemetry'] = parse_sample(self.sample(), time.time()) + previous = self.resource['telemetry'] + sample = self.sample();sample['gpus'] = sample['gpus'].replace('75', 'NaN') + self.complete(sample) + self.assertIs(self.resource['telemetry'], previous) + self.assertEqual(self.resource['status'], 'unknown') + self.assertIsNone(self.resource['probe']) + self.adapter.refresh() + record = read_json(self.home/'.local/state/serverbox-gpu/registry.json')['resources'][0] + self.assertEqual(record['gpus'][0]['utilization_gpu_pct'], 75) + + def test_topology_change_resets_parallel_admission_history(self): + self.resource.update(telemetry=parse_sample(self.sample(uid='GPU-old'), time.time()), + samples=[dict(at=time.time()-100, utilization=0)]) + self.complete(self.sample(uid='GPU-new')) + self.assertEqual(len(self.resource['samples']), 1) + self.assertEqual(self.resource['telemetry']['gpus'][0]['uuid'], 'GPU-new') + + def test_old_or_future_sample_cannot_refresh_timestamp(self): + self.resource['telemetry'] = parse_sample(self.sample(), time.time()) + old = self.resource['telemetry']['at'] + for stamp in [old-1, time.time()+50]: + sample = self.sample();sample['at'] = stamp + self.complete(sample) + self.assertEqual(self.resource['telemetry']['at'], old) + self.assertEqual(self.resource['status'], 'unknown') + + def test_ambiguous_probe_submit_reconciles_unique_spec_before_retry(self): + self.adapter.submit_job.side_effect = PlatformError('回执不明') + with self.assertRaises(PlatformError): + self.adapter.refresh_probe(self.resource) + submission = self.resource['probe_submission'] + self.resource['probe_retry_at'] = 0 + self.adapter.refresh_probe(self.resource) + self.assertEqual(self.adapter.submit_job.call_count, 1) + atomic_json(self.adapter.job_folder(REQUEST)/(JOB+'.json'), dict(argv=submission['argv'])) + self.adapter.refresh_probe(self.resource) + self.assertEqual(self.resource['probe']['job'], JOB) + self.assertNotIn('probe_submission', self.resource) + self.assertEqual(self.adapter.submit_job.call_count, 1) + + def test_optional_counters_are_unknown_and_process_query_blocks_reuse(self): + sample=self.sample();sample.update(gpus='0, GPU-test, H100, N/A, N/A, [Not Supported], N/A', apps_known=False) + self.complete(sample) + self.assertEqual(self.resource['status'], 'online') + self.assertIsNone(self.resource['telemetry']['gpus'][0]['utilization']) + self.assertEqual(self.resource['samples'][0]['utilization'], 100) + self.resource['type']='H卡离线' + task=dict(doc=dict(resources=dict(type='H卡离线',gpus=1,memory_gb=2)),internal={}) + self.assertIsNone(self.adapter.select(task, [])[0]) + + def test_probe_script_collects_even_when_temperature_or_apps_unsupported(self): + results=[subprocess.CompletedProcess([],1,'','unsupported'), + subprocess.CompletedProcess([],0,'0, GPU-test, H100, 81559, 100, 0',''), + subprocess.CompletedProcess([],1,'','unsupported')] + with patch('subprocess.run',side_effect=results), patch('builtins.print') as output: + exec(PROBE_SCRIPT, {}) + payload=json.loads(output.call_args.args[0].split('=',1)[1]) + self.assertFalse(payload['apps_known']) + self.assertEqual(parse_sample(payload,time.time())['gpus'][0]['name'],'H100') + + def test_mixed_models_and_multiple_cards_are_parsed_without_name_mapping(self): + sample=self.sample();sample['gpus']+='\n1, GPU-h100, NVIDIA H100 80GB HBM3, 81559, 40000, 88, 63' + telemetry=parse_sample(sample,time.time()) + self.assertEqual(len(telemetry['gpus']),2) + self.assertEqual(telemetry['gpus'][1]['temperature_c'],63) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_request_diagnostics.py b/tests/test_request_diagnostics.py index 59ee75a..ea73e83 100644 --- a/tests/test_request_diagnostics.py +++ b/tests/test_request_diagnostics.py @@ -42,9 +42,9 @@ def notice(self, title, message='ready-original'): return dict(sender=dict(id=TRUSTED_SENDER), message_id=message, content=title+'\n申请：'+REQUEST) - def test_historical_ready_notice_does_not_restart_timed_out_probes(self): + def test_historical_ready_notice_does_not_bypass_probe_backoff(self): resource = dict(id=REQUEST, status='unknown', ready_message='ready-original', - error='GPU 检查超时', samples=[]) + error='GPU 检查超时', samples=[], probe_retry_at=time.time()+300) self.adapter.resources = {REQUEST: resource} self.adapter.lark = Mock(return_value=dict(messages=[ self.notice('GPU已启动并通过成员权限与共享盘检查'), diff --git a/tests/test_serverbox.py b/tests/test_serverbox.py index 3670e64..c2bf4bf 100644 --- a/tests/test_serverbox.py +++ b/tests/test_serverbox.py @@ -74,7 +74,8 @@ def test_idle_online_resource_reuses_existing_probe_not_another_collector(self): self.adapter.submit_job = Mock(return_value=SECOND) self.adapter.refresh(active=False) argv = self.adapter.submit_job.call_args.args[1] - self.assertEqual(argv, ['/usr/bin/python3', '-c', PROBE_SCRIPT]) + self.assertEqual(argv[:2], ['/usr/bin/python3', '-c']) + self.assertTrue(argv[2].startswith(PROBE_SCRIPT+'\n# gpuq-probe ')) self.assertEqual(read_json(self.path)['resources'][0]['status'], 'online') def test_release_intent_is_not_reported_as_destroyed(self): diff --git a/web/app.js b/web/app.js index 3a760c0..fb81ae4 100644 --- a/web/app.js +++ b/web/app.js @@ -1,6 +1,7 @@ -import {stepState,canStop} from './model.js'; +import {stepState,canStop,resourceView} from './model.js'; const $=s=>document.querySelector(s),$$=s=>[...document.querySelectorAll(s)]; let state={tasks:[],selected:new URL(location.href).searchParams.get('task')},revision=0,heartbeat=null,filter='all',query='',detailTab='overview',selectedStep=null,dragId=null,touchDrag=false,busy=false,timer,stopTarget=null; +let networkOffline=false; let interactionMode='edit'; try{if(localStorage.getItem('gpuq.interactionMode')==='auto')interactionMode='auto';}catch{} const entryMode=new URL(location.href).searchParams.get('mode'); @@ -19,11 +20,37 @@ const date=value=>value?new Date(value).toLocaleString('zh-CN',{timeZone:'Asia/S function toast(text){$('#toast').textContent=text;$('#toast').classList.add('show');clearTimeout(timer);timer=setTimeout(()=>$('#toast').classList.remove('show'),4000);} async function api(path,data){const response=await fetch(path,{method:data?'POST':'GET',credentials:'same-origin',headers:data?{'content-type':'application/json','x-gpuq-action':'1'}:{},body:data?JSON.stringify(data):undefined,cache:'no-store'});const body=await response.json();if(!response.ok)throw new Error(body.error||'连接任务台失败');return body;} function apply(data){state.tasks=data.tasks;revision=data.revision;heartbeat=data.heartbeat;if(!state.tasks.some(t=>t.id===state.selected))state.selected=state.tasks[0]?.id;render();} -async function load(){if(busy||dragId||touchDrag)return;busy=true;try{apply(await api('/api/state'));}catch(error){$('#connection-note').innerHTML='!'+esc(error.message)+'。刷新可重试；已运行任务由服务器继续执行。';$('.prototype-tag').textContent='连接异常';}finally{busy=false;}} +async function load(){if(busy||dragId||touchDrag)return;busy=true;try{const data=await api('/api/state');networkOffline=false;apply(data);}catch(error){networkOffline=true;renderResource();$('#connection-note').innerHTML='!'+esc(error.message)+'。刷新可重试；已运行任务由服务器继续执行。';$('.prototype-tag').textContent='连接异常';}finally{busy=false;}} function badge(t){return `${t.state==='running'?'● ':''}${labels[t.state]||esc(t.state)}`;} function render(){renderStats();renderResource();renderList();renderDetail();renderMode();} function renderStats(){const count=fn=>state.tasks.filter(fn).length;$('#stats').innerHTML=[['待你启动',count(t=>t.state==='ready'),'Agent 已准备好，等待你的决定','◷',''],['执行队列',count(queued),'已批准，可拖动调整顺序','☷',''],['正在运行',count(t=>group(t)==='running'),'顺序执行，空闲时增加并行','▷','running'],['需要处理',count(t=>group(t)==='failed'),'查看原因，已有产物保留','!','attention']].map(([name,value,desc,icon,cls])=>`
${name}${icon}
${String(value).padStart(2,'0')}
${desc}
`).join('');$('#nav-count').textContent=state.tasks.length;} -function renderResource(){const policy=heartbeat?.resource_policy,releaseNote=$('#resource-policy');releaseNote.hidden=!policy?.release_when_queue_idle;releaseNote.textContent=(heartbeat?.resources||[]).some(r=>r.release?.state==='pending_interface')?'任务已结束，GPU 待释放。平台尚无主动释放入口，当前未确认释放。':'已保存用完即释放的偏好。平台尚无主动释放入口，暂不能立即释放。';const online=heartbeat&&Date.now()-Date.parse(heartbeat.at)<120000;$('.prototype-tag').textContent=online?'服务器已连接':'等待服务器';const resources=heartbeat?.resources||[];const fresh=resources.filter(r=>r.status==='online'&&r.telemetry&&Date.now()/1000-r.telemetry.at<90);const gpus=fresh.flatMap(r=>r.telemetry.gpus);const used=gpus.reduce((n,g)=>n+g.used_mb,0)/1024,total=gpus.reduce((n,g)=>n+g.total_mb,0)/1024;const utilization=gpus.length?Math.round(gpus.reduce((n,g)=>n+g.utilization,0)/gpus.length):null;$('#gpu-value').innerHTML=utilization===null?'—':`${utilization}%`;$('#gpu-bar').style.width=(utilization||0)+'%';$('#memory-value').innerHTML=gpus.length?`${used.toFixed(1)} / ${Math.round(total)} GB`:'— GB';$('#memory-bar').style.width=(total?used/total*100:0)+'%';const pending=resources.filter(r=>r.status==='pending').length;$('.resource-title div>span').innerHTML=` ${gpus.length?`${gpus.length} 张 GPU 已核验`:pending?`${pending} 份已有申请等待就绪`:'开始任务后核查并复用资源'}`;$('#schedule-label').textContent=state.tasks.filter(t=>group(t)==='running').length>1?'● 复用资源 · 并行中':'● 按顺序执行';$('#schedule-hint').textContent='持续低占用且显存充足时，最多并行 2 个任务';$('#connection-note').innerHTML=`${online?'◈':'◷'}${heartbeat?.error?esc(heartbeat.error):online?'服务器已连接。提交只进入待启动列表；点击开始后才会复用或申请 GPU。':'等待服务器同步。启动指令会持久保存，服务连接后再处理。'}${heartbeat?' · 最近同步 '+date(heartbeat.at):''}`;} +function renderResource(){ + const resources=heartbeat?.resources||[],policy=heartbeat?.resource_policy,releaseNote=$('#resource-policy'); + releaseNote.hidden=!policy?.release_when_queue_idle; + releaseNote.textContent=resources.some(r=>r.release?.state==='pending_interface')?'任务已结束，GPU 待释放。平台尚无主动释放入口，当前未确认释放。':'已保存用完即释放的偏好。平台尚无主动释放入口，暂不能立即释放。'; + const online=Boolean(!networkOffline&&heartbeat&&Date.now()-Date.parse(heartbeat.at)<120000); + $('.prototype-tag').textContent=online?'服务器已连接':'等待服务器'; + const view=resourceView(resources,Date.now()/1000,online),{gpus,used,total,utilization}=view; + $('#gpu-value').innerHTML=utilization===null?'—':`${utilization}%`; + $('#gpu-bar').style.width=(utilization||0)+'%'; + $('#memory-value').innerHTML=total===null?'— GiB':`${used.toFixed(1)} / ${total.toFixed(1)} GiB`; + $('#memory-bar').style.width=(total?used/total*100:0)+'%'; + const stale=view.rows.filter(r=>!r.fresh&&!['closed','failed'].includes(r.status)&&r.telemetry).length; + $('.resource-title div>span').textContent=gpus.length?`${gpus.length} 张 GPU 持续监测${view.partial?' · 部分指标不可用':''}`:stale?'采样已过期，正在恢复监测':resources.some(r=>r.status==='pending')?'已有申请等待就绪':'等待 GPU 接入'; + const statuses={pending:'等待就绪',ready:'正在采样',online:'采样已过期',unknown:'连接待恢复',closed:'已回收',failed:'申请失败'}; + const metric=(n,unit)=>Number.isFinite(n)?`${n.toFixed(0)}${unit}`:'—'; + const row=r=>{ + const label=r.fresh?'实时':statuses[r.status]||'等待核查'; + const stamp=r.telemetry?.at?date(r.telemetry.at*1000):'尚无采样'; + return `
${esc(r.type||'GPU')} · ${esc(r.id.slice(0,8))}${label} · ${stamp}
${(r.telemetry?.gpus||[]).map(g=>`
${esc(g.name)}GPU ${esc(g.index)} · ${esc(g.uuid)}${r.fresh?'':' · 最后读数'}
占用 ${metric(g.utilization,'%')}显存 ${Number.isFinite(g.used_mb)&&Number.isFinite(g.total_mb)?(g.used_mb/1024).toFixed(1)+' / '+(g.total_mb/1024).toFixed(1)+' GiB':'—'}温度 ${metric(g.temperature_c,'°C')}
`).join('')}${r.error?`

${esc(r.error)}${r.probe_retry_at?' · 下次重试 '+date(r.probe_retry_at*1000):''}

`:''}${r.telemetry?.apps_known===false?'

进程占用信息不可用，暂不复用此资源。

':''}
`; + }; + const historyOpen=$('.gpu-history')?.open; + const current=view.rows.filter(r=>!['closed','failed'].includes(r.status)),history=view.rows.filter(r=>['closed','failed'].includes(r.status)); + $('#gpu-devices').innerHTML=current.map(row).join('')+(history.length?`
已结束资源 · ${history.length}${history.map(row).join('')}
`:'')||'

接入已就绪的 GPU 后自动采样，任务结束后继续监测。

'; + $('#schedule-label').textContent=state.tasks.filter(t=>group(t)==='running').length>1?'● 复用资源 · 并行中':'● 按顺序执行'; + $('#schedule-hint').textContent='持续低占用且显存充足时，最多并行 2 个任务'; + $('#connection-note').innerHTML=`${online?'◈':'◷'}${heartbeat?.error?esc(heartbeat.error):online?'服务器已连接。提交只进入待启动列表；点击开始后才会复用或申请 GPU。':'等待服务器同步。启动指令会持久保存，服务连接后再处理。'}${heartbeat?' · 最近同步 '+date(heartbeat.at):''}`; +} function matches(t){return (!query||`${t.name} ${t.project} ${t.agent} ${t.id}`.toLowerCase().includes(query.toLowerCase()))&&(filter==='all'||filter==='queue'&&['queued','running'].includes(group(t))||filter==='ready'&&t.state==='ready'||filter==='history'&&['failed','completed','stopped'].includes(group(t)));} function renderList(){let html='';for(const [status,title,caption]of [['running','正在运行','流程成功后自动衔接'],['queued','执行队列','仅已批准的任务会运行'],['ready','待你启动','准备完成后，由你决定'],['failed','需要处理','已有产物保留，不自动重跑'],['stopped','已停止','已有产物保留，重新运行需提交新版本'],['completed','已完成','结果已保存']]){const tasks=state.tasks.filter(t=>group(t)===status&&matches(t));if(!tasks.length)continue;html+=`
${title}${tasks.length}${caption}
`;for(const t of tasks){const queue=state.tasks.filter(reorderable),pos=queue.findIndex(x=>x.id===t.id)+1;html+=`
${reorderable(t)?'':`${group(t)==='running'?'◉':group(t)==='failed'?'!':t.state==='completed'?'✓':'◇'}`}
${esc(t.name)}${esc(t.agent)}
${esc(t.project)} · ${t.id.slice(0,11)}
${t.steps.map((s,i)=>`${i?'→':''}${stepState(t,i)==='done'?'✓ ':''}${esc(s)}`).join('')}
${t.state==='running'?`
`:''}
${badge(t)}${t.state==='ready'?``:reorderable(t)?`队列第 ${pos} 位
`:`${esc(t.elapsed)}`}
`;}}$('#task-list').innerHTML=html||`
${state.tasks.length?'没有符合条件的任务。':'任务队列为空。'}
${state.tasks.length?'试试其他筛选条件。':'让 Codex 或 Claude 准备任务并通过 gpuq submit 提交。'}
`;bindRows();renderMode();} function renderDetail(){const t=state.tasks.find(t=>t.id===state.selected);if(!t){$('#detail').innerHTML='
选择一个任务
查看准备情况和执行步骤
';return;}const index=selectedStep??Math.min(t.stage,t.steps.length-1),completed=t.step_records.filter(s=>s.state==='completed').length;$('#detail').innerHTML=`
任务详情 · ${t.id.slice(0,11)}${badge(t)}

${esc(t.name)}

${esc(t.description)}

${esc(t.agent)}${date(t.created_at)} 提交 · 北京时间
${t.state==='ready'?``:group(t)==='failed'?'':['completed','stopped','cancelled'].includes(t.state)?'':''}${canStop(t)?``:t.state==='stopping'?'':''}${t.phase&&(queued(t)||t.state==='stopping')?`
${esc(t.phase)}
`:''}
资源需求 · Agent 判断
${t.resources.gpus} 卡${t.memory?' · 每卡 '+t.memory+' GB':''}
运行时长
${esc(t.elapsed)}
固定版本
${esc(t.version)}
启动授权
${t.approved_at?date(t.approved_at):'等待你点击开始'}
执行步骤${completed} / ${t.steps.length} 已完成
${t.steps.map((name,i)=>{const st=stepState(t,i);return ``;}).join('')}
${[['overview','准备检查'],['logs','步骤日志'],['results','结果文件'],['events','事件']].map(([id,label])=>``).join('')}
${detailContent(t,index)}
`;$('#detail [data-detail-start]')?.addEventListener('click',()=>start(t.id));$('#detail [data-detail-stop]')?.addEventListener('click',()=>requestStop(t.id));$('#diagnostic')?.addEventListener('click',()=>copy(`任务：${t.id}\n版本：${t.version}\n状态：${labels[t.state]}\n原因：${t.failureReason||'待核查'}\n申请：${t.request_uuid||'未申请'}\n任务回执：${t.job_uuid||'未提交'}\n${t.step_records.map(s=>s.name+'\n'+(s.log||'')).join('\n')}`));$('#view-results')?.addEventListener('click',()=>{detailTab='results';renderDetail();});$$('[data-step]').forEach(b=>b.addEventListener('click',()=>{selectedStep=Number(b.dataset.step);detailTab='logs';renderDetail();}));$$('[data-detail-tab]').forEach(b=>b.addEventListener('click',()=>{detailTab=b.dataset.detailTab;renderDetail();}));$$('[data-output]').forEach(b=>b.addEventListener('click',()=>copy(b.dataset.output)));renderMode();} diff --git a/web/index.html b/web/index.html index c710a38..a3d2fd6 100644 --- a/web/index.html +++ b/web/index.html @@ -29,6 +29,7 @@
● 按顺序执行持续低占用时，最多并行 2 个任务
低占用时自动判断并行 +
diff --git a/web/model.js b/web/model.js index 1878b32..167a979 100644 --- a/web/model.js +++ b/web/model.js @@ -9,3 +9,17 @@ export function stepState(task,index){ if(['running','stopping'].includes(task.state)&&task.stage===index)return 'current'; return 'pending'; } + +// The same GPU can appear in a historical and a current allocation. +export function resourceView(resources, at=Date.now()/1000, connected=true){ + const rows=resources.map(r=>({...r,fresh:connected&&r.status==='online'&&Number.isFinite(r.telemetry?.at)&&at-r.telemetry.at>=-5&&at-r.telemetry.at<90})); + const unique=new Map(); + for(const r of rows.filter(r=>r.fresh))for(const g of r.telemetry.gpus||[])unique.set(g.uuid,g); + const gpus=[...unique.values()]; + const known=gpus.filter(g=>Number.isFinite(g.utilization)); + const memory=gpus.filter(g=>Number.isFinite(g.used_mb)&&Number.isFinite(g.total_mb)); + return {rows,gpus,utilization:known.length?Math.round(known.reduce((n,g)=>n+g.utilization,0)/known.length):null, + used:memory.length?memory.reduce((n,g)=>n+g.used_mb,0)/1024:null, + total:memory.length?memory.reduce((n,g)=>n+g.total_mb,0)/1024:null, + partial:known.lengthspan{color:#42745d}.gpu-device{display:flex;align-items:center;justify-content:space-between;gap:16px;padding-top:12px}.gpu-device strong{font-size:14px;font-weight:500}.gpu-device small{display:block;font-size:11px;color:var(--muted,#777);overflow-wrap:anywhere;margin-top:4px}.gpu-counters{display:flex;flex-wrap:wrap;gap:18px;flex-shrink:0;font-size:12px;color:var(--muted,#777)}.gpu-counters b{font-weight:500;color:var(--text,#302f2b);margin-left:4px}.gpu-monitor-note{font-size:12px;line-height:1.6;color:#8a6945;margin:10px 0 0;overflow-wrap:anywhere}.gpu-history summary{cursor:pointer;min-height:36px;display:flex;align-items:center;font-size:12px;color:var(--muted,#777)}.gpu-history .gpu-allocation{margin-top:8px}.gpu-allocation.stale .gpu-counters{opacity:.7}@media(max-width:800px){.gpu-device{align-items:flex-start;flex-direction:column;gap:10px}.gpu-counters{gap:10px;flex-shrink:1}.gpu-allocation{padding:14px}}