From 3ccbf3df56185001e9d2c01d7e95b8490cd15753 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Thu, 28 Mar 2024 23:37:16 -0400 Subject: [PATCH 01/12] Update setup.py --- setup.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index ab3aca97b..7229ce484 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,9 @@ -#!/usr/bin/env python -# Copyright (c) Megvii, Inc. and its affiliates. All Rights Reserved - -import re +from setuptools import setup, Extension, find_namespace_packages import setuptools import glob from os import path -import torch -from torch.utils.cpp_extension import CppExtension - -torch_ver = [int(x) for x in torch.__version__.split(".")[:2]] -assert torch_ver >= [1, 3], "Requires PyTorch >= 1.3" - +import re +import pybind11 # Import Pybind11 def get_extensions(): this_dir = path.dirname(path.abspath(__file__)) @@ -20,19 +13,15 @@ def get_extensions(): sources = glob.glob(path.join(extensions_dir, "**", "*.cpp")) sources = [main_source] + sources - extension = CppExtension - - extra_compile_args = {"cxx": ["-O3"]} - define_macros = [] - include_dirs = [extensions_dir] + extra_compile_args = ["-O3"] + include_dirs = [extensions_dir, pybind11.get_include()] # Add Pybind11 include directory ext_modules = [ - extension( + Extension( "yolox._C", sources, include_dirs=include_dirs, - define_macros=define_macros, extra_compile_args=extra_compile_args, ) ] @@ -47,11 +36,11 @@ def get_extensions(): ).group(1) -with open("README.md", "r") as f: +with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() -setuptools.setup( +setup( name="yolox", version=version, author="basedet team", @@ -59,6 +48,5 @@ def get_extensions(): long_description=long_description, ext_modules=get_extensions(), classifiers=["Programming Language :: Python :: 3", "Operating System :: OS Independent"], - cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, - packages=setuptools.find_namespace_packages(), + packages=find_namespace_packages(), ) From 74574f37f29fd4bd1a4813af11c1f4ee79c65875 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Thu, 28 Mar 2024 23:38:13 -0400 Subject: [PATCH 02/12] Update byte_tracker.py --- yolox/tracker/byte_tracker.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index 2d004599b..72068fdcc 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -3,8 +3,6 @@ import os import os.path as osp import copy -import torch -import torch.nn.functional as F from .kalman_filter import KalmanFilter from yolox.tracker import matching @@ -156,7 +154,7 @@ def __init__(self, args, frame_rate=30): self.max_time_lost = self.buffer_size self.kalman_filter = KalmanFilter() - def update(self, output_results, img_info, img_size): + def update(self, output_results): self.frame_id += 1 activated_starcks = [] refind_stracks = [] @@ -170,9 +168,9 @@ def update(self, output_results, img_info, img_size): output_results = output_results.cpu().numpy() scores = output_results[:, 4] * output_results[:, 5] bboxes = output_results[:, :4] # x1y1x2y2 - img_h, img_w = img_info[0], img_info[1] - scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w)) - bboxes /= scale + #img_h, img_w = img_info[0], img_info[1] + #scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w)) + #bboxes /= scale remain_inds = scores > self.args.track_thresh inds_low = scores > 0.1 From c0310dec1c97ebd79ca7fb720712274f685a498f Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Mon, 1 Apr 2024 14:11:22 -0400 Subject: [PATCH 03/12] Update setup.py enforcing c++11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7229ce484..c0724f180 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def get_extensions(): sources = [main_source] + sources - extra_compile_args = ["-O3"] + extra_compile_args = ["-O3", "-std=c++11"] include_dirs = [extensions_dir, pybind11.get_include()] # Add Pybind11 include directory ext_modules = [ From 081d3d4cc7ba90093f2180e7a6c3d1ac10b40a83 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Mon, 1 Apr 2024 16:40:44 -0400 Subject: [PATCH 04/12] removed unnecessary imports --- yolox/utils/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yolox/utils/__init__.py b/yolox/utils/__init__.py index a268c1a45..33410074c 100644 --- a/yolox/utils/__init__.py +++ b/yolox/utils/__init__.py @@ -2,15 +2,15 @@ # -*- coding:utf-8 -*- # Copyright (c) 2014-2021 Megvii Inc. All rights reserved. -from .allreduce_norm import * -from .boxes import * -from .checkpoint import load_ckpt, save_checkpoint +#from .allreduce_norm import * +#from .boxes import * +#from .checkpoint import load_ckpt, save_checkpoint from .demo_utils import * -from .dist import * -from .ema import ModelEMA +#from .dist import * +#from .ema import ModelEMA from .logger import setup_logger from .lr_scheduler import LRScheduler -from .metric import * -from .model_utils import * +#from .metric import * +#from .model_utils import * from .setup_env import * from .visualize import * From 8c8441183f62fabe9ee6d6ce0bc739ced7a448f7 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Tue, 2 Apr 2024 18:40:46 -0400 Subject: [PATCH 05/12] refactor --- yolox/tracker/byte_tracker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index 72068fdcc..23f2219b9 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -149,8 +149,8 @@ def __init__(self, args, frame_rate=30): self.frame_id = 0 self.args = args #self.det_thresh = args.track_thresh - self.det_thresh = args.track_thresh + 0.1 - self.buffer_size = int(frame_rate / 30.0 * args.track_buffer) + self.det_thresh = args["track_threshold"] + 0.1 + self.buffer_size = int(frame_rate / 30.0 * args["track_buffer"]) self.max_time_lost = self.buffer_size self.kalman_filter = KalmanFilter() @@ -172,9 +172,9 @@ def update(self, output_results): #scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w)) #bboxes /= scale - remain_inds = scores > self.args.track_thresh + remain_inds = scores > self.args["track_threshold"] inds_low = scores > 0.1 - inds_high = scores < self.args.track_thresh + inds_high = scores < self.args["track_threshold"] inds_second = np.logical_and(inds_low, inds_high) dets_second = bboxes[inds_second] @@ -203,9 +203,9 @@ def update(self, output_results): # Predict the current location with KF STrack.multi_predict(strack_pool) dists = matching.iou_distance(strack_pool, detections) - if not self.args.mot20: + if not self.args["mot20"]: dists = matching.fuse_score(dists, detections) - matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh) + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args["match_threshold"]) for itracked, idet in matches: track = strack_pool[itracked] @@ -247,7 +247,7 @@ def update(self, output_results): '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' detections = [detections[i] for i in u_detection] dists = matching.iou_distance(unconfirmed, detections) - if not self.args.mot20: + if not self.args["mot20"]: dists = matching.fuse_score(dists, detections) matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) for itracked, idet in matches: From a76d7e9db97b1f68928b43fd3febe357efe287b4 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Wed, 3 Apr 2024 19:11:57 -0400 Subject: [PATCH 06/12] Integrate person in byte_tracker --- yolox/tracker/byte_tracker.py | 53 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index 23f2219b9..d821850fc 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -10,7 +10,7 @@ class STrack(BaseTrack): shared_kalman = KalmanFilter() - def __init__(self, tlwh, score): + def __init__(self, tlwh, score, person): # wait activate self._tlwh = np.asarray(tlwh, dtype=np.float) @@ -20,6 +20,8 @@ def __init__(self, tlwh, score): self.score = score self.tracklet_len = 0 + + self.person_instance = person def predict(self): mean_state = self.mean.copy() @@ -48,9 +50,9 @@ def activate(self, kalman_filter, frame_id): self.tracklet_len = 0 self.state = TrackState.Tracked - if frame_id == 1: - self.is_activated = True - # self.is_activated = True + #if frame_id == 1: + # self.is_activated = True + self.is_activated = True self.frame_id = frame_id self.start_frame = frame_id @@ -148,8 +150,7 @@ def __init__(self, args, frame_rate=30): self.frame_id = 0 self.args = args - #self.det_thresh = args.track_thresh - self.det_thresh = args["track_threshold"] + 0.1 + self.init_new_track_thresh = args["track_threshold"]# + 0.1 self.buffer_size = int(frame_rate / 30.0 * args["track_buffer"]) self.max_time_lost = self.buffer_size self.kalman_filter = KalmanFilter() @@ -160,17 +161,14 @@ def update(self, output_results): refind_stracks = [] lost_stracks = [] removed_stracks = [] + newly_created_stracks = [] + + if output_results.size == 0: + return [], [], [] - if output_results.shape[1] == 5: - scores = output_results[:, 4] - bboxes = output_results[:, :4] - else: - output_results = output_results.cpu().numpy() - scores = output_results[:, 4] * output_results[:, 5] - bboxes = output_results[:, :4] # x1y1x2y2 - #img_h, img_w = img_info[0], img_info[1] - #scale = min(img_size[0] / float(img_h), img_size[1] / float(img_w)) - #bboxes /= scale + scores = output_results[:, 4] + people = output_results[:, 5] + bboxes = output_results[:, :4] remain_inds = scores > self.args["track_threshold"] inds_low = scores > 0.1 @@ -184,8 +182,8 @@ def update(self, output_results): if len(dets) > 0: '''Detections''' - detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s) for - (tlbr, s) in zip(dets, scores_keep)] + detections = [STrack(STrack.tlbr_to_tlwh(tlbr), s, person) for + (tlbr, s, person) in zip(dets, scores_keep, people)] else: detections = [] @@ -203,8 +201,9 @@ def update(self, output_results): # Predict the current location with KF STrack.multi_predict(strack_pool) dists = matching.iou_distance(strack_pool, detections) - if not self.args["mot20"]: - dists = matching.fuse_score(dists, detections) + + dists = matching.fuse_score(dists, detections) + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args["match_threshold"]) for itracked, idet in matches: @@ -221,8 +220,8 @@ def update(self, output_results): # association the untrack to the low score detections if len(dets_second) > 0: '''Detections''' - detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s) for - (tlbr, s) in zip(dets_second, scores_second)] + detections_second = [STrack(STrack.tlbr_to_tlwh(tlbr), s, person) for + (tlbr, s, person) in zip(dets_second, scores_second, people)] else: detections_second = [] r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] @@ -247,8 +246,9 @@ def update(self, output_results): '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' detections = [detections[i] for i in u_detection] dists = matching.iou_distance(unconfirmed, detections) - if not self.args["mot20"]: - dists = matching.fuse_score(dists, detections) + + dists = matching.fuse_score(dists, detections) + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) @@ -261,10 +261,11 @@ def update(self, output_results): """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] - if track.score < self.det_thresh: + if track.score < self.init_new_track_thresh: continue track.activate(self.kalman_filter, self.frame_id) activated_starcks.append(track) + newly_created_stracks.append(track) """ Step 5: Update state""" for track in self.lost_stracks: if self.frame_id - track.end_frame > self.max_time_lost: @@ -284,7 +285,7 @@ def update(self, output_results): # get scores of lost tracks output_stracks = [track for track in self.tracked_stracks if track.is_activated] - return output_stracks + return output_stracks, lost_stracks, newly_created_stracks def joint_stracks(tlista, tlistb): From 343c0ef5d0700e2608b3530e19d0ec5eabee0ee0 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Wed, 3 Apr 2024 20:54:00 -0400 Subject: [PATCH 07/12] Create macos_installation.txt --- macos_installation.txt | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 macos_installation.txt diff --git a/macos_installation.txt b/macos_installation.txt new file mode 100644 index 000000000..2252d3a31 --- /dev/null +++ b/macos_installation.txt @@ -0,0 +1,27 @@ +conda create --name btrack_exp python=3.8 +conda activate btrack_exp + +cd $BASE_STATION_ROOT$ +pip3 install -r requirements.txt +pip3 uninstall numpy opencv-python +conda install numpy=1.19.5 opencv=4.6 +conda install pybind11 loguru pandas seaborn scipy + +cd ~ +mkdir 3rd_party +cd 3rd_party + +git clone https://github.com/gatagat/lap +cd lap +python3 setup.py build +python3 setup.py install + +cd .. +git clone https://github.com/samson-wang/cython_bbox.git +cd cython_bbox +python3 setup.py build + +cd.. +git clone git@github.com:inspiren-inc/ins_btrack.git -b ins-btrack +cd ins_btrack +python3 setup.py develop From 2d91274bda74e309d6120bcef5c5aee07f3601f6 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Wed, 3 Apr 2024 20:56:03 -0400 Subject: [PATCH 08/12] Update macos_installation.txt --- macos_installation.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/macos_installation.txt b/macos_installation.txt index 2252d3a31..cb1c62bca 100644 --- a/macos_installation.txt +++ b/macos_installation.txt @@ -20,6 +20,7 @@ cd .. git clone https://github.com/samson-wang/cython_bbox.git cd cython_bbox python3 setup.py build +python3 setup.py install cd.. git clone git@github.com:inspiren-inc/ins_btrack.git -b ins-btrack From f69e67c79b1e165cb1acf56b152e260ac6f20eb7 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Thu, 4 Apr 2024 13:03:54 -0400 Subject: [PATCH 09/12] tuning tracker --- yolox/tracker/byte_tracker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index d821850fc..ea78d279f 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -151,7 +151,8 @@ def __init__(self, args, frame_rate=30): self.frame_id = 0 self.args = args self.init_new_track_thresh = args["track_threshold"]# + 0.1 - self.buffer_size = int(frame_rate / 30.0 * args["track_buffer"]) + #self.buffer_size = int(frame_rate / 30.0 * args["track_buffer"]) + self.buffer_size = args["track_buffer"] self.max_time_lost = self.buffer_size self.kalman_filter = KalmanFilter() @@ -249,7 +250,7 @@ def update(self, output_results): dists = matching.fuse_score(dists, detections) - matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.3) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) activated_starcks.append(unconfirmed[itracked]) From f5aa9195151d12c77a1befdbda244c27e393c788 Mon Sep 17 00:00:00 2001 From: Wassim El Ahmar Date: Mon, 8 Apr 2024 12:06:08 -0400 Subject: [PATCH 10/12] Update byte_tracker.py --- yolox/tracker/byte_tracker.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index ea78d279f..a4f50e1ec 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -150,7 +150,8 @@ def __init__(self, args, frame_rate=30): self.frame_id = 0 self.args = args - self.init_new_track_thresh = args["track_threshold"]# + 0.1 + self.high_score_cutoff = args["high_score_cutoff"] + self.low_score_cutoff = args["low_score_cutoff"] #self.buffer_size = int(frame_rate / 30.0 * args["track_buffer"]) self.buffer_size = args["track_buffer"] self.max_time_lost = self.buffer_size @@ -171,9 +172,9 @@ def update(self, output_results): people = output_results[:, 5] bboxes = output_results[:, :4] - remain_inds = scores > self.args["track_threshold"] - inds_low = scores > 0.1 - inds_high = scores < self.args["track_threshold"] + remain_inds = scores > self.args["high_score_cutoff"] + inds_low = scores > self.low_score_cutoff + inds_high = scores < self.args["high_score_cutoff"] inds_second = np.logical_and(inds_low, inds_high) dets_second = bboxes[inds_second] @@ -227,7 +228,7 @@ def update(self, output_results): detections_second = [] r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] dists = matching.iou_distance(r_tracked_stracks, detections_second) - matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5) + matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.2) for itracked, idet in matches: track = r_tracked_stracks[itracked] det = detections_second[idet] @@ -262,7 +263,7 @@ def update(self, output_results): """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] - if track.score < self.init_new_track_thresh: + if track.score < self.high_score_cutoff: continue track.activate(self.kalman_filter, self.frame_id) activated_starcks.append(track) From 38168de25ab8dced202677d2e57ab09809277d7f Mon Sep 17 00:00:00 2001 From: Bryce Chudomelka Date: Thu, 11 Apr 2024 11:25:56 -0400 Subject: [PATCH 11/12] convert all np.float to float --- tutorials/centertrack/byte_tracker.py | 2 +- tutorials/centertrack/mot_online/matching.py | 36 +- tutorials/cstrack/byte_tracker.py | 2 +- tutorials/cstrack/tracker.py | 2 +- tutorials/ctracker/byte_tracker.py | 2 +- tutorials/ctracker/mot_online/matching.py | 36 +- tutorials/fairmot/byte_tracker.py | 2 +- tutorials/fairmot/tracker.py | 2 +- tutorials/jde/byte_tracker.py | 738 ++++++++-------- tutorials/jde/evaluation.py | 208 ++--- tutorials/jde/tracker.py | 828 +++++++++--------- tutorials/motr/byte_tracker.py | 2 +- tutorials/motr/mot_online/matching.py | 36 +- tutorials/qdtrack/byte_tracker.py | 2 +- tutorials/qdtrack/mot_online/matching.py | 36 +- tutorials/qdtrack/tracker_reid_motion.py | 2 +- tutorials/trades/byte_tracker.py | 2 +- tutorials/trades/mot_online/matching.py | 36 +- tutorials/trades/tracker.py | 2 +- tutorials/transtrack/main_track.py | 750 ++++++++-------- .../transtrack/mot_online/byte_tracker.py | 2 +- tutorials/transtrack/mot_online/matching.py | 12 +- yolox/deepsort_tracker/detection.py | 2 +- yolox/motdt_tracker/matching.py | 10 +- yolox/motdt_tracker/motdt_tracker.py | 2 +- yolox/motdt_tracker/reid_model.py | 2 +- yolox/tracker/byte_tracker.py | 2 +- yolox/tracker/matching.py | 12 +- 28 files changed, 1385 insertions(+), 1385 deletions(-) diff --git a/tutorials/centertrack/byte_tracker.py b/tutorials/centertrack/byte_tracker.py index 8cb757e0f..ee1579c47 100644 --- a/tutorials/centertrack/byte_tracker.py +++ b/tutorials/centertrack/byte_tracker.py @@ -15,7 +15,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/centertrack/mot_online/matching.py b/tutorials/centertrack/mot_online/matching.py index 54cb4be09..b86608a7d 100644 --- a/tutorials/centertrack/mot_online/matching.py +++ b/tutorials/centertrack/mot_online/matching.py @@ -69,13 +69,13 @@ def ious(atlbrs, btlbrs): :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -109,13 +109,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix @@ -127,17 +127,17 @@ def embedding_distance2(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[0] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[0] for track in tracks], dtype=float) cost_matrix2 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=float) cost_matrix3 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features for row in range(len(cost_matrix)): cost_matrix[row] = (cost_matrix[row]+cost_matrix2[row]+cost_matrix3[row])/3 @@ -149,11 +149,11 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): det_features = [] leg1 = len(tracks) leg2 = len(detections) - cost_matrix = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_det = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_track = np.zeros((leg1, leg2), dtype=np.float) - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.zeros((leg1, leg2), dtype=float) + cost_matrix_det = np.zeros((leg1, leg2), dtype=float) + cost_matrix_track = np.zeros((leg1, leg2), dtype=float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) if leg2 != 0: cost_matrix_det = np.maximum(0.0, cdist(det_features, det_features, metric)) if leg1 != 0: @@ -167,8 +167,8 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): if leg2 > 10: leg2 = 10 detections = detections[:10] - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) return track_features, det_features, cost_matrix, cost_matrix_det, cost_matrix_track def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): diff --git a/tutorials/cstrack/byte_tracker.py b/tutorials/cstrack/byte_tracker.py index 5c54f078f..7b83bff28 100644 --- a/tutorials/cstrack/byte_tracker.py +++ b/tutorials/cstrack/byte_tracker.py @@ -23,7 +23,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/cstrack/tracker.py b/tutorials/cstrack/tracker.py index 99434852c..abff0d963 100644 --- a/tutorials/cstrack/tracker.py +++ b/tutorials/cstrack/tracker.py @@ -23,7 +23,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, temp_feat, buffer_size=30): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/ctracker/byte_tracker.py b/tutorials/ctracker/byte_tracker.py index 0a6ae8011..7f73fb5c7 100644 --- a/tutorials/ctracker/byte_tracker.py +++ b/tutorials/ctracker/byte_tracker.py @@ -17,7 +17,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/ctracker/mot_online/matching.py b/tutorials/ctracker/mot_online/matching.py index 54cb4be09..b86608a7d 100644 --- a/tutorials/ctracker/mot_online/matching.py +++ b/tutorials/ctracker/mot_online/matching.py @@ -69,13 +69,13 @@ def ious(atlbrs, btlbrs): :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -109,13 +109,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix @@ -127,17 +127,17 @@ def embedding_distance2(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[0] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[0] for track in tracks], dtype=float) cost_matrix2 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=float) cost_matrix3 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features for row in range(len(cost_matrix)): cost_matrix[row] = (cost_matrix[row]+cost_matrix2[row]+cost_matrix3[row])/3 @@ -149,11 +149,11 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): det_features = [] leg1 = len(tracks) leg2 = len(detections) - cost_matrix = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_det = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_track = np.zeros((leg1, leg2), dtype=np.float) - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.zeros((leg1, leg2), dtype=float) + cost_matrix_det = np.zeros((leg1, leg2), dtype=float) + cost_matrix_track = np.zeros((leg1, leg2), dtype=float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) if leg2 != 0: cost_matrix_det = np.maximum(0.0, cdist(det_features, det_features, metric)) if leg1 != 0: @@ -167,8 +167,8 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): if leg2 > 10: leg2 = 10 detections = detections[:10] - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) return track_features, det_features, cost_matrix, cost_matrix_det, cost_matrix_track def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): diff --git a/tutorials/fairmot/byte_tracker.py b/tutorials/fairmot/byte_tracker.py index 7bb384dcb..1f64bdf4c 100644 --- a/tutorials/fairmot/byte_tracker.py +++ b/tutorials/fairmot/byte_tracker.py @@ -25,7 +25,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/fairmot/tracker.py b/tutorials/fairmot/tracker.py index b3af90ee0..8f0535d16 100644 --- a/tutorials/fairmot/tracker.py +++ b/tutorials/fairmot/tracker.py @@ -25,7 +25,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, temp_feat, buffer_size=30): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/jde/byte_tracker.py b/tutorials/jde/byte_tracker.py index 185dd67c3..93ea05ccc 100644 --- a/tutorials/jde/byte_tracker.py +++ b/tutorials/jde/byte_tracker.py @@ -1,369 +1,369 @@ -from collections import deque -import torch -import numpy as np -from utils.kalman_filter import KalmanFilter -from utils.log import logger -from models import * -from tracker import matching -from .basetrack import BaseTrack, TrackState - - -class STrack(BaseTrack): - - def __init__(self, tlwh, score): - - # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) - self.kalman_filter = None - self.mean, self.covariance = None, None - self.is_activated = False - - self.score = score - self.tracklet_len = 0 - - def predict(self): - mean_state = self.mean.copy() - if self.state != TrackState.Tracked: - mean_state[7] = 0 - self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) - - @staticmethod - def multi_predict(stracks, kalman_filter): - if len(stracks) > 0: - multi_mean = np.asarray([st.mean.copy() for st in stracks]) - multi_covariance = np.asarray([st.covariance for st in stracks]) - for i, st in enumerate(stracks): - if st.state != TrackState.Tracked: - multi_mean[i][7] = 0 -# multi_mean, multi_covariance = STrack.kalman_filter.multi_predict(multi_mean, multi_covariance) - multi_mean, multi_covariance = kalman_filter.multi_predict(multi_mean, multi_covariance) - for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): - stracks[i].mean = mean - stracks[i].covariance = cov - - def activate(self, kalman_filter, frame_id): - """Start a new tracklet""" - self.kalman_filter = kalman_filter - self.track_id = self.next_id() - self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh)) - - self.tracklet_len = 0 - self.state = TrackState.Tracked - #self.is_activated = True - self.frame_id = frame_id - self.start_frame = frame_id - - def re_activate(self, new_track, frame_id, new_id=False): - self.mean, self.covariance = self.kalman_filter.update( - self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) - ) - - self.tracklet_len = 0 - self.state = TrackState.Tracked - self.is_activated = True - self.frame_id = frame_id - if new_id: - self.track_id = self.next_id() - - def update(self, new_track, frame_id, update_feature=True): - """ - Update a matched track - :type new_track: STrack - :type frame_id: int - :type update_feature: bool - :return: - """ - self.frame_id = frame_id - self.tracklet_len += 1 - - new_tlwh = new_track.tlwh - self.mean, self.covariance = self.kalman_filter.update( - self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)) - self.state = TrackState.Tracked - self.is_activated = True - - self.score = new_track.score - - @property - def tlwh(self): - """Get current position in bounding box format `(top left x, top left y, - width, height)`. - """ - if self.mean is None: - return self._tlwh.copy() - ret = self.mean[:4].copy() - ret[2] *= ret[3] - ret[:2] -= ret[2:] / 2 - return ret - - @property - def tlbr(self): - """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., - `(top left, bottom right)`. - """ - ret = self.tlwh.copy() - ret[2:] += ret[:2] - return ret - - @staticmethod - def tlwh_to_xyah(tlwh): - """Convert bounding box to format `(center x, center y, aspect ratio, - height)`, where the aspect ratio is `width / height`. - """ - ret = np.asarray(tlwh).copy() - ret[:2] += ret[2:] / 2 - ret[2] /= ret[3] - return ret - - def to_xyah(self): - return self.tlwh_to_xyah(self.tlwh) - - @staticmethod - def tlbr_to_tlwh(tlbr): - ret = np.asarray(tlbr).copy() - ret[2:] -= ret[:2] - return ret - - @staticmethod - def tlwh_to_tlbr(tlwh): - ret = np.asarray(tlwh).copy() - ret[2:] += ret[:2] - return ret - - def __repr__(self): - return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame) - - -class BYTETracker(object): - def __init__(self, opt, frame_rate=30): - self.opt = opt - self.model = Darknet(opt.cfg, nID=14455) - # load_darknet_weights(self.model, opt.weights) - self.model.load_state_dict(torch.load(opt.weights, map_location='cpu')['model'], strict=False) - self.model.cuda().eval() - - self.tracked_stracks = [] # type: list[STrack] - self.lost_stracks = [] # type: list[STrack] - self.removed_stracks = [] # type: list[STrack] - - self.frame_id = 0 - self.det_thresh = opt.conf_thres - self.init_thresh = self.det_thresh + 0.2 - self.low_thresh = 0.3 - self.buffer_size = int(frame_rate / 30.0 * opt.track_buffer) - self.max_time_lost = self.buffer_size - - self.kalman_filter = KalmanFilter() - - def update(self, im_blob, img0): - """ - Processes the image frame and finds bounding box(detections). - - Associates the detection with corresponding tracklets and also handles lost, removed, refound and active tracklets - - Parameters - ---------- - im_blob : torch.float32 - Tensor of shape depending upon the size of image. By default, shape of this tensor is [1, 3, 608, 1088] - - img0 : ndarray - ndarray of shape depending on the input image sequence. By default, shape is [608, 1080, 3] - - Returns - ------- - output_stracks : list of Strack(instances) - The list contains information regarding the online_tracklets for the recieved image tensor. - - """ - - self.frame_id += 1 - activated_starcks = [] # for storing active tracks, for the current frame - refind_stracks = [] # Lost Tracks whose detections are obtained in the current frame - lost_stracks = [] # The tracks which are not obtained in the current frame but are not removed.(Lost for some time lesser than the threshold for removing) - removed_stracks = [] - - t1 = time.time() - ''' Step 1: Network forward, get detections & embeddings''' - with torch.no_grad(): - pred = self.model(im_blob) - # pred is tensor of all the proposals (default number of proposals: 54264). Proposals have information associated with the bounding box and embeddings - pred = pred[pred[:, :, 4] > self.low_thresh] - # pred now has lesser number of proposals. Proposals rejected on basis of object confidence score - if len(pred) > 0: - dets = non_max_suppression(pred.unsqueeze(0), self.low_thresh, self.opt.nms_thres)[0].cpu() - # Final proposals are obtained in dets. Information of bounding box and embeddings also included - # Next step changes the detection scales - scale_coords(self.opt.img_size, dets[:, :4], img0.shape).round() - '''Detections is list of (x1, y1, x2, y2, object_conf, class_score, class_pred)''' - # class_pred is the embeddings. - - dets = dets.numpy() - remain_inds = dets[:, 4] > self.det_thresh - inds_low = dets[:, 4] > self.low_thresh - inds_high = dets[:, 4] < self.det_thresh - inds_second = np.logical_and(inds_low, inds_high) - dets_second = dets[inds_second] - dets = dets[remain_inds] - - detections = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4]) for - tlbrs in dets[:, :5]] - else: - detections = [] - dets_second = [] - - t2 = time.time() - # print('Forward: {} s'.format(t2-t1)) - - ''' Add newly detected tracklets to tracked_stracks''' - unconfirmed = [] - tracked_stracks = [] # type: list[STrack] - for track in self.tracked_stracks: - if not track.is_activated: - # previous tracks which are not active in the current frame are added in unconfirmed list - unconfirmed.append(track) - # print("Should not be here, in unconfirmed") - else: - # Active tracks are added to the local list 'tracked_stracks' - tracked_stracks.append(track) - - ''' Step 2: First association, with embedding''' - # Combining currently tracked_stracks and lost_stracks - strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) - # Predict the current location with KF - STrack.multi_predict(strack_pool, self.kalman_filter) - dists = matching.iou_distance(strack_pool, detections) - # The dists is the list of distances of the detection with the tracks in strack_pool - matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.8) - # The matches is the array for corresponding matches of the detection with the corresponding strack_pool - - for itracked, idet in matches: - # itracked is the id of the track and idet is the detection - track = strack_pool[itracked] - det = detections[idet] - if track.state == TrackState.Tracked: - # If the track is active, add the detection to the track - track.update(detections[idet], self.frame_id) - activated_starcks.append(track) - else: - # We have obtained a detection from a track which is not active, hence put the track in refind_stracks list - track.re_activate(det, self.frame_id, new_id=False) - refind_stracks.append(track) - - # association the untrack to the low score detections - if len(dets_second) > 0: - detections_second = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4]) for - tlbrs in dets_second[:, :5]] - else: - detections_second = [] - r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] - dists = matching.iou_distance(r_tracked_stracks, detections_second) - matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.4) - for itracked, idet in matches: - track = r_tracked_stracks[itracked] - det = detections_second[idet] - if track.state == TrackState.Tracked: - track.update(det, self.frame_id) - activated_starcks.append(track) - else: - track.re_activate(det, self.frame_id, new_id=False) - refind_stracks.append(track) - - for it in u_track: - track = r_tracked_stracks[it] - if not track.state == TrackState.Lost: - track.mark_lost() - lost_stracks.append(track) - # If no detections are obtained for tracks (u_track), the tracks are added to lost_tracks list and are marked lost - - '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' - detections = [detections[i] for i in u_detection] - dists = matching.iou_distance(unconfirmed, detections) - matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) - for itracked, idet in matches: - unconfirmed[itracked].update(detections[idet], self.frame_id) - activated_starcks.append(unconfirmed[itracked]) - - # The tracks which are yet not matched - for it in u_unconfirmed: - track = unconfirmed[it] - track.mark_removed() - removed_stracks.append(track) - - # after all these confirmation steps, if a new detection is found, it is initialized for a new track - """ Step 4: Init new stracks""" - for inew in u_detection: - track = detections[inew] - if track.score < self.init_thresh: - continue - track.activate(self.kalman_filter, self.frame_id) - activated_starcks.append(track) - - """ Step 5: Update state""" - # If the tracks are lost for more frames than the threshold number, the tracks are removed. - for track in self.lost_stracks: - if self.frame_id - track.end_frame > self.max_time_lost: - track.mark_removed() - removed_stracks.append(track) - # print('Remained match {} s'.format(t4-t3)) - - # Update the self.tracked_stracks and self.lost_stracks using the updates in this step. - self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] - self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) - self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) - # self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack] - self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) - self.lost_stracks.extend(lost_stracks) - self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) - self.removed_stracks.extend(removed_stracks) - self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) - - # get scores of lost tracks - output_stracks = [track for track in self.tracked_stracks if track.is_activated] - - logger.debug('===========Frame {}=========='.format(self.frame_id)) - logger.debug('Activated: {}'.format([track.track_id for track in activated_starcks])) - logger.debug('Refind: {}'.format([track.track_id for track in refind_stracks])) - logger.debug('Lost: {}'.format([track.track_id for track in lost_stracks])) - logger.debug('Removed: {}'.format([track.track_id for track in removed_stracks])) - # print('Final {} s'.format(t5-t4)) - return output_stracks - -def joint_stracks(tlista, tlistb): - exists = {} - res = [] - for t in tlista: - exists[t.track_id] = 1 - res.append(t) - for t in tlistb: - tid = t.track_id - if not exists.get(tid, 0): - exists[tid] = 1 - res.append(t) - return res - -def sub_stracks(tlista, tlistb): - stracks = {} - for t in tlista: - stracks[t.track_id] = t - for t in tlistb: - tid = t.track_id - if stracks.get(tid, 0): - del stracks[tid] - return list(stracks.values()) - -def remove_duplicate_stracks(stracksa, stracksb): - pdist = matching.iou_distance(stracksa, stracksb) - pairs = np.where(pdist<0.15) - dupa, dupb = list(), list() - for p,q in zip(*pairs): - timep = stracksa[p].frame_id - stracksa[p].start_frame - timeq = stracksb[q].frame_id - stracksb[q].start_frame - if timep > timeq: - dupb.append(q) - else: - dupa.append(p) - resa = [t for i,t in enumerate(stracksa) if not i in dupa] - resb = [t for i,t in enumerate(stracksb) if not i in dupb] - return resa, resb - - +from collections import deque +import torch +import numpy as np +from utils.kalman_filter import KalmanFilter +from utils.log import logger +from models import * +from tracker import matching +from .basetrack import BaseTrack, TrackState + + +class STrack(BaseTrack): + + def __init__(self, tlwh, score): + + # wait activate + self._tlwh = np.asarray(tlwh, dtype=float) + self.kalman_filter = None + self.mean, self.covariance = None, None + self.is_activated = False + + self.score = score + self.tracklet_len = 0 + + def predict(self): + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[7] = 0 + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + @staticmethod + def multi_predict(stracks, kalman_filter): + if len(stracks) > 0: + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][7] = 0 +# multi_mean, multi_covariance = STrack.kalman_filter.multi_predict(multi_mean, multi_covariance) + multi_mean, multi_covariance = kalman_filter.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + def activate(self, kalman_filter, frame_id): + """Start a new tracklet""" + self.kalman_filter = kalman_filter + self.track_id = self.next_id() + self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh)) + + self.tracklet_len = 0 + self.state = TrackState.Tracked + #self.is_activated = True + self.frame_id = frame_id + self.start_frame = frame_id + + def re_activate(self, new_track, frame_id, new_id=False): + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) + ) + + self.tracklet_len = 0 + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + + def update(self, new_track, frame_id, update_feature=True): + """ + Update a matched track + :type new_track: STrack + :type frame_id: int + :type update_feature: bool + :return: + """ + self.frame_id = frame_id + self.tracklet_len += 1 + + new_tlwh = new_track.tlwh + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)) + self.state = TrackState.Tracked + self.is_activated = True + + self.score = new_track.score + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:] / 2 + return ret + + @property + def tlbr(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + + @staticmethod + def tlwh_to_xyah(tlwh): + """Convert bounding box to format `(center x, center y, aspect ratio, + height)`, where the aspect ratio is `width / height`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + return ret + + def to_xyah(self): + return self.tlwh_to_xyah(self.tlwh) + + @staticmethod + def tlbr_to_tlwh(tlbr): + ret = np.asarray(tlbr).copy() + ret[2:] -= ret[:2] + return ret + + @staticmethod + def tlwh_to_tlbr(tlwh): + ret = np.asarray(tlwh).copy() + ret[2:] += ret[:2] + return ret + + def __repr__(self): + return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame) + + +class BYTETracker(object): + def __init__(self, opt, frame_rate=30): + self.opt = opt + self.model = Darknet(opt.cfg, nID=14455) + # load_darknet_weights(self.model, opt.weights) + self.model.load_state_dict(torch.load(opt.weights, map_location='cpu')['model'], strict=False) + self.model.cuda().eval() + + self.tracked_stracks = [] # type: list[STrack] + self.lost_stracks = [] # type: list[STrack] + self.removed_stracks = [] # type: list[STrack] + + self.frame_id = 0 + self.det_thresh = opt.conf_thres + self.init_thresh = self.det_thresh + 0.2 + self.low_thresh = 0.3 + self.buffer_size = int(frame_rate / 30.0 * opt.track_buffer) + self.max_time_lost = self.buffer_size + + self.kalman_filter = KalmanFilter() + + def update(self, im_blob, img0): + """ + Processes the image frame and finds bounding box(detections). + + Associates the detection with corresponding tracklets and also handles lost, removed, refound and active tracklets + + Parameters + ---------- + im_blob : torch.float32 + Tensor of shape depending upon the size of image. By default, shape of this tensor is [1, 3, 608, 1088] + + img0 : ndarray + ndarray of shape depending on the input image sequence. By default, shape is [608, 1080, 3] + + Returns + ------- + output_stracks : list of Strack(instances) + The list contains information regarding the online_tracklets for the recieved image tensor. + + """ + + self.frame_id += 1 + activated_starcks = [] # for storing active tracks, for the current frame + refind_stracks = [] # Lost Tracks whose detections are obtained in the current frame + lost_stracks = [] # The tracks which are not obtained in the current frame but are not removed.(Lost for some time lesser than the threshold for removing) + removed_stracks = [] + + t1 = time.time() + ''' Step 1: Network forward, get detections & embeddings''' + with torch.no_grad(): + pred = self.model(im_blob) + # pred is tensor of all the proposals (default number of proposals: 54264). Proposals have information associated with the bounding box and embeddings + pred = pred[pred[:, :, 4] > self.low_thresh] + # pred now has lesser number of proposals. Proposals rejected on basis of object confidence score + if len(pred) > 0: + dets = non_max_suppression(pred.unsqueeze(0), self.low_thresh, self.opt.nms_thres)[0].cpu() + # Final proposals are obtained in dets. Information of bounding box and embeddings also included + # Next step changes the detection scales + scale_coords(self.opt.img_size, dets[:, :4], img0.shape).round() + '''Detections is list of (x1, y1, x2, y2, object_conf, class_score, class_pred)''' + # class_pred is the embeddings. + + dets = dets.numpy() + remain_inds = dets[:, 4] > self.det_thresh + inds_low = dets[:, 4] > self.low_thresh + inds_high = dets[:, 4] < self.det_thresh + inds_second = np.logical_and(inds_low, inds_high) + dets_second = dets[inds_second] + dets = dets[remain_inds] + + detections = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4]) for + tlbrs in dets[:, :5]] + else: + detections = [] + dets_second = [] + + t2 = time.time() + # print('Forward: {} s'.format(t2-t1)) + + ''' Add newly detected tracklets to tracked_stracks''' + unconfirmed = [] + tracked_stracks = [] # type: list[STrack] + for track in self.tracked_stracks: + if not track.is_activated: + # previous tracks which are not active in the current frame are added in unconfirmed list + unconfirmed.append(track) + # print("Should not be here, in unconfirmed") + else: + # Active tracks are added to the local list 'tracked_stracks' + tracked_stracks.append(track) + + ''' Step 2: First association, with embedding''' + # Combining currently tracked_stracks and lost_stracks + strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) + # Predict the current location with KF + STrack.multi_predict(strack_pool, self.kalman_filter) + dists = matching.iou_distance(strack_pool, detections) + # The dists is the list of distances of the detection with the tracks in strack_pool + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.8) + # The matches is the array for corresponding matches of the detection with the corresponding strack_pool + + for itracked, idet in matches: + # itracked is the id of the track and idet is the detection + track = strack_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + # If the track is active, add the detection to the track + track.update(detections[idet], self.frame_id) + activated_starcks.append(track) + else: + # We have obtained a detection from a track which is not active, hence put the track in refind_stracks list + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + # association the untrack to the low score detections + if len(dets_second) > 0: + detections_second = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4]) for + tlbrs in dets_second[:, :5]] + else: + detections_second = [] + r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] + dists = matching.iou_distance(r_tracked_stracks, detections_second) + matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.4) + for itracked, idet in matches: + track = r_tracked_stracks[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_starcks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + for it in u_track: + track = r_tracked_stracks[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_stracks.append(track) + # If no detections are obtained for tracks (u_track), the tracks are added to lost_tracks list and are marked lost + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = matching.iou_distance(unconfirmed, detections) + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_starcks.append(unconfirmed[itracked]) + + # The tracks which are yet not matched + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_stracks.append(track) + + # after all these confirmation steps, if a new detection is found, it is initialized for a new track + """ Step 4: Init new stracks""" + for inew in u_detection: + track = detections[inew] + if track.score < self.init_thresh: + continue + track.activate(self.kalman_filter, self.frame_id) + activated_starcks.append(track) + + """ Step 5: Update state""" + # If the tracks are lost for more frames than the threshold number, the tracks are removed. + for track in self.lost_stracks: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_stracks.append(track) + # print('Remained match {} s'.format(t4-t3)) + + # Update the self.tracked_stracks and self.lost_stracks using the updates in this step. + self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] + self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) + self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) + # self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack] + self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) + self.lost_stracks.extend(lost_stracks) + self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) + self.removed_stracks.extend(removed_stracks) + self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) + + # get scores of lost tracks + output_stracks = [track for track in self.tracked_stracks if track.is_activated] + + logger.debug('===========Frame {}=========='.format(self.frame_id)) + logger.debug('Activated: {}'.format([track.track_id for track in activated_starcks])) + logger.debug('Refind: {}'.format([track.track_id for track in refind_stracks])) + logger.debug('Lost: {}'.format([track.track_id for track in lost_stracks])) + logger.debug('Removed: {}'.format([track.track_id for track in removed_stracks])) + # print('Final {} s'.format(t5-t4)) + return output_stracks + +def joint_stracks(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + +def sub_stracks(tlista, tlistb): + stracks = {} + for t in tlista: + stracks[t.track_id] = t + for t in tlistb: + tid = t.track_id + if stracks.get(tid, 0): + del stracks[tid] + return list(stracks.values()) + +def remove_duplicate_stracks(stracksa, stracksb): + pdist = matching.iou_distance(stracksa, stracksb) + pairs = np.where(pdist<0.15) + dupa, dupb = list(), list() + for p,q in zip(*pairs): + timep = stracksa[p].frame_id - stracksa[p].start_frame + timeq = stracksb[q].frame_id - stracksb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i,t in enumerate(stracksa) if not i in dupa] + resb = [t for i,t in enumerate(stracksb) if not i in dupb] + return resa, resb + + diff --git a/tutorials/jde/evaluation.py b/tutorials/jde/evaluation.py index d6702cf6e..8b812ce75 100644 --- a/tutorials/jde/evaluation.py +++ b/tutorials/jde/evaluation.py @@ -1,104 +1,104 @@ -import os -import numpy as np -import copy -import motmetrics as mm -mm.lap.default_solver = 'lap' -from utils.io import read_results, unzip_objs - - -class Evaluator(object): - - def __init__(self, data_root, seq_name, data_type): - self.data_root = data_root - self.seq_name = seq_name - self.data_type = data_type - - self.load_annotations() - self.reset_accumulator() - - def load_annotations(self): - assert self.data_type == 'mot' - - gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt') - self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True) - self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True) - - def reset_accumulator(self): - self.acc = mm.MOTAccumulator(auto_id=True) - - def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False): - # results - trk_tlwhs = np.copy(trk_tlwhs) - trk_ids = np.copy(trk_ids) - - # gts - gt_objs = self.gt_frame_dict.get(frame_id, []) - gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2] - - # ignore boxes - ignore_objs = self.gt_ignore_frame_dict.get(frame_id, []) - ignore_tlwhs = unzip_objs(ignore_objs)[0] - - - # remove ignored results - keep = np.ones(len(trk_tlwhs), dtype=bool) - iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5) - if len(iou_distance) > 0: - match_is, match_js = mm.lap.linear_sum_assignment(iou_distance) - match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js]) - match_ious = iou_distance[match_is, match_js] - - match_js = np.asarray(match_js, dtype=int) - match_js = match_js[np.logical_not(np.isnan(match_ious))] - keep[match_js] = False - trk_tlwhs = trk_tlwhs[keep] - trk_ids = trk_ids[keep] - - # get distance matrix - iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5) - - # acc - self.acc.update(gt_ids, trk_ids, iou_distance) - - if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'): - events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics - else: - events = None - return events - - def eval_file(self, filename): - self.reset_accumulator() - - result_frame_dict = read_results(filename, self.data_type, is_gt=False) - #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys()))) - frames = sorted(list(set(result_frame_dict.keys()))) - for frame_id in frames: - trk_objs = result_frame_dict.get(frame_id, []) - trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2] - self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False) - - return self.acc - - @staticmethod - def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')): - names = copy.deepcopy(names) - if metrics is None: - metrics = mm.metrics.motchallenge_metrics - metrics = copy.deepcopy(metrics) - - mh = mm.metrics.create() - summary = mh.compute_many( - accs, - metrics=metrics, - names=names, - generate_overall=True - ) - - return summary - - @staticmethod - def save_summary(summary, filename): - import pandas as pd - writer = pd.ExcelWriter(filename) - summary.to_excel(writer) - writer.save() +import os +import numpy as np +import copy +import motmetrics as mm +mm.lap.default_solver = 'lap' +from utils.io import read_results, unzip_objs + + +class Evaluator(object): + + def __init__(self, data_root, seq_name, data_type): + self.data_root = data_root + self.seq_name = seq_name + self.data_type = data_type + + self.load_annotations() + self.reset_accumulator() + + def load_annotations(self): + assert self.data_type == 'mot' + + gt_filename = os.path.join(self.data_root, self.seq_name, 'gt', 'gt.txt') + self.gt_frame_dict = read_results(gt_filename, self.data_type, is_gt=True) + self.gt_ignore_frame_dict = read_results(gt_filename, self.data_type, is_ignore=True) + + def reset_accumulator(self): + self.acc = mm.MOTAccumulator(auto_id=True) + + def eval_frame(self, frame_id, trk_tlwhs, trk_ids, rtn_events=False): + # results + trk_tlwhs = np.copy(trk_tlwhs) + trk_ids = np.copy(trk_ids) + + # gts + gt_objs = self.gt_frame_dict.get(frame_id, []) + gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2] + + # ignore boxes + ignore_objs = self.gt_ignore_frame_dict.get(frame_id, []) + ignore_tlwhs = unzip_objs(ignore_objs)[0] + + + # remove ignored results + keep = np.ones(len(trk_tlwhs), dtype=bool) + iou_distance = mm.distances.iou_matrix(ignore_tlwhs, trk_tlwhs, max_iou=0.5) + if len(iou_distance) > 0: + match_is, match_js = mm.lap.linear_sum_assignment(iou_distance) + match_is, match_js = map(lambda a: np.asarray(a, dtype=int), [match_is, match_js]) + match_ious = iou_distance[match_is, match_js] + + match_js = np.asarray(match_js, dtype=int) + match_js = match_js[np.logical_not(np.isnan(match_ious))] + keep[match_js] = False + trk_tlwhs = trk_tlwhs[keep] + trk_ids = trk_ids[keep] + + # get distance matrix + iou_distance = mm.distances.iou_matrix(gt_tlwhs, trk_tlwhs, max_iou=0.5) + + # acc + self.acc.update(gt_ids, trk_ids, iou_distance) + + if rtn_events and iou_distance.size > 0 and hasattr(self.acc, 'last_mot_events'): + events = self.acc.last_mot_events # only supported by https://github.com/longcw/py-motmetrics + else: + events = None + return events + + def eval_file(self, filename): + self.reset_accumulator() + + result_frame_dict = read_results(filename, self.data_type, is_gt=False) + #frames = sorted(list(set(self.gt_frame_dict.keys()) | set(result_frame_dict.keys()))) + frames = sorted(list(set(result_frame_dict.keys()))) + for frame_id in frames: + trk_objs = result_frame_dict.get(frame_id, []) + trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2] + self.eval_frame(frame_id, trk_tlwhs, trk_ids, rtn_events=False) + + return self.acc + + @staticmethod + def get_summary(accs, names, metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1', 'precision', 'recall')): + names = copy.deepcopy(names) + if metrics is None: + metrics = mm.metrics.motchallenge_metrics + metrics = copy.deepcopy(metrics) + + mh = mm.metrics.create() + summary = mh.compute_many( + accs, + metrics=metrics, + names=names, + generate_overall=True + ) + + return summary + + @staticmethod + def save_summary(summary, filename): + import pandas as pd + writer = pd.ExcelWriter(filename) + summary.to_excel(writer) + writer.save() diff --git a/tutorials/jde/tracker.py b/tutorials/jde/tracker.py index 2b0cab6bf..425e34668 100644 --- a/tutorials/jde/tracker.py +++ b/tutorials/jde/tracker.py @@ -1,414 +1,414 @@ - -from collections import deque -import torch -import numpy as np -from utils.kalman_filter import KalmanFilter -from utils.log import logger -from models import * -from tracker import matching -from .basetrack import BaseTrack, TrackState - - -class STrack(BaseTrack): - - def __init__(self, tlwh, score, temp_feat, buffer_size=30): - - # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) - self.kalman_filter = None - self.mean, self.covariance = None, None - self.is_activated = False - - self.score = score - self.tracklet_len = 0 - - self.smooth_feat = None - self.update_features(temp_feat) - self.features = deque([], maxlen=buffer_size) - self.alpha = 0.9 - - def update_features(self, feat): - feat /= np.linalg.norm(feat) - self.curr_feat = feat - if self.smooth_feat is None: - self.smooth_feat = feat - else: - self.smooth_feat = self.alpha *self.smooth_feat + (1-self.alpha) * feat - self.features.append(feat) - self.smooth_feat /= np.linalg.norm(self.smooth_feat) - - def predict(self): - mean_state = self.mean.copy() - if self.state != TrackState.Tracked: - mean_state[7] = 0 - self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) - - @staticmethod - def multi_predict(stracks, kalman_filter): - if len(stracks) > 0: - multi_mean = np.asarray([st.mean.copy() for st in stracks]) - multi_covariance = np.asarray([st.covariance for st in stracks]) - for i, st in enumerate(stracks): - if st.state != TrackState.Tracked: - multi_mean[i][7] = 0 -# multi_mean, multi_covariance = STrack.kalman_filter.multi_predict(multi_mean, multi_covariance) - multi_mean, multi_covariance = kalman_filter.multi_predict(multi_mean, multi_covariance) - for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): - stracks[i].mean = mean - stracks[i].covariance = cov - - def activate(self, kalman_filter, frame_id): - """Start a new tracklet""" - self.kalman_filter = kalman_filter - self.track_id = self.next_id() - self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh)) - - self.tracklet_len = 0 - self.state = TrackState.Tracked - #self.is_activated = True - self.frame_id = frame_id - self.start_frame = frame_id - - def re_activate(self, new_track, frame_id, new_id=False): - self.mean, self.covariance = self.kalman_filter.update( - self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) - ) - - self.update_features(new_track.curr_feat) - self.tracklet_len = 0 - self.state = TrackState.Tracked - self.is_activated = True - self.frame_id = frame_id - if new_id: - self.track_id = self.next_id() - - def update(self, new_track, frame_id, update_feature=True): - """ - Update a matched track - :type new_track: STrack - :type frame_id: int - :type update_feature: bool - :return: - """ - self.frame_id = frame_id - self.tracklet_len += 1 - - new_tlwh = new_track.tlwh - self.mean, self.covariance = self.kalman_filter.update( - self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)) - self.state = TrackState.Tracked - self.is_activated = True - - self.score = new_track.score - if update_feature: - self.update_features(new_track.curr_feat) - - @property - def tlwh(self): - """Get current position in bounding box format `(top left x, top left y, - width, height)`. - """ - if self.mean is None: - return self._tlwh.copy() - ret = self.mean[:4].copy() - ret[2] *= ret[3] - ret[:2] -= ret[2:] / 2 - return ret - - @property - def tlbr(self): - """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., - `(top left, bottom right)`. - """ - ret = self.tlwh.copy() - ret[2:] += ret[:2] - return ret - - @staticmethod - def tlwh_to_xyah(tlwh): - """Convert bounding box to format `(center x, center y, aspect ratio, - height)`, where the aspect ratio is `width / height`. - """ - ret = np.asarray(tlwh).copy() - ret[:2] += ret[2:] / 2 - ret[2] /= ret[3] - return ret - - def to_xyah(self): - return self.tlwh_to_xyah(self.tlwh) - - @staticmethod - def tlbr_to_tlwh(tlbr): - ret = np.asarray(tlbr).copy() - ret[2:] -= ret[:2] - return ret - - @staticmethod - def tlwh_to_tlbr(tlwh): - ret = np.asarray(tlwh).copy() - ret[2:] += ret[:2] - return ret - - def __repr__(self): - return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame) - - -class JDETracker(object): - def __init__(self, opt, frame_rate=30): - self.opt = opt - self.model = Darknet(opt.cfg, nID=14455) - # load_darknet_weights(self.model, opt.weights) - self.model.load_state_dict(torch.load(opt.weights, map_location='cpu')['model'], strict=False) - self.model.cuda().eval() - - self.tracked_stracks = [] # type: list[STrack] - self.lost_stracks = [] # type: list[STrack] - self.removed_stracks = [] # type: list[STrack] - - self.frame_id = 0 - self.det_thresh = opt.conf_thres - self.init_thresh = self.det_thresh + 0.2 - self.low_thresh = 0.4 - self.buffer_size = int(frame_rate / 30.0 * opt.track_buffer) - self.max_time_lost = self.buffer_size - - self.kalman_filter = KalmanFilter() - - def update(self, im_blob, img0): - """ - Processes the image frame and finds bounding box(detections). - - Associates the detection with corresponding tracklets and also handles lost, removed, refound and active tracklets - - Parameters - ---------- - im_blob : torch.float32 - Tensor of shape depending upon the size of image. By default, shape of this tensor is [1, 3, 608, 1088] - - img0 : ndarray - ndarray of shape depending on the input image sequence. By default, shape is [608, 1080, 3] - - Returns - ------- - output_stracks : list of Strack(instances) - The list contains information regarding the online_tracklets for the recieved image tensor. - - """ - - self.frame_id += 1 - activated_starcks = [] # for storing active tracks, for the current frame - refind_stracks = [] # Lost Tracks whose detections are obtained in the current frame - lost_stracks = [] # The tracks which are not obtained in the current frame but are not removed.(Lost for some time lesser than the threshold for removing) - removed_stracks = [] - - t1 = time.time() - ''' Step 1: Network forward, get detections & embeddings''' - with torch.no_grad(): - pred = self.model(im_blob) - # pred is tensor of all the proposals (default number of proposals: 54264). Proposals have information associated with the bounding box and embeddings - pred = pred[pred[:, :, 4] > self.low_thresh] - # pred now has lesser number of proposals. Proposals rejected on basis of object confidence score - if len(pred) > 0: - dets = non_max_suppression(pred.unsqueeze(0), self.low_thresh, self.opt.nms_thres)[0].cpu() - # Final proposals are obtained in dets. Information of bounding box and embeddings also included - # Next step changes the detection scales - scale_coords(self.opt.img_size, dets[:, :4], img0.shape).round() - '''Detections is list of (x1, y1, x2, y2, object_conf, class_score, class_pred)''' - # class_pred is the embeddings. - - dets = dets.numpy() - remain_inds = dets[:, 4] > self.det_thresh - inds_low = dets[:, 4] > self.low_thresh - inds_high = dets[:, 4] < self.det_thresh - inds_second = np.logical_and(inds_low, inds_high) - dets_second = dets[inds_second] - dets = dets[remain_inds] - - detections = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for - (tlbrs, f) in zip(dets[:, :5], dets[:, 6:])] - else: - detections = [] - dets_second = [] - - t2 = time.time() - # print('Forward: {} s'.format(t2-t1)) - - ''' Add newly detected tracklets to tracked_stracks''' - unconfirmed = [] - tracked_stracks = [] # type: list[STrack] - for track in self.tracked_stracks: - if not track.is_activated: - # previous tracks which are not active in the current frame are added in unconfirmed list - unconfirmed.append(track) - # print("Should not be here, in unconfirmed") - else: - # Active tracks are added to the local list 'tracked_stracks' - tracked_stracks.append(track) - - ''' Step 2: First association, with embedding''' - # Combining currently tracked_stracks and lost_stracks - strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) - # Predict the current location with KF - STrack.multi_predict(strack_pool, self.kalman_filter) - - dists = matching.embedding_distance(strack_pool, detections) - dists = matching.fuse_motion(self.kalman_filter, dists, strack_pool, detections) - #dists = matching.iou_distance(strack_pool, detections) - # The dists is the list of distances of the detection with the tracks in strack_pool - matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.7) - # The matches is the array for corresponding matches of the detection with the corresponding strack_pool - - for itracked, idet in matches: - # itracked is the id of the track and idet is the detection - track = strack_pool[itracked] - det = detections[idet] - if track.state == TrackState.Tracked: - # If the track is active, add the detection to the track - track.update(detections[idet], self.frame_id) - activated_starcks.append(track) - else: - # We have obtained a detection from a track which is not active, hence put the track in refind_stracks list - track.re_activate(det, self.frame_id, new_id=False) - refind_stracks.append(track) - - # None of the steps below happen if there are no undetected tracks. - ''' Step 3: Second association, with IOU''' - detections = [detections[i] for i in u_detection] - # detections is now a list of the unmatched detections - r_tracked_stracks = [] # This is container for stracks which were tracked till the - # previous frame but no detection was found for it in the current frame - for i in u_track: - if strack_pool[i].state == TrackState.Tracked: - r_tracked_stracks.append(strack_pool[i]) - dists = matching.iou_distance(r_tracked_stracks, detections) - matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.5) - # matches is the list of detections which matched with corresponding tracks by IOU distance method - for itracked, idet in matches: - track = r_tracked_stracks[itracked] - det = detections[idet] - if track.state == TrackState.Tracked: - track.update(det, self.frame_id) - activated_starcks.append(track) - else: - track.re_activate(det, self.frame_id, new_id=False) - refind_stracks.append(track) - # Same process done for some unmatched detections, but now considering IOU_distance as measure - - # association the untrack to the low score detections - if len(dets_second) > 0: - detections_second = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for - (tlbrs, f) in zip(dets_second[:, :5], dets_second[:, 6:])] - else: - detections_second = [] - second_tracked_stracks = [r_tracked_stracks[i] for i in u_track if r_tracked_stracks[i].state == TrackState.Tracked] - dists = matching.iou_distance(second_tracked_stracks, detections_second) - matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.4) - for itracked, idet in matches: - track = second_tracked_stracks[itracked] - det = detections_second[idet] - if track.state == TrackState.Tracked: - track.update(det, self.frame_id) - activated_starcks.append(track) - else: - track.re_activate(det, self.frame_id, new_id=False) - refind_stracks.append(track) - - for it in u_track: - track = second_tracked_stracks[it] - if not track.state == TrackState.Lost: - track.mark_lost() - lost_stracks.append(track) - # If no detections are obtained for tracks (u_track), the tracks are added to lost_tracks list and are marked lost - - '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' - detections = [detections[i] for i in u_detection] - dists = matching.iou_distance(unconfirmed, detections) - matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) - for itracked, idet in matches: - unconfirmed[itracked].update(detections[idet], self.frame_id) - activated_starcks.append(unconfirmed[itracked]) - - # The tracks which are yet not matched - for it in u_unconfirmed: - track = unconfirmed[it] - track.mark_removed() - removed_stracks.append(track) - - # after all these confirmation steps, if a new detection is found, it is initialized for a new track - """ Step 4: Init new stracks""" - for inew in u_detection: - track = detections[inew] - if track.score < self.init_thresh: - continue - track.activate(self.kalman_filter, self.frame_id) - activated_starcks.append(track) - - """ Step 5: Update state""" - # If the tracks are lost for more frames than the threshold number, the tracks are removed. - for track in self.lost_stracks: - if self.frame_id - track.end_frame > self.max_time_lost: - track.mark_removed() - removed_stracks.append(track) - # print('Remained match {} s'.format(t4-t3)) - - # Update the self.tracked_stracks and self.lost_stracks using the updates in this step. - self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] - self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) - self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) - # self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack] - self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) - self.lost_stracks.extend(lost_stracks) - self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) - self.removed_stracks.extend(removed_stracks) - self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) - - # get scores of lost tracks - output_stracks = [track for track in self.tracked_stracks if track.is_activated] - - logger.debug('===========Frame {}=========='.format(self.frame_id)) - logger.debug('Activated: {}'.format([track.track_id for track in activated_starcks])) - logger.debug('Refind: {}'.format([track.track_id for track in refind_stracks])) - logger.debug('Lost: {}'.format([track.track_id for track in lost_stracks])) - logger.debug('Removed: {}'.format([track.track_id for track in removed_stracks])) - # print('Final {} s'.format(t5-t4)) - return output_stracks - -def joint_stracks(tlista, tlistb): - exists = {} - res = [] - for t in tlista: - exists[t.track_id] = 1 - res.append(t) - for t in tlistb: - tid = t.track_id - if not exists.get(tid, 0): - exists[tid] = 1 - res.append(t) - return res - -def sub_stracks(tlista, tlistb): - stracks = {} - for t in tlista: - stracks[t.track_id] = t - for t in tlistb: - tid = t.track_id - if stracks.get(tid, 0): - del stracks[tid] - return list(stracks.values()) - -def remove_duplicate_stracks(stracksa, stracksb): - pdist = matching.iou_distance(stracksa, stracksb) - pairs = np.where(pdist<0.15) - dupa, dupb = list(), list() - for p,q in zip(*pairs): - timep = stracksa[p].frame_id - stracksa[p].start_frame - timeq = stracksb[q].frame_id - stracksb[q].start_frame - if timep > timeq: - dupb.append(q) - else: - dupa.append(p) - resa = [t for i,t in enumerate(stracksa) if not i in dupa] - resb = [t for i,t in enumerate(stracksb) if not i in dupb] - return resa, resb - - + +from collections import deque +import torch +import numpy as np +from utils.kalman_filter import KalmanFilter +from utils.log import logger +from models import * +from tracker import matching +from .basetrack import BaseTrack, TrackState + + +class STrack(BaseTrack): + + def __init__(self, tlwh, score, temp_feat, buffer_size=30): + + # wait activate + self._tlwh = np.asarray(tlwh, dtype=float) + self.kalman_filter = None + self.mean, self.covariance = None, None + self.is_activated = False + + self.score = score + self.tracklet_len = 0 + + self.smooth_feat = None + self.update_features(temp_feat) + self.features = deque([], maxlen=buffer_size) + self.alpha = 0.9 + + def update_features(self, feat): + feat /= np.linalg.norm(feat) + self.curr_feat = feat + if self.smooth_feat is None: + self.smooth_feat = feat + else: + self.smooth_feat = self.alpha *self.smooth_feat + (1-self.alpha) * feat + self.features.append(feat) + self.smooth_feat /= np.linalg.norm(self.smooth_feat) + + def predict(self): + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[7] = 0 + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + @staticmethod + def multi_predict(stracks, kalman_filter): + if len(stracks) > 0: + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][7] = 0 +# multi_mean, multi_covariance = STrack.kalman_filter.multi_predict(multi_mean, multi_covariance) + multi_mean, multi_covariance = kalman_filter.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + def activate(self, kalman_filter, frame_id): + """Start a new tracklet""" + self.kalman_filter = kalman_filter + self.track_id = self.next_id() + self.mean, self.covariance = self.kalman_filter.initiate(self.tlwh_to_xyah(self._tlwh)) + + self.tracklet_len = 0 + self.state = TrackState.Tracked + #self.is_activated = True + self.frame_id = frame_id + self.start_frame = frame_id + + def re_activate(self, new_track, frame_id, new_id=False): + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.tlwh_to_xyah(new_track.tlwh) + ) + + self.update_features(new_track.curr_feat) + self.tracklet_len = 0 + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + + def update(self, new_track, frame_id, update_feature=True): + """ + Update a matched track + :type new_track: STrack + :type frame_id: int + :type update_feature: bool + :return: + """ + self.frame_id = frame_id + self.tracklet_len += 1 + + new_tlwh = new_track.tlwh + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.tlwh_to_xyah(new_tlwh)) + self.state = TrackState.Tracked + self.is_activated = True + + self.score = new_track.score + if update_feature: + self.update_features(new_track.curr_feat) + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:] / 2 + return ret + + @property + def tlbr(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + + @staticmethod + def tlwh_to_xyah(tlwh): + """Convert bounding box to format `(center x, center y, aspect ratio, + height)`, where the aspect ratio is `width / height`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + return ret + + def to_xyah(self): + return self.tlwh_to_xyah(self.tlwh) + + @staticmethod + def tlbr_to_tlwh(tlbr): + ret = np.asarray(tlbr).copy() + ret[2:] -= ret[:2] + return ret + + @staticmethod + def tlwh_to_tlbr(tlwh): + ret = np.asarray(tlwh).copy() + ret[2:] += ret[:2] + return ret + + def __repr__(self): + return 'OT_{}_({}-{})'.format(self.track_id, self.start_frame, self.end_frame) + + +class JDETracker(object): + def __init__(self, opt, frame_rate=30): + self.opt = opt + self.model = Darknet(opt.cfg, nID=14455) + # load_darknet_weights(self.model, opt.weights) + self.model.load_state_dict(torch.load(opt.weights, map_location='cpu')['model'], strict=False) + self.model.cuda().eval() + + self.tracked_stracks = [] # type: list[STrack] + self.lost_stracks = [] # type: list[STrack] + self.removed_stracks = [] # type: list[STrack] + + self.frame_id = 0 + self.det_thresh = opt.conf_thres + self.init_thresh = self.det_thresh + 0.2 + self.low_thresh = 0.4 + self.buffer_size = int(frame_rate / 30.0 * opt.track_buffer) + self.max_time_lost = self.buffer_size + + self.kalman_filter = KalmanFilter() + + def update(self, im_blob, img0): + """ + Processes the image frame and finds bounding box(detections). + + Associates the detection with corresponding tracklets and also handles lost, removed, refound and active tracklets + + Parameters + ---------- + im_blob : torch.float32 + Tensor of shape depending upon the size of image. By default, shape of this tensor is [1, 3, 608, 1088] + + img0 : ndarray + ndarray of shape depending on the input image sequence. By default, shape is [608, 1080, 3] + + Returns + ------- + output_stracks : list of Strack(instances) + The list contains information regarding the online_tracklets for the recieved image tensor. + + """ + + self.frame_id += 1 + activated_starcks = [] # for storing active tracks, for the current frame + refind_stracks = [] # Lost Tracks whose detections are obtained in the current frame + lost_stracks = [] # The tracks which are not obtained in the current frame but are not removed.(Lost for some time lesser than the threshold for removing) + removed_stracks = [] + + t1 = time.time() + ''' Step 1: Network forward, get detections & embeddings''' + with torch.no_grad(): + pred = self.model(im_blob) + # pred is tensor of all the proposals (default number of proposals: 54264). Proposals have information associated with the bounding box and embeddings + pred = pred[pred[:, :, 4] > self.low_thresh] + # pred now has lesser number of proposals. Proposals rejected on basis of object confidence score + if len(pred) > 0: + dets = non_max_suppression(pred.unsqueeze(0), self.low_thresh, self.opt.nms_thres)[0].cpu() + # Final proposals are obtained in dets. Information of bounding box and embeddings also included + # Next step changes the detection scales + scale_coords(self.opt.img_size, dets[:, :4], img0.shape).round() + '''Detections is list of (x1, y1, x2, y2, object_conf, class_score, class_pred)''' + # class_pred is the embeddings. + + dets = dets.numpy() + remain_inds = dets[:, 4] > self.det_thresh + inds_low = dets[:, 4] > self.low_thresh + inds_high = dets[:, 4] < self.det_thresh + inds_second = np.logical_and(inds_low, inds_high) + dets_second = dets[inds_second] + dets = dets[remain_inds] + + detections = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for + (tlbrs, f) in zip(dets[:, :5], dets[:, 6:])] + else: + detections = [] + dets_second = [] + + t2 = time.time() + # print('Forward: {} s'.format(t2-t1)) + + ''' Add newly detected tracklets to tracked_stracks''' + unconfirmed = [] + tracked_stracks = [] # type: list[STrack] + for track in self.tracked_stracks: + if not track.is_activated: + # previous tracks which are not active in the current frame are added in unconfirmed list + unconfirmed.append(track) + # print("Should not be here, in unconfirmed") + else: + # Active tracks are added to the local list 'tracked_stracks' + tracked_stracks.append(track) + + ''' Step 2: First association, with embedding''' + # Combining currently tracked_stracks and lost_stracks + strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) + # Predict the current location with KF + STrack.multi_predict(strack_pool, self.kalman_filter) + + dists = matching.embedding_distance(strack_pool, detections) + dists = matching.fuse_motion(self.kalman_filter, dists, strack_pool, detections) + #dists = matching.iou_distance(strack_pool, detections) + # The dists is the list of distances of the detection with the tracks in strack_pool + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.7) + # The matches is the array for corresponding matches of the detection with the corresponding strack_pool + + for itracked, idet in matches: + # itracked is the id of the track and idet is the detection + track = strack_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + # If the track is active, add the detection to the track + track.update(detections[idet], self.frame_id) + activated_starcks.append(track) + else: + # We have obtained a detection from a track which is not active, hence put the track in refind_stracks list + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + # None of the steps below happen if there are no undetected tracks. + ''' Step 3: Second association, with IOU''' + detections = [detections[i] for i in u_detection] + # detections is now a list of the unmatched detections + r_tracked_stracks = [] # This is container for stracks which were tracked till the + # previous frame but no detection was found for it in the current frame + for i in u_track: + if strack_pool[i].state == TrackState.Tracked: + r_tracked_stracks.append(strack_pool[i]) + dists = matching.iou_distance(r_tracked_stracks, detections) + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=0.5) + # matches is the list of detections which matched with corresponding tracks by IOU distance method + for itracked, idet in matches: + track = r_tracked_stracks[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_starcks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + # Same process done for some unmatched detections, but now considering IOU_distance as measure + + # association the untrack to the low score detections + if len(dets_second) > 0: + detections_second = [STrack(STrack.tlbr_to_tlwh(tlbrs[:4]), tlbrs[4], f, 30) for + (tlbrs, f) in zip(dets_second[:, :5], dets_second[:, 6:])] + else: + detections_second = [] + second_tracked_stracks = [r_tracked_stracks[i] for i in u_track if r_tracked_stracks[i].state == TrackState.Tracked] + dists = matching.iou_distance(second_tracked_stracks, detections_second) + matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.4) + for itracked, idet in matches: + track = second_tracked_stracks[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_starcks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + for it in u_track: + track = second_tracked_stracks[it] + if not track.state == TrackState.Lost: + track.mark_lost() + lost_stracks.append(track) + # If no detections are obtained for tracks (u_track), the tracks are added to lost_tracks list and are marked lost + + '''Deal with unconfirmed tracks, usually tracks with only one beginning frame''' + detections = [detections[i] for i in u_detection] + dists = matching.iou_distance(unconfirmed, detections) + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_starcks.append(unconfirmed[itracked]) + + # The tracks which are yet not matched + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_stracks.append(track) + + # after all these confirmation steps, if a new detection is found, it is initialized for a new track + """ Step 4: Init new stracks""" + for inew in u_detection: + track = detections[inew] + if track.score < self.init_thresh: + continue + track.activate(self.kalman_filter, self.frame_id) + activated_starcks.append(track) + + """ Step 5: Update state""" + # If the tracks are lost for more frames than the threshold number, the tracks are removed. + for track in self.lost_stracks: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_stracks.append(track) + # print('Remained match {} s'.format(t4-t3)) + + # Update the self.tracked_stracks and self.lost_stracks using the updates in this step. + self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] + self.tracked_stracks = joint_stracks(self.tracked_stracks, activated_starcks) + self.tracked_stracks = joint_stracks(self.tracked_stracks, refind_stracks) + # self.lost_stracks = [t for t in self.lost_stracks if t.state == TrackState.Lost] # type: list[STrack] + self.lost_stracks = sub_stracks(self.lost_stracks, self.tracked_stracks) + self.lost_stracks.extend(lost_stracks) + self.lost_stracks = sub_stracks(self.lost_stracks, self.removed_stracks) + self.removed_stracks.extend(removed_stracks) + self.tracked_stracks, self.lost_stracks = remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) + + # get scores of lost tracks + output_stracks = [track for track in self.tracked_stracks if track.is_activated] + + logger.debug('===========Frame {}=========='.format(self.frame_id)) + logger.debug('Activated: {}'.format([track.track_id for track in activated_starcks])) + logger.debug('Refind: {}'.format([track.track_id for track in refind_stracks])) + logger.debug('Lost: {}'.format([track.track_id for track in lost_stracks])) + logger.debug('Removed: {}'.format([track.track_id for track in removed_stracks])) + # print('Final {} s'.format(t5-t4)) + return output_stracks + +def joint_stracks(tlista, tlistb): + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + +def sub_stracks(tlista, tlistb): + stracks = {} + for t in tlista: + stracks[t.track_id] = t + for t in tlistb: + tid = t.track_id + if stracks.get(tid, 0): + del stracks[tid] + return list(stracks.values()) + +def remove_duplicate_stracks(stracksa, stracksb): + pdist = matching.iou_distance(stracksa, stracksb) + pairs = np.where(pdist<0.15) + dupa, dupb = list(), list() + for p,q in zip(*pairs): + timep = stracksa[p].frame_id - stracksa[p].start_frame + timeq = stracksb[q].frame_id - stracksb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i,t in enumerate(stracksa) if not i in dupa] + resb = [t for i,t in enumerate(stracksb) if not i in dupb] + return resa, resb + + diff --git a/tutorials/motr/byte_tracker.py b/tutorials/motr/byte_tracker.py index d5bc6dd47..1152d6182 100644 --- a/tutorials/motr/byte_tracker.py +++ b/tutorials/motr/byte_tracker.py @@ -17,7 +17,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/motr/mot_online/matching.py b/tutorials/motr/mot_online/matching.py index cc7abab60..da24f4f11 100644 --- a/tutorials/motr/mot_online/matching.py +++ b/tutorials/motr/mot_online/matching.py @@ -68,13 +68,13 @@ def ious(atlbrs, btlbrs): :type atlbrs: list[tlbr] | np.ndarray :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -107,13 +107,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix @@ -125,17 +125,17 @@ def embedding_distance2(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[0] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[0] for track in tracks], dtype=float) cost_matrix2 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=float) cost_matrix3 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features for row in range(len(cost_matrix)): cost_matrix[row] = (cost_matrix[row]+cost_matrix2[row]+cost_matrix3[row])/3 @@ -147,11 +147,11 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): det_features = [] leg1 = len(tracks) leg2 = len(detections) - cost_matrix = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_det = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_track = np.zeros((leg1, leg2), dtype=np.float) - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.zeros((leg1, leg2), dtype=float) + cost_matrix_det = np.zeros((leg1, leg2), dtype=float) + cost_matrix_track = np.zeros((leg1, leg2), dtype=float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) if leg2 != 0: cost_matrix_det = np.maximum(0.0, cdist(det_features, det_features, metric)) if leg1 != 0: @@ -165,8 +165,8 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): if leg2 > 10: leg2 = 10 detections = detections[:10] - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) return track_features, det_features, cost_matrix, cost_matrix_det, cost_matrix_track def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): diff --git a/tutorials/qdtrack/byte_tracker.py b/tutorials/qdtrack/byte_tracker.py index 0a292293e..5d0a638be 100644 --- a/tutorials/qdtrack/byte_tracker.py +++ b/tutorials/qdtrack/byte_tracker.py @@ -17,7 +17,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/qdtrack/mot_online/matching.py b/tutorials/qdtrack/mot_online/matching.py index 54cb4be09..b86608a7d 100644 --- a/tutorials/qdtrack/mot_online/matching.py +++ b/tutorials/qdtrack/mot_online/matching.py @@ -69,13 +69,13 @@ def ious(atlbrs, btlbrs): :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -109,13 +109,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix @@ -127,17 +127,17 @@ def embedding_distance2(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[0] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[0] for track in tracks], dtype=float) cost_matrix2 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=float) cost_matrix3 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features for row in range(len(cost_matrix)): cost_matrix[row] = (cost_matrix[row]+cost_matrix2[row]+cost_matrix3[row])/3 @@ -149,11 +149,11 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): det_features = [] leg1 = len(tracks) leg2 = len(detections) - cost_matrix = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_det = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_track = np.zeros((leg1, leg2), dtype=np.float) - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.zeros((leg1, leg2), dtype=float) + cost_matrix_det = np.zeros((leg1, leg2), dtype=float) + cost_matrix_track = np.zeros((leg1, leg2), dtype=float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) if leg2 != 0: cost_matrix_det = np.maximum(0.0, cdist(det_features, det_features, metric)) if leg1 != 0: @@ -167,8 +167,8 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): if leg2 > 10: leg2 = 10 detections = detections[:10] - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) return track_features, det_features, cost_matrix, cost_matrix_det, cost_matrix_track def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): diff --git a/tutorials/qdtrack/tracker_reid_motion.py b/tutorials/qdtrack/tracker_reid_motion.py index 406a0a413..f012ca513 100644 --- a/tutorials/qdtrack/tracker_reid_motion.py +++ b/tutorials/qdtrack/tracker_reid_motion.py @@ -17,7 +17,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, temp_feat, buffer_size=30): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/trades/byte_tracker.py b/tutorials/trades/byte_tracker.py index d154045b6..7a2d952d4 100644 --- a/tutorials/trades/byte_tracker.py +++ b/tutorials/trades/byte_tracker.py @@ -16,7 +16,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/trades/mot_online/matching.py b/tutorials/trades/mot_online/matching.py index cc7abab60..da24f4f11 100644 --- a/tutorials/trades/mot_online/matching.py +++ b/tutorials/trades/mot_online/matching.py @@ -68,13 +68,13 @@ def ious(atlbrs, btlbrs): :type atlbrs: list[tlbr] | np.ndarray :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -107,13 +107,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix @@ -125,17 +125,17 @@ def embedding_distance2(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[0] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[0] for track in tracks], dtype=float) cost_matrix2 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features - track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=np.float) + track_features = np.asarray([track.features[len(track.features)-1] for track in tracks], dtype=float) cost_matrix3 = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features for row in range(len(cost_matrix)): cost_matrix[row] = (cost_matrix[row]+cost_matrix2[row]+cost_matrix3[row])/3 @@ -147,11 +147,11 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): det_features = [] leg1 = len(tracks) leg2 = len(detections) - cost_matrix = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_det = np.zeros((leg1, leg2), dtype=np.float) - cost_matrix_track = np.zeros((leg1, leg2), dtype=np.float) - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + cost_matrix = np.zeros((leg1, leg2), dtype=float) + cost_matrix_det = np.zeros((leg1, leg2), dtype=float) + cost_matrix_track = np.zeros((leg1, leg2), dtype=float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) if leg2 != 0: cost_matrix_det = np.maximum(0.0, cdist(det_features, det_features, metric)) if leg1 != 0: @@ -165,8 +165,8 @@ def vis_id_feature_A_distance(tracks, detections, metric='cosine'): if leg2 > 10: leg2 = 10 detections = detections[:10] - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) return track_features, det_features, cost_matrix, cost_matrix_det, cost_matrix_track def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): diff --git a/tutorials/trades/tracker.py b/tutorials/trades/tracker.py index a607935cc..1ea5b44a5 100644 --- a/tutorials/trades/tracker.py +++ b/tutorials/trades/tracker.py @@ -272,7 +272,7 @@ def bbox_overlaps_py(self, boxes, query_boxes): """ n_ = boxes.shape[0] k_ = query_boxes.shape[0] - overlaps = np.zeros((n_, k_), dtype=np.float) + overlaps = np.zeros((n_, k_), dtype=float) for k in range(k_): query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) for n in range(n_): diff --git a/tutorials/transtrack/main_track.py b/tutorials/transtrack/main_track.py index e38b8d64a..efe1a0ac1 100644 --- a/tutorials/transtrack/main_track.py +++ b/tutorials/transtrack/main_track.py @@ -1,375 +1,375 @@ -# Modified by Peize Sun, Rufeng Zhang -# ------------------------------------------------------------------------ -# Deformable DETR -# Copyright (c) 2020 SenseTime. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ -# Modified from DETR (https://github.com/facebookresearch/detr) -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -# ------------------------------------------------------------------------ -import argparse -import datetime -import json -import random -import time -from pathlib import Path - -import numpy as np -import torch -from torch.utils.data import DataLoader -import datasets -import util.misc as utils -import datasets.samplers as samplers -from datasets import build_dataset, get_coco_api_from_dataset -from engine_track import evaluate, train_one_epoch, evaluate_track -from models import build_tracktrain_model, build_tracktest_model, build_model -from models import Tracker -from models import save_track -from mot_online.byte_tracker import BYTETracker - -from collections import defaultdict - - -def get_args_parser(): - parser = argparse.ArgumentParser('Deformable DETR Detector', add_help=False) - parser.add_argument('--lr', default=2e-4, type=float) - parser.add_argument('--lr_backbone_names', default=["backbone.0"], type=str, nargs='+') - parser.add_argument('--lr_backbone', default=2e-5, type=float) - parser.add_argument('--lr_linear_proj_names', default=['reference_points', 'sampling_offsets'], type=str, nargs='+') - parser.add_argument('--lr_linear_proj_mult', default=0.1, type=float) - parser.add_argument('--batch_size', default=1, type=int) - parser.add_argument('--weight_decay', default=1e-4, type=float) - parser.add_argument('--epochs', default=50, type=int) - parser.add_argument('--lr_drop', default=40, type=int) - parser.add_argument('--lr_drop_epochs', default=None, type=int, nargs='+') - parser.add_argument('--clip_max_norm', default=0.1, type=float, - help='gradient clipping max norm') - - parser.add_argument('--sgd', action='store_true') - - # Variants of Deformable DETR - parser.add_argument('--with_box_refine', default=True, action='store_true') - parser.add_argument('--two_stage', default=False, action='store_true') - - # Model parameters - parser.add_argument('--frozen_weights', type=str, default=None, - help="Path to the pretrained model. If set, only the mask head will be trained") - - # * Backbone - parser.add_argument('--backbone', default='resnet50', type=str, - help="Name of the convolutional backbone to use") - parser.add_argument('--dilation', action='store_true', - help="If true, we replace stride with dilation in the last convolutional block (DC5)") - parser.add_argument('--position_embedding', default='sine', type=str, choices=('sine', 'learned'), - help="Type of positional embedding to use on top of the image features") - parser.add_argument('--position_embedding_scale', default=2 * np.pi, type=float, - help="position / size * scale") - parser.add_argument('--num_feature_levels', default=4, type=int, help='number of feature levels') - - # * Transformer - parser.add_argument('--enc_layers', default=6, type=int, - help="Number of encoding layers in the transformer") - parser.add_argument('--dec_layers', default=6, type=int, - help="Number of decoding layers in the transformer") - parser.add_argument('--dim_feedforward', default=1024, type=int, - help="Intermediate size of the feedforward layers in the transformer blocks") - parser.add_argument('--hidden_dim', default=256, type=int, - help="Size of the embeddings (dimension of the transformer)") - parser.add_argument('--dropout', default=0.1, type=float, - help="Dropout applied in the transformer") - parser.add_argument('--nheads', default=8, type=int, - help="Number of attention heads inside the transformer's attentions") - parser.add_argument('--num_queries', default=500, type=int, - help="Number of query slots") - parser.add_argument('--dec_n_points', default=4, type=int) - parser.add_argument('--enc_n_points', default=4, type=int) - - # * Segmentation - parser.add_argument('--masks', action='store_true', - help="Train segmentation head if the flag is provided") - - # Loss - parser.add_argument('--no_aux_loss', dest='aux_loss', action='store_false', - help="Disables auxiliary decoding losses (loss at each layer)") - - # * Matcher - parser.add_argument('--set_cost_class', default=2, type=float, - help="Class coefficient in the matching cost") - parser.add_argument('--set_cost_bbox', default=5, type=float, - help="L1 box coefficient in the matching cost") - parser.add_argument('--set_cost_giou', default=2, type=float, - help="giou box coefficient in the matching cost") - - # * Loss coefficients - parser.add_argument('--mask_loss_coef', default=1, type=float) - parser.add_argument('--dice_loss_coef', default=1, type=float) - parser.add_argument('--cls_loss_coef', default=2, type=float) - parser.add_argument('--bbox_loss_coef', default=5, type=float) - parser.add_argument('--giou_loss_coef', default=2, type=float) - parser.add_argument('--focal_alpha', default=0.25, type=float) - parser.add_argument('--id_loss_coef', default=1, type=float) - - # dataset parameters - parser.add_argument('--dataset_file', default='coco') - parser.add_argument('--coco_path', default='./data/coco', type=str) - parser.add_argument('--coco_panoptic_path', type=str) - parser.add_argument('--remove_difficult', action='store_true') - - parser.add_argument('--output_dir', default='', - help='path where to save, empty for no saving') - parser.add_argument('--device', default='cuda', - help='device to use for training / testing') - parser.add_argument('--seed', default=42, type=int) - parser.add_argument('--resume', default='', help='resume from checkpoint') - parser.add_argument('--start_epoch', default=0, type=int, metavar='N', - help='start epoch') - parser.add_argument('--eval', action='store_true') - parser.add_argument('--num_workers', default=2, type=int) - parser.add_argument('--cache_mode', default=False, action='store_true', help='whether to cache images on memory') - - # PyTorch checkpointing for saving memory (torch.utils.checkpoint.checkpoint) - parser.add_argument('--checkpoint_enc_ffn', default=False, action='store_true') - parser.add_argument('--checkpoint_dec_ffn', default=False, action='store_true') - - # appended for track. - parser.add_argument('--track_train_split', default='train', type=str) - parser.add_argument('--track_eval_split', default='val', type=str) - parser.add_argument('--track_thresh', default=0.4, type=float) - parser.add_argument('--reid_shared', default=False, type=bool) - parser.add_argument('--reid_dim', default=128, type=int) - parser.add_argument('--num_ids', default=360, type=int) - - - # detector for track. - parser.add_argument('--det_val', default=False, action='store_true') - - - return parser - - -def main(args): - utils.init_distributed_mode(args) - print("git:\n {}\n".format(utils.get_sha())) - - if args.frozen_weights is not None: - assert args.masks, "Frozen training is meant for segmentation only" - print(args) - - device = torch.device(args.device) - - # fix the seed for reproducibility - seed = args.seed + utils.get_rank() - torch.manual_seed(seed) - np.random.seed(seed) - random.seed(seed) - - if args.det_val: - assert args.eval, 'only support eval mode of detector for track' - model, criterion, postprocessors = build_model(args) - elif args.eval: - model, criterion, postprocessors = build_tracktest_model(args) - else: - model, criterion, postprocessors = build_tracktrain_model(args) - - model.to(device) - - model_without_ddp = model - n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) - print('number of params:', n_parameters) - - dataset_train = build_dataset(image_set=args.track_train_split, args=args) - dataset_val = build_dataset(image_set=args.track_eval_split, args=args) - - if args.distributed: - if args.cache_mode: - sampler_train = samplers.NodeDistributedSampler(dataset_train) - sampler_val = samplers.NodeDistributedSampler(dataset_val, shuffle=False) - else: - sampler_train = samplers.DistributedSampler(dataset_train) - sampler_val = samplers.DistributedSampler(dataset_val, shuffle=False) - else: - sampler_train = torch.utils.data.RandomSampler(dataset_train) - sampler_val = torch.utils.data.SequentialSampler(dataset_val) - - batch_sampler_train = torch.utils.data.BatchSampler( - sampler_train, args.batch_size, drop_last=True) - - data_loader_train = DataLoader(dataset_train, batch_sampler=batch_sampler_train, - collate_fn=utils.collate_fn, num_workers=args.num_workers, - pin_memory=True) - data_loader_val = DataLoader(dataset_val, args.batch_size, sampler=sampler_val, - drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers, - pin_memory=True) - - # lr_backbone_names = ["backbone.0", "backbone.neck", "input_proj", "transformer.encoder"] - def match_name_keywords(n, name_keywords): - out = False - for b in name_keywords: - if b in n: - out = True - break - return out - - for n, p in model_without_ddp.named_parameters(): - print(n) - - param_dicts = [ - { - "params": - [p for n, p in model_without_ddp.named_parameters() - if not match_name_keywords(n, args.lr_backbone_names) and not match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], - "lr": args.lr, - }, - { - "params": [p for n, p in model_without_ddp.named_parameters() if match_name_keywords(n, args.lr_backbone_names) and p.requires_grad], - "lr": args.lr_backbone, - }, - { - "params": [p for n, p in model_without_ddp.named_parameters() if match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], - "lr": args.lr * args.lr_linear_proj_mult, - } - ] - if args.sgd: - optimizer = torch.optim.SGD(param_dicts, lr=args.lr, momentum=0.9, - weight_decay=args.weight_decay) - else: - optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, - weight_decay=args.weight_decay) - lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop) - - if args.distributed: - model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True) - model_without_ddp = model.module - - if args.dataset_file == "coco_panoptic": - # We also evaluate AP during panoptic training, on original coco DS - coco_val = datasets.coco.build("val", args) - base_ds = get_coco_api_from_dataset(coco_val) - else: - base_ds = get_coco_api_from_dataset(dataset_val) - - if args.frozen_weights is not None: - checkpoint = torch.load(args.frozen_weights, map_location='cpu') - model_without_ddp.detr.load_state_dict(checkpoint['model']) - - output_dir = Path(args.output_dir) - if args.resume: - if args.resume.startswith('https'): - checkpoint = torch.hub.load_state_dict_from_url( - args.resume, map_location='cpu', check_hash=True) - else: - checkpoint = torch.load(args.resume, map_location='cpu') - missing_keys, unexpected_keys = model_without_ddp.load_state_dict(checkpoint['model'], strict=False) - unexpected_keys = [k for k in unexpected_keys if not (k.endswith('total_params') or k.endswith('total_ops'))] - if len(missing_keys) > 0: - print('Missing Keys: {}'.format(missing_keys)) - if len(unexpected_keys) > 0: - print('Unexpected Keys: {}'.format(unexpected_keys)) - if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: - import copy - p_groups = copy.deepcopy(optimizer.param_groups) - optimizer.load_state_dict(checkpoint['optimizer']) - for pg, pg_old in zip(optimizer.param_groups, p_groups): - pg['lr'] = pg_old['lr'] - pg['initial_lr'] = pg_old['initial_lr'] - print(optimizer.param_groups) - lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) - # todo: this is a hack for doing experiment that resume from checkpoint and also modify lr scheduler (e.g., decrease lr in advance). - args.override_resumed_lr_drop = True - if args.override_resumed_lr_drop: - print('Warning: (hack) args.override_resumed_lr_drop is set to True, so args.lr_drop would override lr_drop in resumed lr_scheduler.') - lr_scheduler.step_size = args.lr_drop - lr_scheduler.base_lrs = list(map(lambda group: group['initial_lr'], optimizer.param_groups)) - lr_scheduler.step(lr_scheduler.last_epoch) - args.start_epoch = checkpoint['epoch'] + 1 - # check the resumed model -# if not args.eval: -# test_stats, coco_evaluator, _ = evaluate( -# model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir -# ) - - if args.eval: - assert args.batch_size == 1, print("Now only support 1.") - # tracker = MOTXTracker(score_thresh=args.track_thresh) - # test_stats, coco_evaluator, res_tracks = evaluate(model, criterion, postprocessors, data_loader_val, - # base_ds, device, args.output_dir, tracker=tracker, - # phase='eval', det_val=args.det_val) - tracker = BYTETracker(args) - test_stats, coco_evaluator, res_tracks = evaluate_track(args, model, criterion, postprocessors, data_loader_val, - base_ds, device, args.output_dir, tracker=tracker, - phase='eval', det_val=args.det_val) - if args.output_dir: - utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") - if res_tracks is not None: - print("Creating video index for {}.".format(args.dataset_file)) - video_to_images = defaultdict(list) - video_names = defaultdict() - for _, info in dataset_val.coco.imgs.items(): - video_to_images[info["video_id"]].append({"image_id": info["id"], - "frame_id": info["frame_id"]}) - video_name = info["file_name"].split("/")[0] - if video_name not in video_names: - video_names[info["video_id"]] = video_name - assert len(video_to_images) == len(video_names) - # save mot results. - save_track(res_tracks, args.output_dir, video_to_images, video_names, args.track_eval_split) - - return - - print("Start training") - start_time = time.time() - for epoch in range(args.start_epoch, args.epochs): - if args.distributed: - sampler_train.set_epoch(epoch) - train_stats = train_one_epoch( - model, criterion, data_loader_train, optimizer, device, epoch, args.clip_max_norm) - lr_scheduler.step() - if args.output_dir: - checkpoint_paths = [output_dir / 'checkpoint.pth'] - # extra checkpoint before LR drop and every 5 epochs - if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 5 == 0: - checkpoint_paths.append(output_dir / f'checkpoint{epoch:04}.pth') - for checkpoint_path in checkpoint_paths: - utils.save_on_master({ - 'model': model_without_ddp.state_dict(), - 'optimizer': optimizer.state_dict(), - 'lr_scheduler': lr_scheduler.state_dict(), - 'epoch': epoch, - 'args': args, - }, checkpoint_path) - if epoch % 10 == 0 or epoch > args.epochs - 5: - test_stats, coco_evaluator, _ = evaluate( - model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir, - ) - - log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, - **{f'test_{k}': v for k, v in test_stats.items()}, - 'epoch': epoch, - 'n_parameters': n_parameters} - - if args.output_dir and utils.is_main_process(): - with (output_dir / "log.txt").open("a") as f: - f.write(json.dumps(log_stats) + "\n") - - # for evaluation logs - if coco_evaluator is not None: - (output_dir / 'eval').mkdir(exist_ok=True) - if "bbox" in coco_evaluator.coco_eval: - filenames = ['latest.pth'] - if epoch % 50 == 0: - filenames.append(f'{epoch:03}.pth') - for name in filenames: - torch.save(coco_evaluator.coco_eval["bbox"].eval, - output_dir / "eval" / name) - - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print('Training time {}'.format(total_time_str)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser('Deformable DETR training and evaluation script', parents=[get_args_parser()]) - args = parser.parse_args() - if args.output_dir: - Path(args.output_dir).mkdir(parents=True, exist_ok=True) - main(args) +# Modified by Peize Sun, Rufeng Zhang +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ +import argparse +import datetime +import json +import random +import time +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader +import datasets +import util.misc as utils +import datasets.samplers as samplers +from datasets import build_dataset, get_coco_api_from_dataset +from engine_track import evaluate, train_one_epoch, evaluate_track +from models import build_tracktrain_model, build_tracktest_model, build_model +from models import Tracker +from models import save_track +from mot_online.byte_tracker import BYTETracker + +from collections import defaultdict + + +def get_args_parser(): + parser = argparse.ArgumentParser('Deformable DETR Detector', add_help=False) + parser.add_argument('--lr', default=2e-4, type=float) + parser.add_argument('--lr_backbone_names', default=["backbone.0"], type=str, nargs='+') + parser.add_argument('--lr_backbone', default=2e-5, type=float) + parser.add_argument('--lr_linear_proj_names', default=['reference_points', 'sampling_offsets'], type=str, nargs='+') + parser.add_argument('--lr_linear_proj_mult', default=0.1, type=float) + parser.add_argument('--batch_size', default=1, type=int) + parser.add_argument('--weight_decay', default=1e-4, type=float) + parser.add_argument('--epochs', default=50, type=int) + parser.add_argument('--lr_drop', default=40, type=int) + parser.add_argument('--lr_drop_epochs', default=None, type=int, nargs='+') + parser.add_argument('--clip_max_norm', default=0.1, type=float, + help='gradient clipping max norm') + + parser.add_argument('--sgd', action='store_true') + + # Variants of Deformable DETR + parser.add_argument('--with_box_refine', default=True, action='store_true') + parser.add_argument('--two_stage', default=False, action='store_true') + + # Model parameters + parser.add_argument('--frozen_weights', type=str, default=None, + help="Path to the pretrained model. If set, only the mask head will be trained") + + # * Backbone + parser.add_argument('--backbone', default='resnet50', type=str, + help="Name of the convolutional backbone to use") + parser.add_argument('--dilation', action='store_true', + help="If true, we replace stride with dilation in the last convolutional block (DC5)") + parser.add_argument('--position_embedding', default='sine', type=str, choices=('sine', 'learned'), + help="Type of positional embedding to use on top of the image features") + parser.add_argument('--position_embedding_scale', default=2 * np.pi, type=float, + help="position / size * scale") + parser.add_argument('--num_feature_levels', default=4, type=int, help='number of feature levels') + + # * Transformer + parser.add_argument('--enc_layers', default=6, type=int, + help="Number of encoding layers in the transformer") + parser.add_argument('--dec_layers', default=6, type=int, + help="Number of decoding layers in the transformer") + parser.add_argument('--dim_feedforward', default=1024, type=int, + help="Intermediate size of the feedforward layers in the transformer blocks") + parser.add_argument('--hidden_dim', default=256, type=int, + help="Size of the embeddings (dimension of the transformer)") + parser.add_argument('--dropout', default=0.1, type=float, + help="Dropout applied in the transformer") + parser.add_argument('--nheads', default=8, type=int, + help="Number of attention heads inside the transformer's attentions") + parser.add_argument('--num_queries', default=500, type=int, + help="Number of query slots") + parser.add_argument('--dec_n_points', default=4, type=int) + parser.add_argument('--enc_n_points', default=4, type=int) + + # * Segmentation + parser.add_argument('--masks', action='store_true', + help="Train segmentation head if the flag is provided") + + # Loss + parser.add_argument('--no_aux_loss', dest='aux_loss', action='store_false', + help="Disables auxiliary decoding losses (loss at each layer)") + + # * Matcher + parser.add_argument('--set_cost_class', default=2, type=float, + help="Class coefficient in the matching cost") + parser.add_argument('--set_cost_bbox', default=5, type=float, + help="L1 box coefficient in the matching cost") + parser.add_argument('--set_cost_giou', default=2, type=float, + help="giou box coefficient in the matching cost") + + # * Loss coefficients + parser.add_argument('--mask_loss_coef', default=1, type=float) + parser.add_argument('--dice_loss_coef', default=1, type=float) + parser.add_argument('--cls_loss_coef', default=2, type=float) + parser.add_argument('--bbox_loss_coef', default=5, type=float) + parser.add_argument('--giou_loss_coef', default=2, type=float) + parser.add_argument('--focal_alpha', default=0.25, type=float) + parser.add_argument('--id_loss_coef', default=1, type=float) + + # dataset parameters + parser.add_argument('--dataset_file', default='coco') + parser.add_argument('--coco_path', default='./data/coco', type=str) + parser.add_argument('--coco_panoptic_path', type=str) + parser.add_argument('--remove_difficult', action='store_true') + + parser.add_argument('--output_dir', default='', + help='path where to save, empty for no saving') + parser.add_argument('--device', default='cuda', + help='device to use for training / testing') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--resume', default='', help='resume from checkpoint') + parser.add_argument('--start_epoch', default=0, type=int, metavar='N', + help='start epoch') + parser.add_argument('--eval', action='store_true') + parser.add_argument('--num_workers', default=2, type=int) + parser.add_argument('--cache_mode', default=False, action='store_true', help='whether to cache images on memory') + + # PyTorch checkpointing for saving memory (torch.utils.checkpoint.checkpoint) + parser.add_argument('--checkpoint_enc_ffn', default=False, action='store_true') + parser.add_argument('--checkpoint_dec_ffn', default=False, action='store_true') + + # appended for track. + parser.add_argument('--track_train_split', default='train', type=str) + parser.add_argument('--track_eval_split', default='val', type=str) + parser.add_argument('--track_thresh', default=0.4, type=float) + parser.add_argument('--reid_shared', default=False, type=bool) + parser.add_argument('--reid_dim', default=128, type=int) + parser.add_argument('--num_ids', default=360, type=int) + + + # detector for track. + parser.add_argument('--det_val', default=False, action='store_true') + + + return parser + + +def main(args): + utils.init_distributed_mode(args) + print("git:\n {}\n".format(utils.get_sha())) + + if args.frozen_weights is not None: + assert args.masks, "Frozen training is meant for segmentation only" + print(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + if args.det_val: + assert args.eval, 'only support eval mode of detector for track' + model, criterion, postprocessors = build_model(args) + elif args.eval: + model, criterion, postprocessors = build_tracktest_model(args) + else: + model, criterion, postprocessors = build_tracktrain_model(args) + + model.to(device) + + model_without_ddp = model + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print('number of params:', n_parameters) + + dataset_train = build_dataset(image_set=args.track_train_split, args=args) + dataset_val = build_dataset(image_set=args.track_eval_split, args=args) + + if args.distributed: + if args.cache_mode: + sampler_train = samplers.NodeDistributedSampler(dataset_train) + sampler_val = samplers.NodeDistributedSampler(dataset_val, shuffle=False) + else: + sampler_train = samplers.DistributedSampler(dataset_train) + sampler_val = samplers.DistributedSampler(dataset_val, shuffle=False) + else: + sampler_train = torch.utils.data.RandomSampler(dataset_train) + sampler_val = torch.utils.data.SequentialSampler(dataset_val) + + batch_sampler_train = torch.utils.data.BatchSampler( + sampler_train, args.batch_size, drop_last=True) + + data_loader_train = DataLoader(dataset_train, batch_sampler=batch_sampler_train, + collate_fn=utils.collate_fn, num_workers=args.num_workers, + pin_memory=True) + data_loader_val = DataLoader(dataset_val, args.batch_size, sampler=sampler_val, + drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers, + pin_memory=True) + + # lr_backbone_names = ["backbone.0", "backbone.neck", "input_proj", "transformer.encoder"] + def match_name_keywords(n, name_keywords): + out = False + for b in name_keywords: + if b in n: + out = True + break + return out + + for n, p in model_without_ddp.named_parameters(): + print(n) + + param_dicts = [ + { + "params": + [p for n, p in model_without_ddp.named_parameters() + if not match_name_keywords(n, args.lr_backbone_names) and not match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], + "lr": args.lr, + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() if match_name_keywords(n, args.lr_backbone_names) and p.requires_grad], + "lr": args.lr_backbone, + }, + { + "params": [p for n, p in model_without_ddp.named_parameters() if match_name_keywords(n, args.lr_linear_proj_names) and p.requires_grad], + "lr": args.lr * args.lr_linear_proj_mult, + } + ] + if args.sgd: + optimizer = torch.optim.SGD(param_dicts, lr=args.lr, momentum=0.9, + weight_decay=args.weight_decay) + else: + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, + weight_decay=args.weight_decay) + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True) + model_without_ddp = model.module + + if args.dataset_file == "coco_panoptic": + # We also evaluate AP during panoptic training, on original coco DS + coco_val = datasets.coco.build("val", args) + base_ds = get_coco_api_from_dataset(coco_val) + else: + base_ds = get_coco_api_from_dataset(dataset_val) + + if args.frozen_weights is not None: + checkpoint = torch.load(args.frozen_weights, map_location='cpu') + model_without_ddp.detr.load_state_dict(checkpoint['model']) + + output_dir = Path(args.output_dir) + if args.resume: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(args.resume, map_location='cpu') + missing_keys, unexpected_keys = model_without_ddp.load_state_dict(checkpoint['model'], strict=False) + unexpected_keys = [k for k in unexpected_keys if not (k.endswith('total_params') or k.endswith('total_ops'))] + if len(missing_keys) > 0: + print('Missing Keys: {}'.format(missing_keys)) + if len(unexpected_keys) > 0: + print('Unexpected Keys: {}'.format(unexpected_keys)) + if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + import copy + p_groups = copy.deepcopy(optimizer.param_groups) + optimizer.load_state_dict(checkpoint['optimizer']) + for pg, pg_old in zip(optimizer.param_groups, p_groups): + pg['lr'] = pg_old['lr'] + pg['initial_lr'] = pg_old['initial_lr'] + print(optimizer.param_groups) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + # todo: this is a hack for doing experiment that resume from checkpoint and also modify lr scheduler (e.g., decrease lr in advance). + args.override_resumed_lr_drop = True + if args.override_resumed_lr_drop: + print('Warning: (hack) args.override_resumed_lr_drop is set to True, so args.lr_drop would override lr_drop in resumed lr_scheduler.') + lr_scheduler.step_size = args.lr_drop + lr_scheduler.base_lrs = list(map(lambda group: group['initial_lr'], optimizer.param_groups)) + lr_scheduler.step(lr_scheduler.last_epoch) + args.start_epoch = checkpoint['epoch'] + 1 + # check the resumed model +# if not args.eval: +# test_stats, coco_evaluator, _ = evaluate( +# model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir +# ) + + if args.eval: + assert args.batch_size == 1, print("Now only support 1.") + # tracker = MOTXTracker(score_thresh=args.track_thresh) + # test_stats, coco_evaluator, res_tracks = evaluate(model, criterion, postprocessors, data_loader_val, + # base_ds, device, args.output_dir, tracker=tracker, + # phase='eval', det_val=args.det_val) + tracker = BYTETracker(args) + test_stats, coco_evaluator, res_tracks = evaluate_track(args, model, criterion, postprocessors, data_loader_val, + base_ds, device, args.output_dir, tracker=tracker, + phase='eval', det_val=args.det_val) + if args.output_dir: + utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") + if res_tracks is not None: + print("Creating video index for {}.".format(args.dataset_file)) + video_to_images = defaultdict(list) + video_names = defaultdict() + for _, info in dataset_val.coco.imgs.items(): + video_to_images[info["video_id"]].append({"image_id": info["id"], + "frame_id": info["frame_id"]}) + video_name = info["file_name"].split("/")[0] + if video_name not in video_names: + video_names[info["video_id"]] = video_name + assert len(video_to_images) == len(video_names) + # save mot results. + save_track(res_tracks, args.output_dir, video_to_images, video_names, args.track_eval_split) + + return + + print("Start training") + start_time = time.time() + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + sampler_train.set_epoch(epoch) + train_stats = train_one_epoch( + model, criterion, data_loader_train, optimizer, device, epoch, args.clip_max_norm) + lr_scheduler.step() + if args.output_dir: + checkpoint_paths = [output_dir / 'checkpoint.pth'] + # extra checkpoint before LR drop and every 5 epochs + if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 5 == 0: + checkpoint_paths.append(output_dir / f'checkpoint{epoch:04}.pth') + for checkpoint_path in checkpoint_paths: + utils.save_on_master({ + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'epoch': epoch, + 'args': args, + }, checkpoint_path) + if epoch % 10 == 0 or epoch > args.epochs - 5: + test_stats, coco_evaluator, _ = evaluate( + model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir, + ) + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'test_{k}': v for k, v in test_stats.items()}, + 'epoch': epoch, + 'n_parameters': n_parameters} + + if args.output_dir and utils.is_main_process(): + with (output_dir / "log.txt").open("a") as f: + f.write(json.dumps(log_stats) + "\n") + + # for evaluation logs + if coco_evaluator is not None: + (output_dir / 'eval').mkdir(exist_ok=True) + if "bbox" in coco_evaluator.coco_eval: + filenames = ['latest.pth'] + if epoch % 50 == 0: + filenames.append(f'{epoch:03}.pth') + for name in filenames: + torch.save(coco_evaluator.coco_eval["bbox"].eval, + output_dir / "eval" / name) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser('Deformable DETR training and evaluation script', parents=[get_args_parser()]) + args = parser.parse_args() + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + main(args) diff --git a/tutorials/transtrack/mot_online/byte_tracker.py b/tutorials/transtrack/mot_online/byte_tracker.py index be6e179a5..d6a68a8dc 100644 --- a/tutorials/transtrack/mot_online/byte_tracker.py +++ b/tutorials/transtrack/mot_online/byte_tracker.py @@ -15,7 +15,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, buffer_size=30): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/tutorials/transtrack/mot_online/matching.py b/tutorials/transtrack/mot_online/matching.py index d21c95823..57e79ff38 100644 --- a/tutorials/transtrack/mot_online/matching.py +++ b/tutorials/transtrack/mot_online/matching.py @@ -58,13 +58,13 @@ def ious(atlbrs, btlbrs): :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -98,11 +98,11 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix diff --git a/yolox/deepsort_tracker/detection.py b/yolox/deepsort_tracker/detection.py index 9bd84977d..2ec5d3a5f 100644 --- a/yolox/deepsort_tracker/detection.py +++ b/yolox/deepsort_tracker/detection.py @@ -24,7 +24,7 @@ class Detection(object): """ def __init__(self, tlwh, confidence, feature): - self.tlwh = np.asarray(tlwh, dtype=np.float) + self.tlwh = np.asarray(tlwh, dtype=float) self.confidence = float(confidence) self.feature = np.asarray(feature, dtype=np.float32) diff --git a/yolox/motdt_tracker/matching.py b/yolox/motdt_tracker/matching.py index 01d07da87..f64ba9cdb 100644 --- a/yolox/motdt_tracker/matching.py +++ b/yolox/motdt_tracker/matching.py @@ -39,13 +39,13 @@ def ious(atlbrs, btlbrs): :type atlbrs: list[tlbr] | np.ndarray :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -73,7 +73,7 @@ def nearest_reid_distance(tracks, detections, metric='cosine'): :type detections: list[BaseTrack] :rtype cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix @@ -92,7 +92,7 @@ def mean_reid_distance(tracks, detections, metric='cosine'): :type metric: str :rtype cost_matrix np.ndarray """ - cost_matrix = np.empty((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.empty((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix diff --git a/yolox/motdt_tracker/motdt_tracker.py b/yolox/motdt_tracker/motdt_tracker.py index b3521f2ff..21abb3900 100644 --- a/yolox/motdt_tracker/motdt_tracker.py +++ b/yolox/motdt_tracker/motdt_tracker.py @@ -21,7 +21,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, max_n_features=100, from_det=True): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/yolox/motdt_tracker/reid_model.py b/yolox/motdt_tracker/reid_model.py index 6ad49e340..7b1b54aa6 100644 --- a/yolox/motdt_tracker/reid_model.py +++ b/yolox/motdt_tracker/reid_model.py @@ -60,7 +60,7 @@ def load_net(fname, net, prefix='', load_state_dict=False): lr = h5f.attrs['learning_rates'] else: lr = h5f.attrs.get('lr', -1) - lr = np.asarray([lr] if lr > 0 else [], dtype=np.float) + lr = np.asarray([lr] if lr > 0 else [], dtype=float) return epoch, lr diff --git a/yolox/tracker/byte_tracker.py b/yolox/tracker/byte_tracker.py index a4f50e1ec..db98ff21f 100644 --- a/yolox/tracker/byte_tracker.py +++ b/yolox/tracker/byte_tracker.py @@ -13,7 +13,7 @@ class STrack(BaseTrack): def __init__(self, tlwh, score, person): # wait activate - self._tlwh = np.asarray(tlwh, dtype=np.float) + self._tlwh = np.asarray(tlwh, dtype=float) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False diff --git a/yolox/tracker/matching.py b/yolox/tracker/matching.py index d36a6cf5b..d8c4372c3 100644 --- a/yolox/tracker/matching.py +++ b/yolox/tracker/matching.py @@ -58,13 +58,13 @@ def ious(atlbrs, btlbrs): :rtype ious np.ndarray """ - ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float) + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=float) if ious.size == 0: return ious ious = bbox_ious( - np.ascontiguousarray(atlbrs, dtype=np.float), - np.ascontiguousarray(btlbrs, dtype=np.float) + np.ascontiguousarray(atlbrs, dtype=float), + np.ascontiguousarray(btlbrs, dtype=float) ) return ious @@ -118,13 +118,13 @@ def embedding_distance(tracks, detections, metric='cosine'): :return: cost_matrix np.ndarray """ - cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float) + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=float) if cost_matrix.size == 0: return cost_matrix - det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float) + det_features = np.asarray([track.curr_feat for track in detections], dtype=float) #for i, track in enumerate(tracks): #cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) - track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=float) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Nomalized features return cost_matrix From caa353a7949b1943d037d81c7aead4352fa4d715 Mon Sep 17 00:00:00 2001 From: Bryce Chudomelka Date: Thu, 11 Apr 2024 11:27:44 -0400 Subject: [PATCH 12/12] Added script for converting np.float to float --- convert_floats.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 convert_floats.py diff --git a/convert_floats.py b/convert_floats.py new file mode 100644 index 000000000..07d66488b --- /dev/null +++ b/convert_floats.py @@ -0,0 +1,24 @@ +import os +import re + +def replace_np_float(directory): + # Compile a regex pattern for 'float' surrounded by word boundaries + pattern = re.compile(r'\bnp.float\b') + + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith('.py'): + file_path = os.path.join(root, file) + with open(file_path, 'r') as f: + content = f.read() + + # Use the regex pattern to replace 'float' with 'float' + modified_content = pattern.sub('float', content) + + # Write the modified content back to the file + with open(file_path, 'w') as f: + f.write(modified_content) + +# Specify the directory +directory = './' +replace_np_float(directory)