From 9b62a9e6ac4a983b8abb36c325d56e73c3c4c715 Mon Sep 17 00:00:00 2001 From: gkane Date: Tue, 9 Jun 2020 13:23:12 -0400 Subject: [PATCH 01/28] dlclive: add tf_config as parameter --- dlclive/dlclive.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index d8b3654..1bdb043 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -21,9 +21,9 @@ except Exception: pass -from dlclive.graph import read_graph, finalize_graph, get_output_nodes, get_output_tensors, extract_graph +from dlclive.graph import read_graph, finalize_graph, get_output_nodes, get_output_tensors, extract_graph, load_graph from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict -from dlclive.display import Display +from dlclive.display import Display from dlclive import utils from dlclive.exceptions import DLCLiveError, DLCLiveWarning @@ -75,13 +75,14 @@ class DLCLive(object): ''' def __init__(self, model_path, - model_type='base', precision='FP16', + model_type='base', precision='FP16', tf_config=None, cropping=None, dynamic=(False,.5,10), resize=None, processor=None, display=False, display_lik=0.5): self.path = model_path self.cfg = None self.model_type = model_type + self.tf_config = tf_config self.precision = precision self.cropping = cropping self.dynamic = dynamic @@ -215,9 +216,11 @@ def init_inference(self, frame=None): if self.model_type == 'base': - graph_def = read_graph(model_file) - self.graph, self.inputs = finalize_graph(graph_def) - self.sess, self.outputs = extract_graph(self.graph) + # graph_def = read_graph(model_file) + # self.graph, self.inputs = finalize_graph(graph_def) + # self.sess, self.outputs = extract_graph(self.graph, tf_config=self.tf_config) + + self.sess, self.inputs, self.outputs = load_graph(model_file) elif self.model_type == 'tflite': @@ -268,7 +271,7 @@ def init_inference(self, frame=None): is_dynamic_op=True) self.graph, self.inputs = finalize_graph(graph_def) - self.sess, self.outputs = extract_graph(self.graph) + self.sess, self.outputs = extract_graph(self.graph, tf_config=self.tf_config) else: From bd2d3cd3ad88109e0740984477b58c39e5cf79f5 Mon Sep 17 00:00:00 2001 From: gkane Date: Tue, 9 Jun 2020 13:26:38 -0400 Subject: [PATCH 02/28] graph: add tfconfig, new full load graph function --- dlclive/graph.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/dlclive/graph.py b/dlclive/graph.py index af43f8e..69adce7 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -25,10 +25,10 @@ def read_graph(file): ''' graph = tf.Graph() - graph_def = tf.GraphDef() - graph_def.ParseFromString(tf.gfile.GFile(file, 'rb').read()) - - return graph_def + with tf.gfile.GFile(file, 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + return graph_def def finalize_graph(graph_def): @@ -101,7 +101,7 @@ def get_output_tensors(graph): return output_tensor -def extract_graph(graph): +def extract_graph(graph, tf_config=None): ''' Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs @@ -109,6 +109,7 @@ def extract_graph(graph): ----------- graph :class:`tensorflow.Graph` a tensorflow graph containing the desired model + tf_config :class:`tensorflow.ConfigProto` Returns -------- @@ -119,7 +120,29 @@ def extract_graph(graph): ''' output_tensor = get_output_tensors(graph) - sess = tf.Session(graph=graph) + sess = tf.Session(graph=graph, config=tf_config) outputs = [graph.get_tensor_by_name(out) for out in output_tensor] return sess, outputs + + +def load_graph(pb_file): + + graph = tf.Graph() + with tf.gfile.GFile(pb_file, 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + + with graph.as_default(): + tf.import_graph_def(graph_def, name='DLC') + graph.finalize() + + op_names = [op.name for op in graph.get_operations()] + inputs = graph.get_tensor_by_name(op_names[0] + ':0') + outputs = [graph.get_tensor_by_name(op_names[-1] + ':0')] + if 'Sigmoid' in op_names[-1]: + outputs.append(graph.get_tensor_by_name(op_names[-2] + ':0')) + + sess = tf.Session(graph=graph) + + return sess, inputs, outputs \ No newline at end of file From eab43708069082115b9496b22746824b79a1692a Mon Sep 17 00:00:00 2001 From: gkane Date: Tue, 9 Jun 2020 13:27:15 -0400 Subject: [PATCH 03/28] bench + display: add display to benchmarking, fix bugs in display --- dlclive/bench.py | 30 ++++++++++++++++++++++-------- dlclive/display.py | 15 ++++++++------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dlclive/bench.py b/dlclive/bench.py index 5ad23cf..9d521cc 100644 --- a/dlclive/bench.py +++ b/dlclive/bench.py @@ -24,7 +24,7 @@ def get_system_info(): """ Return summary info for system running benchmark - + Returns ------- str @@ -64,7 +64,7 @@ def get_system_info(): return host_name, op_sys, host_python, (dev_type, dev) -def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=10000, print_rate=False): +def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=None, n_frames=10000, print_rate=False, display=False, display_lik=0.0, dotsize=3): """ Benchmark on inference times for a given DLC model and video Parameters @@ -94,6 +94,7 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100 cap = cv2.VideoCapture(video_path) ret, frame = cap.read() + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) im_size = (cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) ### get resize factor @@ -103,7 +104,7 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100 ### initialize live object - live = DLCLive(model_path, resize=resize) + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, dotsize=dotsize) live.init_inference(frame) TFGPUinference = True if len(live.outputs) == 1 else False @@ -119,6 +120,8 @@ def run_benchmark(model_path, video_path, resize=None, pixels=None, n_frames=100 if not ret: warnings.warn("Did not complete {:d} frames. There probably were not enough frames in the video {}.".format(n_frames, video_path)) break + + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) start_pose = time.time() live.get_pose(frame) @@ -199,7 +202,7 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_ return True -def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, resize=None, pixels=None, print_rate=False): +def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, tf_config=None, resize=None, pixels=None, print_rate=False, display=False, display_lik=0.5, dotsize=3): """Benchmark DLC model by image size Parameters @@ -241,10 +244,14 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, inf_times[i], pixels_out[i], TFGPUinference = run_benchmark(model_path, video_path, + tf_config=tf_config, resize=resize[i], pixels=pixels[i], n_frames=n_frames, - print_rate=print_rate) + print_rate=print_rate, + display=display, + display_lik=display_lik, + dotsize=dotsize) ### save results @@ -258,20 +265,27 @@ def main(): parser.add_argument('model_path', type=str) parser.add_argument('video_path', type=str) parser.add_argument('-o', '--output', type=str, default=os.getcwd()) - parser.add_argument('-n', '--n_frames', type=int, default=10000) + parser.add_argument('-n', '--n-frames', type=int, default=10000) parser.add_argument('-r', '--resize', type=float, nargs='+') parser.add_argument('-p', '--pixels', type=float, nargs='+') parser.add_argument('-v', '--print_rate', default=False, action='store_true') + parser.add_argument('-d', '--display', default=False, action='store_true') + parser.add_argument('-l', '--display-lik', default=0.5, type=float) + parser.add_argument('-s', '--dotsize', default=3, type=int) args = parser.parse_args() + benchmark_model_by_size(args.model_path, args.video_path, output=args.output, resize=args.resize, pixels=args.pixels, n_frames=args.n_frames, - print_rate=args.print_rate) + print_rate=args.print_rate, + display=args.display, + display_lik=args.display_lik, + dotsize=args.dotsize) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/dlclive/display.py b/dlclive/display.py index 5c5fbc8..8996134 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -81,16 +81,17 @@ def display_frame(self, frame, pose=None): for i in range(pose.shape[0]): if pose[i,2] > self.lik: try: - x0 = pose[i,0] - self.radius if pose[i,1]-self.radius > 0 else 0 - x1 = pose[i,0] + self.radius if pose[i,1] + self.radius < im_size[1] else im_size[1] - y0 = pose[i,1] - self.radius if pose[i,0] - self.radius > 0 else 0 - y1 = pose[i,1] + self.radius if pose[i,0] + self.radius < im_size[0] else im_size[0] + x0 = pose[i,0] - self.radius if pose[i,0] - self.radius > 0 else 0 + x1 = pose[i,0] + self.radius if pose[i,0] + self.radius < im_size[1] else im_size[1] + y0 = pose[i,1] - self.radius if pose[i,1] - self.radius > 0 else 0 + y1 = pose[i,1] + self.radius if pose[i,1] + self.radius < im_size[0] else im_size[0] coords = [x0, y0, x1, y1] + print(coords) draw.ellipse(coords, fill=self.colors[i], outline=self.colors[i]) - except: - pass + except Exception as e: + print(e) - img_tk = ImageTk.PhotoImage(image=img) + img_tk = ImageTk.PhotoImage(image=img, master=self.window) self.lab.configure(image=img_tk) self.window.update() From c8ca18da2177495f90dadc14e1ab4f7ad9d2a888 Mon Sep 17 00:00:00 2001 From: gkane Date: Tue, 9 Jun 2020 13:37:40 -0400 Subject: [PATCH 04/28] fix bugs in display --- dlclive/bench.py | 14 +++++++------- dlclive/display.py | 1 - dlclive/dlclive.py | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dlclive/bench.py b/dlclive/bench.py index 9d521cc..e794aff 100644 --- a/dlclive/bench.py +++ b/dlclive/bench.py @@ -64,7 +64,7 @@ def get_system_info(): return host_name, op_sys, host_python, (dev_type, dev) -def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=None, n_frames=10000, print_rate=False, display=False, display_lik=0.0, dotsize=3): +def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=None, n_frames=10000, print_rate=False, display=False, display_lik=0.0, display_radius=3): """ Benchmark on inference times for a given DLC model and video Parameters @@ -104,13 +104,13 @@ def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=No ### initialize live object - live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, dotsize=dotsize) + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius) live.init_inference(frame) TFGPUinference = True if len(live.outputs) == 1 else False ### perform inference - iterator = range(n_frames) if print_rate else tqdm(range(n_frames)) + iterator = range(n_frames) if (print_rate) or (display) else tqdm(range(n_frames)) inf_times = np.zeros(n_frames) for i in iterator: @@ -202,7 +202,7 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_ return True -def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, tf_config=None, resize=None, pixels=None, print_rate=False, display=False, display_lik=0.5, dotsize=3): +def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, tf_config=None, resize=None, pixels=None, print_rate=False, display=False, display_lik=0.5, display_radius=3): """Benchmark DLC model by image size Parameters @@ -251,7 +251,7 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, print_rate=print_rate, display=display, display_lik=display_lik, - dotsize=dotsize) + display_radius=display_radius) ### save results @@ -271,7 +271,7 @@ def main(): parser.add_argument('-v', '--print_rate', default=False, action='store_true') parser.add_argument('-d', '--display', default=False, action='store_true') parser.add_argument('-l', '--display-lik', default=0.5, type=float) - parser.add_argument('-s', '--dotsize', default=3, type=int) + parser.add_argument('-s', '--display-radius', default=3, type=int) args = parser.parse_args() @@ -284,7 +284,7 @@ def main(): print_rate=args.print_rate, display=args.display, display_lik=args.display_lik, - dotsize=args.dotsize) + display_radius=args.display_radius) if __name__ == "__main__": diff --git a/dlclive/display.py b/dlclive/display.py index 8996134..2a551bd 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -86,7 +86,6 @@ def display_frame(self, frame, pose=None): y0 = pose[i,1] - self.radius if pose[i,1] - self.radius > 0 else 0 y1 = pose[i,1] + self.radius if pose[i,1] + self.radius < im_size[0] else im_size[0] coords = [x0, y0, x1, y1] - print(coords) draw.ellipse(coords, fill=self.colors[i], outline=self.colors[i]) except Exception as e: print(e) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 1bdb043..81bba7d 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -77,7 +77,7 @@ class DLCLive(object): def __init__(self, model_path, model_type='base', precision='FP16', tf_config=None, cropping=None, dynamic=(False,.5,10), resize=None, - processor=None, display=False, display_lik=0.5): + processor=None, display=False, display_lik=0.5, display_radius=3): self.path = model_path self.cfg = None @@ -89,7 +89,7 @@ def __init__(self, model_path, self.dynamic_cropping = None self.resize = resize self.processor = processor - self.display = Display(lik=display_lik) if display else None + self.display = Display(lik=display_lik, radius=display_radius) if display else None self.sess = None self.inputs = None From d33b5f288957667bb244ff1e4c5eb89eb9710368 Mon Sep 17 00:00:00 2001 From: gkane Date: Sun, 14 Jun 2020 22:41:03 -0400 Subject: [PATCH 05/28] add kwargs to processors --- dlclive/dlclive.py | 4 ++-- dlclive/processor/processor.py | 2 +- dlclive/processor/teensy_laser/teensy_laser.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 81bba7d..396130a 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -286,7 +286,7 @@ def init_inference(self, frame=None): return pose - def get_pose(self, frame=None): + def get_pose(self, frame=None, **kwargs): ''' Get the pose of an image @@ -362,7 +362,7 @@ def get_pose(self, frame=None): # process the pose if self.processor: - self.pose = self.processor.process(self.pose) + self.pose = self.processor.process(self.pose, **kwargs) return self.pose diff --git a/dlclive/processor/processor.py b/dlclive/processor/processor.py index 6d26e91..017f2c1 100644 --- a/dlclive/processor/processor.py +++ b/dlclive/processor/processor.py @@ -19,7 +19,7 @@ class Processor(object): def __init__(self): pass - def process(self, pose): + def process(self, pose, **kwargs): return pose def save(self, file=''): diff --git a/dlclive/processor/teensy_laser/teensy_laser.py b/dlclive/processor/teensy_laser/teensy_laser.py index 30f0479..b3ef888 100644 --- a/dlclive/processor/teensy_laser/teensy_laser.py +++ b/dlclive/processor/teensy_laser/teensy_laser.py @@ -8,7 +8,7 @@ Licensed under GNU Lesser General Public License v3.0 """ -from ..processor import Processor +from dlclive.processor.processor import Processor import serial import struct import time @@ -50,7 +50,7 @@ def stim_off(self): self.stim_off_time.append(time.time()) - def process(self, pose): + def process(self, pose, **kwargs): # define criteria to stimulate (e.g. if first point is in a corner of the video) box = [[0,100],[0,100]] From a818b6d731ae4b3891051ed3a6cb5b84acab5f62 Mon Sep 17 00:00:00 2001 From: gkane Date: Mon, 15 Jun 2020 17:25:48 -0400 Subject: [PATCH 06/28] clean up load graph, color conversions --- dlclive/bench.py | 4 +--- dlclive/dlclive.py | 53 ++++++++++++++++++++++++++++-------------- dlclive/graph.py | 51 ++++++++++++++++++++++++---------------- dlclive/utils.py | 58 ++++++++++++++++++++++++++++++++++++---------- 4 files changed, 113 insertions(+), 53 deletions(-) diff --git a/dlclive/bench.py b/dlclive/bench.py index e794aff..8b6e196 100644 --- a/dlclive/bench.py +++ b/dlclive/bench.py @@ -10,6 +10,7 @@ import os import time import sys +import warnings import argparse import pickle @@ -94,7 +95,6 @@ def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=No cap = cv2.VideoCapture(video_path) ret, frame = cap.read() - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) im_size = (cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) ### get resize factor @@ -121,8 +121,6 @@ def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=No warnings.warn("Did not complete {:d} frames. There probably were not enough frames in the video {}.".format(n_frames, video_path)) break - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - start_pose = time.time() live.get_pose(frame) inf_times[i] = time.time() - start_pose diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 396130a..2830d35 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -21,7 +21,7 @@ except Exception: pass -from dlclive.graph import read_graph, finalize_graph, get_output_nodes, get_output_tensors, extract_graph, load_graph +from dlclive.graph import read_graph, finalize_graph, get_output_nodes, get_output_tensors, extract_graph from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict from dlclive.display import Display from dlclive import utils @@ -66,18 +66,33 @@ class DLCLive(object): i) to run a forward predicting model that will predict the future pose from past history of poses (history can be stored in the processor object, but is not stored in this DLCLive object) ii) to trigger external hardware based on pose estimation (e.g. see 'TeensyLaser' processor) + convert2rgb : bool, optional + boolean flag to convert frames from BGR to RGB color scheme + display : bool, optional Display frames with DeepLabCut labels? This is useful for testing model accuracy and cropping parameters, but it is very slow. display_lik : float, optional Likelihood threshold for display + + display_raidus : int, optional + radius for keypoint display in pixels, default=3 ''' - def __init__(self, model_path, - model_type='base', precision='FP16', tf_config=None, - cropping=None, dynamic=(False,.5,10), resize=None, - processor=None, display=False, display_lik=0.5, display_radius=3): + def __init__(self, + model_path, + model_type='base', + precision='FP32', + tf_config=None, + cropping=None, + dynamic=(False,.5,10), + resize=None, + convert2rgb=True, + processor=None, + display=False, + display_lik=0.5, + display_radius=3): self.path = model_path self.cfg = None @@ -89,6 +104,7 @@ def __init__(self, model_path, self.dynamic_cropping = None self.resize = resize self.processor = processor + self.convert2rgb = convert2rgb self.display = Display(lik=display_lik, radius=display_radius) if display else None self.sess = None @@ -177,9 +193,8 @@ def process_frame(self, frame): if self.resize != 1: frame = utils.resize_frame(frame, self.resize) - - if frame.ndim == 2: - frame = utils.gray_to_rgb(frame) + if self.convert2rgb: + frame = utils.img_to_rgb(frame) return frame @@ -210,17 +225,19 @@ def init_inference(self, frame=None): if frame is None: raise DLCLiveError("No image was passed to initialize inference. An image must be passed to the init_inference method") - process_frame = self.process_frame(frame) + if frame.ndim == 2: + self.convert2rgb = True + frame = self.process_frame(frame) ### load model if self.model_type == 'base': - # graph_def = read_graph(model_file) - # self.graph, self.inputs = finalize_graph(graph_def) - # self.sess, self.outputs = extract_graph(self.graph, tf_config=self.tf_config) + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph(graph, tf_config=self.tf_config) - self.sess, self.inputs, self.outputs = load_graph(model_file) + #self.sess, self.inputs, self.outputs = load_graph(model_file) elif self.model_type == 'tflite': @@ -231,13 +248,13 @@ def init_inference(self, frame=None): # get input and output tensor names from graph_def graph_def = read_graph(model_file) - graph, _ = finalize_graph(graph_def) + graph = finalize_graph(graph_def) output_nodes = get_output_nodes(graph) - output_nodes = [on.replace('Placeholder_1/', '') for on in output_nodes] + output_nodes = [on.replace('DLC/', '') for on in output_nodes] converter = tf.lite.TFLiteConverter.from_frozen_graph(model_file, ['Placeholder'], output_nodes, - input_shapes={'Placeholder' : [1, process_frame.shape[0], process_frame.shape[1], 3]}) + input_shapes={'Placeholder' : [1, frame.shape[0], frame.shape[1], 3]}) try: tflite_model = converter.convert() except Exception: @@ -254,9 +271,9 @@ def init_inference(self, frame=None): elif self.model_type == 'tensorrt': graph_def = read_graph(model_file) - graph, _ = finalize_graph(graph_def) + graph = finalize_graph(graph_def) output_tensors = get_output_tensors(graph) - output_tensors = [ot.replace('Placeholder_1/', '') for ot in output_tensors] + output_tensors = [ot.replace('DLC/', '') for ot in output_tensors] if (TFVER[0] > 1) | (TFVER[0]==1 & TFVER[1] >= 14): converter = trt.TrtGraphConverter(input_graph_def=graph_def, diff --git a/dlclive/graph.py b/dlclive/graph.py index 69adce7..c876c63 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -49,12 +49,15 @@ def finalize_graph(graph_def): ''' graph = tf.Graph() + # with graph.as_default(): + # inputs = tf.placeholder(tf.float32, shape=[1, None, None, 3]) + # tf.import_graph_def(graph_def, {'Placeholder' : inputs}, name='Placeholder') with graph.as_default(): - inputs = tf.placeholder(tf.float32, shape=[1, None, None, 3]) - tf.import_graph_def(graph_def, {'Placeholder' : inputs}, name='Placeholder') + tf.import_graph_def(graph_def, name="DLC") graph.finalize() - return graph, inputs + #return graph, inputs + return graph def get_output_nodes(graph): @@ -97,10 +100,16 @@ def get_output_tensors(graph): ''' output_nodes = get_output_nodes(graph) - output_tensor = [out+':0' for out in output_nodes] + output_tensor = [out+":0" for out in output_nodes] return output_tensor +def get_input_tensor(graph): + + input_tensor = str(graph.get_operations()[0].name) + ":0" + return input_tensor + + def extract_graph(graph, tf_config=None): ''' Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs @@ -119,30 +128,32 @@ def extract_graph(graph, tf_config=None): the output tensor(s) for the model ''' + input_tensor = get_input_tensor(graph) output_tensor = get_output_tensors(graph) sess = tf.Session(graph=graph, config=tf_config) + inputs = graph.get_tensor_by_name(input_tensor) outputs = [graph.get_tensor_by_name(out) for out in output_tensor] - return sess, outputs + return sess, inputs, outputs -def load_graph(pb_file): +# def load_graph(pb_file): - graph = tf.Graph() - with tf.gfile.GFile(pb_file, 'rb') as f: - graph_def = tf.GraphDef() - graph_def.ParseFromString(f.read()) +# graph = tf.Graph() +# with tf.gfile.GFile(pb_file, 'rb') as f: +# graph_def = tf.GraphDef() +# graph_def.ParseFromString(f.read()) - with graph.as_default(): - tf.import_graph_def(graph_def, name='DLC') - graph.finalize() +# with graph.as_default(): +# tf.import_graph_def(graph_def, name='DLC') +# graph.finalize() - op_names = [op.name for op in graph.get_operations()] - inputs = graph.get_tensor_by_name(op_names[0] + ':0') - outputs = [graph.get_tensor_by_name(op_names[-1] + ':0')] - if 'Sigmoid' in op_names[-1]: - outputs.append(graph.get_tensor_by_name(op_names[-2] + ':0')) +# op_names = [op.name for op in graph.get_operations()] +# inputs = graph.get_tensor_by_name(op_names[0] + ':0') +# outputs = [graph.get_tensor_by_name(op_names[-1] + ':0')] +# if 'Sigmoid' in op_names[-1]: +# outputs.append(graph.get_tensor_by_name(op_names[-2] + ':0')) - sess = tf.Session(graph=graph) +# sess = tf.Session(graph=graph) - return sess, inputs, outputs \ No newline at end of file +# return sess, inputs, outputs \ No newline at end of file diff --git a/dlclive/utils.py b/dlclive/utils.py index c3d1389..244697f 100644 --- a/dlclive/utils.py +++ b/dlclive/utils.py @@ -55,10 +55,8 @@ def resize_frame(frame, resize=None): an image as a numpy array """ - if (resize is not None) and (resize != 1): - if OPEN_CV: new_x = int(frame.shape[0] * resize) @@ -75,10 +73,9 @@ def resize_frame(frame, resize=None): return frame +def img_to_rgb(frame): + """ Convert an image to RGB. Uses OpenCV is installed, otherwise uses pillow. -def gray_to_rgb(frame): - """ Convert an image from grayscale to RGB. Uses OpenCV is installed, otherwise uses pillow. - Parameters ---------- frame : :class:`numpy.ndarray @@ -87,21 +84,58 @@ def gray_to_rgb(frame): if frame.ndim == 2: - if OPEN_CV: + return gray_to_rgb(frame) - return cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB) - - else: + elif frame.ndim == 3: - img = Image.fromarray(frame) - img = img.convert('RGB') - return np.asarray(img) + return bgr_to_rgb(frame) else: + warnings.warn(f"Image has {frame.ndim} dimensions. Must be 2 or 3 dimensions to convert to RGB", DLCLiveWarning) return frame +def gray_to_rgb(frame): + """ Convert an image from grayscale to RGB. Uses OpenCV is installed, otherwise uses pillow. + + Parameters + ---------- + frame : :class:`numpy.ndarray + an image as a numpy array + """ + + if OPEN_CV: + + return cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB) + + else: + + img = Image.fromarray(frame) + img = img.convert('RGB') + return np.asarray(img) + + +def bgr_to_rgb(frame): + """ Convert an image from BGR to RGB. Uses OpenCV is installed, otherwise uses pillow. + + Parameters + ---------- + frame : :class:`numpy.ndarray + an image as a numpy array + """ + + if OPEN_CV: + + return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + else: + + img = Image.fromarray(frame) + img = img.convert('RGB') + return np.asarray(img) + + def _img_as_ubyte_np(frame): """ Converts an image as a numpy array to unsinged 8-bit integer. As in scikit-image img_as_ubyte, converts negative pixels to 0 and converts range to [0, 255] From 65c0cfe9aa50ebf6d1707a7819c454a06d2672d1 Mon Sep 17 00:00:00 2001 From: gkane Date: Fri, 19 Jun 2020 15:26:40 -0400 Subject: [PATCH 07/28] dlclive: add kwargs to get_pose and init_inference --- dlclive/dlclive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 2830d35..708e7d6 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -199,7 +199,7 @@ def process_frame(self, frame): return frame - def init_inference(self, frame=None): + def init_inference(self, frame=None, **kwargs): ''' Load model and perform inference on first frame -- the first inference is usually very slow. @@ -296,7 +296,7 @@ def init_inference(self, frame=None): ### get pose of first frame (first inference is often very slow) - pose = self.get_pose(frame) + pose = self.get_pose(frame, **kwargs) self.is_initialized = True From f8d35a50ea5f301094ed434f4d5c502029ab29a3 Mon Sep 17 00:00:00 2001 From: gkane Date: Tue, 30 Jun 2020 17:35:19 -0400 Subject: [PATCH 08/28] add kalman filter processor --- dlclive/processor/__init__.py | 1 + dlclive/processor/kalmanfilter.py | 138 ++++++++++++++++++++++++++++++ dlclive/processor/processor.py | 2 +- 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 dlclive/processor/kalmanfilter.py diff --git a/dlclive/processor/__init__.py b/dlclive/processor/__init__.py index 6b7b977..3e6b6b2 100644 --- a/dlclive/processor/__init__.py +++ b/dlclive/processor/__init__.py @@ -1 +1,2 @@ from dlclive.processor.processor import Processor +from dlclive.processor.kalmanfilter import KalmanFilterPredictor diff --git a/dlclive/processor/kalmanfilter.py b/dlclive/processor/kalmanfilter.py new file mode 100644 index 0000000..1891fd7 --- /dev/null +++ b/dlclive/processor/kalmanfilter.py @@ -0,0 +1,138 @@ +""" +DeepLabCut2.0 Toolbox (deeplabcut.org) +© A. & M. Mathis Labs +https://github.com/AlexEMG/DeepLabCut + +Please see AUTHORS for contributors. +https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS +Licensed under GNU Lesser General Public License v3.0 +""" + + +import time +import numpy as np +from dlclive.processor import Processor + + +class KalmanFilterPredictor(Processor): + + + def __init__(self, + adapt=True, + forward=0.002, + fps=30, + nderiv=2, + priors=[10, 1], + initial_var=1, + process_var=1, + dlc_var=20, + **kwargs): + + super().__init__(**kwargs) + + self.adapt=adapt + self.forward = forward + self.dt = 1.0 / fps + self.nderiv = nderiv + self.priors = np.hstack(([1e5], priors)) + self.initial_var = initial_var + self.process_var = process_var + self.dlc_var = dlc_var + self.is_initialized = False + + + def _get_forward_model(self, dt): + + F = np.zeros((self.n_states, self.n_states)) + for d in range(self.nderiv+1): + for i in range(self.n_states - (d * self.bp * 2)): + F[i, i + (2 * self.bp * d)] = (dt ** d) / max(1, d) + + return F + + + def _init_kf(self, pose): + + # get number of body parts + self.bp = pose.shape[0] + self.n_states = self.bp * 2 * (self.nderiv+1) + + # initialize state matrix, set position to first pose + self.X = np.zeros((self.n_states, 1)) + self.X[:(self.bp * 2)] = pose[:, :2].reshape(self.bp * 2, 1) + + # initialize covariance matrix, measurement noise and process noise + self.P = np.eye(self.n_states) * self.initial_var + self.R = np.eye(self.n_states) * self.dlc_var + self.Q = np.eye(self.n_states) * self.process_var + + # initialize forward model: + self.F = self._get_forward_model(self.dt) + + self.H = np.eye(self.n_states) + self.K = np.zeros((self.n_states, self.n_states)) + self.I = np.eye(self.n_states) + + # initialize priors for forward prediction step only + B = np.repeat(self.priors, self.bp * 2) + self.B = B.reshape(B.size, 1) + + self.is_initialized = True + + + def _predict(self): + + self.Xp = np.dot(self.F, self.X) + self.Pp = np.dot(np.dot(self.F, self.P), self.F.T) + self.Q + + + def _get_residuals(self, pose): + + z = np.zeros((self.n_states, 1)) + z[:(self.bp * 2)] = pose[:self.bp, :2].reshape(self.bp * 2, 1) + for i in range(self.bp * 2, self.n_states): + z[i] = (z[i - (self.bp * 2)] - self.X[i - (self.bp * 2)]) / self.dt + self.y = z - np.dot(self.H, self.Xp) + + + def _update(self): + + S = np.dot(self.H, np.dot(self.Pp, self.H.T)) + self.R + K = np.dot(np.dot(self.Pp, self.H.T), np.linalg.inv(S)) + self.X = self.Xp + np.dot(K, self.y) + self.P = np.dot(self.I - np.dot(K, self.H), self.Pp) + + + def _get_future_pose(self, dt): + + print(dt) + + Ff = self._get_forward_model(dt) + + Pf = np.diag(self.P).reshape(self.P.shape[0], 1) + Xf = (1 / ((1 / Pf) + (1 / self.B))) * (self.X / Pf) + Xfp = np.dot(Ff, Xf) + + future_pose = Xfp[:(self.bp * 2)].reshape(self.bp, 2) + + return future_pose + + + def process(self, pose, **kwargs): + + if not self.is_initialized: + + self._init_kf(pose) + return pose + + else: + + self._predict() + self._get_residuals(pose) + self._update() + + forward_time = (time.time() - kwargs['frame_time'] + self.forward) if self.adapt else self.forward + future_pose = self._get_future_pose(forward_time) + future_pose = np.hstack((future_pose, pose[:,2].reshape(self.bp,1))) + + return future_pose \ No newline at end of file diff --git a/dlclive/processor/processor.py b/dlclive/processor/processor.py index 017f2c1..a7d6e83 100644 --- a/dlclive/processor/processor.py +++ b/dlclive/processor/processor.py @@ -16,7 +16,7 @@ class Processor(object): - def __init__(self): + def __init__(self, **kwargs): pass def process(self, pose, **kwargs): From 36475504ccee923c02d97626a6e5a04bddcbda14 Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 07:51:16 -0400 Subject: [PATCH 09/28] kalmanfilter processor: update delta time --- dlclive/processor/kalmanfilter.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/dlclive/processor/kalmanfilter.py b/dlclive/processor/kalmanfilter.py index 1891fd7..44182c0 100644 --- a/dlclive/processor/kalmanfilter.py +++ b/dlclive/processor/kalmanfilter.py @@ -23,9 +23,9 @@ def __init__(self, fps=30, nderiv=2, priors=[10, 1], - initial_var=1, - process_var=1, - dlc_var=20, + initial_var=5, + process_var=5, + dlc_var=10, **kwargs): super().__init__(**kwargs) @@ -39,6 +39,7 @@ def __init__(self, self.process_var = process_var self.dlc_var = dlc_var self.is_initialized = False + self.last_pose_time = 0 def _get_forward_model(self, dt): @@ -82,6 +83,7 @@ def _init_kf(self, pose): def _predict(self): + #self.F = self._get_forward_model(time.time()-self.last_pose_time) self.Xp = np.dot(self.F, self.X) self.Pp = np.dot(np.dot(self.F, self.P), self.F.T) + self.Q @@ -105,8 +107,6 @@ def _update(self): def _get_future_pose(self, dt): - print(dt) - Ff = self._get_forward_model(dt) Pf = np.diag(self.P).reshape(self.P.shape[0], 1) @@ -116,6 +116,15 @@ def _get_future_pose(self, dt): future_pose = Xfp[:(self.bp * 2)].reshape(self.bp, 2) return future_pose + + + def _get_state_likelihood(self, pose): + + liks = pose[:, 2] + liks_xy = np.repeat(liks, 2) + liks_xy_deriv = np.tile(liks_xy, self.nderiv) + liks_state = liks_xy_deriv.reshape(liks_xy_deriv.shape[0], 1) + return(liks_state) def process(self, pose, **kwargs): @@ -123,10 +132,12 @@ def process(self, pose, **kwargs): if not self.is_initialized: self._init_kf(pose) + + self.last_pose_time = time.time() return pose else: - + self._predict() self._get_residuals(pose) self._update() @@ -134,5 +145,6 @@ def process(self, pose, **kwargs): forward_time = (time.time() - kwargs['frame_time'] + self.forward) if self.adapt else self.forward future_pose = self._get_future_pose(forward_time) future_pose = np.hstack((future_pose, pose[:,2].reshape(self.bp,1))) - + + self.last_pose_time = time.time() return future_pose \ No newline at end of file From cca80d8c9063ed5dff8098a6a8f0a41c07f068ee Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 10:25:42 -0400 Subject: [PATCH 10/28] dlclive: destroy display on close --- dlclive/dlclive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 708e7d6..c80aa0b 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -391,3 +391,5 @@ def close(self): self.sess.close() self.sess = None self.is_initialized = False + if self.display is not None: + self.display.destroy() From 5f4ef0930ef40e9feb18220d8e8e631c208c4131 Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 10:37:26 -0400 Subject: [PATCH 11/28] new analyze tools: speed bench, display, analyze pose and labeled video --- dlclive/__init__.py | 1 + dlclive/analyze.py | 467 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 dlclive/analyze.py diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 3fc2b89..885e23b 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -9,3 +9,4 @@ from dlclive.dlclive import DLCLive from dlclive.processor import Processor from dlclive.bench import benchmark_model_by_size +from dlclive.analyze import analyze, analyze_videos diff --git a/dlclive/analyze.py b/dlclive/analyze.py new file mode 100644 index 0000000..c1f900c --- /dev/null +++ b/dlclive/analyze.py @@ -0,0 +1,467 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + + +import platform +import os +import time +import sys +import warnings +import argparse +import pickle +import colorcet as cc +from PIL import ImageColor +import ruamel +import pandas as pd + +import cpuinfo +from tqdm import tqdm +import numpy as np +import tensorflow as tf +import cv2 + +from dlclive import DLCLive + + +def get_system_info(): + """ Return summary info for system running benchmark + + Returns + ------- + str + name of machine + str + operating system + str + path to python (which conda/virtual environment) + str + device name + """ + + + ### get os + + op_sys = platform.platform() + host_name = platform.node() + if platform.system() == 'Windows': + host_python = sys.executable.split(os.path.sep)[-2] + else: + host_python = sys.executable.split(os.path.sep)[-3] + + ### get device info (GPU or CPU) + + dev = None + if tf.test.is_gpu_available(): + gpu_name = tf.test.gpu_device_name() + from tensorflow.python.client import device_lib + dev_desc = [d.physical_device_desc for d in device_lib.list_local_devices() if d.name == gpu_name] + dev = [d.split(",")[1].split(':')[1].strip() for d in dev_desc] + dev_type = "GPU" + else: + from cpuinfo import get_cpu_info + dev = [get_cpu_info()['brand']] + dev_type = "CPU" + + return host_name, op_sys, host_python, (dev_type, dev) + + +def analyze(model_path, + video_path, + tf_config=None, + resize=None, + pixels=None, + n_frames=1000, + print_rate=False, + display=False, + display_lik=0.0, + display_radius=3, + cmap='bmy', + save_poses=False, + save_video=False, + output=None): + """ Analyze DeepLabCut-live exported model on a video: + Calculate inference time, + display keypoints, or + get poses/create a labeled video + + Parameters + ---------- + model_path : str + path to exported DeepLabCut model + video_path : str + path to video file + tf_config : :class:`tensorflow.ConfigProto` + tensorflow session configuration + resize : int, optional + resize factor. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + pixels : int, optional + downsize image to this number of pixels, maintaining aspect ratio. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + n_frames : int, optional + number of frames to run inference on, by default 1000 + print_rate : bool, optional + flat to print inference rate frame by frame, by default False + display : bool, optional + flag to display keypoints on images. Useful for checking the accuracy of exported models. + display_lik : float, optional + likelihood threshold to display keypoints + display_radius : int, optional + size (radius in pixels) of keypoint to display + cmap : str, optional + a string indicating the :package:`colorcet` colormap, `options here `, by default "bmy" + save_poses : bool, optional + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.analyze_videos`, by default False + save_video : bool, optional + flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + output : str, optional + path to directory to save pose and/or video file. If not specified, will use the directory of video_path, by default None + + Returns + ------- + :class:`numpy.ndarray` + vector of inference times + float + number of pixels in each image + + Example + ------- + Return a vector of inference times for 10000 frames: + dlclive.analyze('/my/exported/model', 'my_video.avi', n_frames=10000) + + Return a vector of inference times, resizing images to half the width and height for inference + dlclive.analyze('/my/exported/model', 'my_video.avi', n_frames=10000, resize=0.5) + + Display keypoints to check the accuracy of an exported model + dlclive.analyze('/my/exported/model', 'my_video.avi', display=True) + + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.analyze_videos` and :function:`create_labeled_video` + dlclive.analyze('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + """ + + ### load video + + cap = cv2.VideoCapture(video_path) + ret, frame = cap.read() + n_frames = n_frames if n_frames < cap.get(cv2.CAP_PROP_FRAME_COUNT)-1 else cap.get(cv2.CAP_PROP_FRAME_COUNT)-1 + n_frames = int(n_frames) + im_size = (cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + + ### get resize factor + + if pixels is not None: + resize = np.sqrt(pixels / (im_size[0] * im_size[1])) + if resize is not None: + im_size = (int(im_size[0]*resize), int(im_size[1]*resize)) + + ### initialize live object + + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius) + live.init_inference(frame) + TFGPUinference = True if len(live.outputs) == 1 else False + + ### create video writer + + if save_video: + colors = None + out_dir = output if output is not None else os.path.dirname(os.path.realpath(video_path)) + out_vid_base = os.path.basename(video_path) + out_vid_file = os.path.normpath(f"{out_dir}/{os.path.splitext(video_path)[0]}_DLCLIVE_LABELED.avi") + fourcc = cv2.VideoWriter_fourcc(*'DIVX') + fps = cap.get(cv2.CAP_PROP_FPS) + vwriter = cv2.VideoWriter(out_vid_file, fourcc, fps, im_size) + + ### perform inference + + iterator = range(n_frames) if (print_rate) or (display) else tqdm(range(n_frames)) + inf_times = np.zeros(n_frames) + poses = [] + + for i in iterator: + + ret, frame = cap.read() + + if not ret: + warnings.warn("Did not complete {:d} frames. There probably were not enough frames in the video {}.".format(n_frames, video_path)) + break + + start_pose = time.time() + poses.append(live.get_pose(frame)) + inf_times[i] = time.time() - start_pose + + if save_video: + + if colors is None: + all_colors = getattr(cc, cmap) + colors = [ImageColor.getcolor(c, "RGB")[::-1] for c in all_colors[::int(len(all_colors)/poses[-1].shape[0])]] + + this_pose = poses[-1] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > display_lik: + x = int(this_pose[j, 0]) + y = int(this_pose[j, 1]) + frame = cv2.circle(frame, (x, y), display_radius, colors[j], thickness=-1) + + if resize is not None: + frame = cv2.resize(frame, im_size) + vwriter.write(frame) + + if print_rate: + print("pose rate = {:d}".format(int(1 / inf_times[i]))) + + if print_rate: + print("mean pose rate = {:d}".format(int(np.mean(1/inf_times)))) + + ### close video and tensorflow session + + cap.release() + live.close() + + if save_video: + vwriter.release() + + if save_poses: + + cfg_path = os.path.normpath(f"{model_path}/pose_cfg.yaml") + ruamel_file = ruamel.yaml.YAML() + dlc_cfg = ruamel_file.load(open(cfg_path, 'r')) + bodyparts = dlc_cfg['all_joints_names'] + poses = np.array(poses) + poses = poses.reshape((poses.shape[0], poses.shape[1]*poses.shape[2])) + pdindex = pd.MultiIndex.from_product([bodyparts, ['x', 'y', 'likelihood']], names=['bodyparts', 'coords']) + pose_df = pd.DataFrame(poses, columns=pdindex) + + out_dir = output if output is not None else os.path.dirname(os.path.realpath(video_path)) + out_vid_base = os.path.basename(video_path) + out_dlc_file = os.path.normpath(f"{out_dir}/{os.path.splitext(video_path)[0]}_DLCLIVE_POSES.h5") + pose_df.to_hdf(out_dlc_file, key='df_with_missing', mode='w') + + return inf_times, im_size[0]*im_size[1], TFGPUinference + + +def save_inf_times(sys_info, + inf_times, + pixels, + TFGPUinference, + model=None, + output=None): + """ Save inference time data collected using :function:`analyze` with system information to a pickle file. + This is primarily used through :function:`analyze_videos` + + Parameters + ---------- + sys_info : tuple + system information generated by :func:`get_system_info` + inf_times : :class:`numpy.ndarray` + array of inference times generated by :func:`analyze` + pixels : float or :class:`numpy.ndarray` + number of pixels for each benchmark run. If an array, each index corresponds to a row in inf_times + TFGPUinference: bool + flag if using tensorflow inference or numpy inference DLC model + model: str, optional + name of model + output : str, optional + path to directory to save data. If None, uses pwd, by default None + + Returns + ------- + bool + flag indicating successful save + """ + + output = output if output is not None else os.getcwd() + host_name, op_sys, host_python, dev = sys_info + host_name = host_name.replace(" ", "") + model_type = None + if model is not None: + if 'resnet' in model: + model_type = 'resnet' + elif 'mobilenet' in model: + model_type = 'mobilenet' + else: + model_type = None + + fn_ind = 0 + base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind) + while os.path.isfile(os.path.normpath(output + '/' + base_name)): + fn_ind += 1 + base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind) + + fn = os.path.normpath(output) + + data = {'host_name' : host_name, + 'op_sys' : op_sys, + 'python' : host_python, + 'device_type' : dev[0], + 'device' : dev[1], + 'model' : model, + 'model_type' : model_type, + 'TFGPUinference' : TFGPUinference, + 'pixels' : pixels, + 'inference_times' : inf_times} + + os.makedirs(os.path.normpath(output), exist_ok=True) + pickle.dump(data, open(os.path.normpath(f"{output}/{base_name}"), 'wb')) + + return True + + +def analyze_videos(model_path, + video_path, + output=None, + n_frames=1000, + tf_config=None, + resize=None, + pixels=None, + print_rate=False, + display=False, + display_lik=0.5, + display_radius=3, + save_poses=False, + save_video=False, + cmap='bmy'): + """Analyze videos using DeepLabCut-live exported models. + Analyze multiple videos and/or multiple options for the size of the video + by specifying a resizing factor or the number of pixels to use in the image (keeping aspect ratio constant). + Options to record inference times (to examine inference speed), + display keypoints to visually check the accuracy, + or save poses to an hdf5 file as in :function:`deeplabcut.analyze_videos` and + create a labeled video as in :function:`deeplabcut.create_labeled_video`. + + Parameters + ---------- + model_path : str + path to exported DeepLabCut model + video_path : str or list + path to video file or list of paths to video files + output : str + path to directory to save results + tf_config : :class:`tensorflow.ConfigProto` + tensorflow session configuration + resize : int, optional + resize factor. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + pixels : int, optional + downsize image to this number of pixels, maintaining aspect ratio. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + n_frames : int, optional + number of frames to run inference on, by default 1000 + print_rate : bool, optional + flat to print inference rate frame by frame, by default False + display : bool, optional + flag to display keypoints on images. Useful for checking the accuracy of exported models. + display_lik : float, optional + likelihood threshold to display keypoints + display_radius : int, optional + size (radius in pixels) of keypoint to display + cmap : str, optional + a string indicating the :package:`colorcet` colormap, `options here `, by default "bmy" + save_poses : bool, optional + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.analyze_videos`, by default False + save_video : bool, optional + flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + + Example + ------- + Return a vector of inference times for 10000 frames on one video or two videos: + dlclive.analyze_videos('/my/exported/model', 'my_video.avi', n_frames=10000) + dlclive.analyze_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000) + + Return a vector of inference times, testing full size and resizing images to half the width and height for inference, for two videos + dlclive.analyze_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000, resize=[1.0, 0.5]) + + Display keypoints to check the accuracy of an exported model + dlclive.analyze_videos('/my/exported/model', 'my_video.avi', display=True) + + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.analyze_videos` and :function:`create_labeled_video` + dlclive.analyze_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + """ + + ### convert video_paths to list + + video_path = video_path if type(video_path) is list else [video_path] + + ### fix resize + + if pixels: + resize = [None for p in pixels] + elif resize: + pixels = [None for r in resize] + else: + resize = [None] + pixels = [None] + + ### loop over videos + + for v in video_path: + + ### initialize full inference times + + inf_times = np.zeros((len(resize), n_frames)) + pixels_out = np.zeros(len(resize)) + + for i in range(len(resize)): + + print("\nRun {:d} / {:d}\n".format(i+1, len(resize))) + + inf_times[i], pixels_out[i], TFGPUinference = analyze(model_path, + v, + tf_config=tf_config, + resize=resize[i], + pixels=pixels[i], + n_frames=n_frames, + print_rate=print_rate, + display=display, + display_lik=display_lik, + display_radius=display_radius, + cmap=cmap, + save_poses=save_poses, + save_video=save_video, + output=output) + + ### save results + + if output is not None: + sys_info = get_system_info() + save_inf_times(sys_info, inf_times, pixels_out, TFGPUinference, model=os.path.basename(model_path), output=output) + + +def main(): + """Provides a command line interface :function:`analyze_videos` + """ + + parser = argparse.ArgumentParser() + parser.add_argument('model_path', type=str) + parser.add_argument('video_path', type=str, nargs='+') + parser.add_argument('-o', '--output', type=str, default=None) + parser.add_argument('-n', '--n-frames', type=int, default=10000) + parser.add_argument('-r', '--resize', type=float, nargs='+') + parser.add_argument('-p', '--pixels', type=float, nargs='+') + parser.add_argument('-v', '--print-rate', default=False, action='store_true') + parser.add_argument('-d', '--display', default=False, action='store_true') + parser.add_argument('-l', '--display-lik', default=0.5, type=float) + parser.add_argument('-s', '--display-radius', default=3, type=int) + parser.add_argument('-c', '--cmap', type=str, default='bmy') + parser.add_argument('--save-poses', action='store_true') + parser.add_argument('--save-video', action='store_true') + args = parser.parse_args() + + + analyze_videos(args.model_path, + args.video_path, + output=args.output, + resize=args.resize, + pixels=args.pixels, + n_frames=args.n_frames, + print_rate=args.print_rate, + display=args.display, + display_lik=args.display_lik, + display_radius=args.display_radius, + cmap=args.cmap, + save_poses=args.save_poses, + save_video=args.save_video) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index a173ff6..99f72bb 100644 --- a/setup.py +++ b/setup.py @@ -41,5 +41,6 @@ "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", "Operating System :: OS Independent", ), - entry_points = {'console_scripts' : ['dlc-live-bench=dlclive.bench:main']} + entry_points = {'console_scripts' : ['dlc-live-bench=dlclive.bench:main', + 'dlc-live-analyze=dlclive.analyze:main']} ) From 34073767560f6d9655300b8d91e56ca4cfd1dc69 Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 10:56:17 -0400 Subject: [PATCH 12/28] dlclive and graph: clean up/benign bug fix --- dlclive/dlclive.py | 5 ++--- dlclive/graph.py | 27 --------------------------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index c80aa0b..4f7e92c 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -110,7 +110,6 @@ def __init__(self, self.sess = None self.inputs = None self.outputs = None - self.graph = None self.tflite_interpreter = None self.pose = None self.is_initialized = False @@ -287,8 +286,8 @@ def init_inference(self, frame=None, **kwargs): precision_mode=self.precision, is_dynamic_op=True) - self.graph, self.inputs = finalize_graph(graph_def) - self.sess, self.outputs = extract_graph(self.graph, tf_config=self.tf_config) + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph(graph, tf_config=self.tf_config) else: diff --git a/dlclive/graph.py b/dlclive/graph.py index c876c63..17716af 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -24,7 +24,6 @@ def read_graph(file): The graph definition of the DeepLabCut model found at the object's path ''' - graph = tf.Graph() with tf.gfile.GFile(file, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) @@ -49,14 +48,10 @@ def finalize_graph(graph_def): ''' graph = tf.Graph() - # with graph.as_default(): - # inputs = tf.placeholder(tf.float32, shape=[1, None, None, 3]) - # tf.import_graph_def(graph_def, {'Placeholder' : inputs}, name='Placeholder') with graph.as_default(): tf.import_graph_def(graph_def, name="DLC") graph.finalize() - #return graph, inputs return graph @@ -135,25 +130,3 @@ def extract_graph(graph, tf_config=None): outputs = [graph.get_tensor_by_name(out) for out in output_tensor] return sess, inputs, outputs - - -# def load_graph(pb_file): - -# graph = tf.Graph() -# with tf.gfile.GFile(pb_file, 'rb') as f: -# graph_def = tf.GraphDef() -# graph_def.ParseFromString(f.read()) - -# with graph.as_default(): -# tf.import_graph_def(graph_def, name='DLC') -# graph.finalize() - -# op_names = [op.name for op in graph.get_operations()] -# inputs = graph.get_tensor_by_name(op_names[0] + ':0') -# outputs = [graph.get_tensor_by_name(op_names[-1] + ':0')] -# if 'Sigmoid' in op_names[-1]: -# outputs.append(graph.get_tensor_by_name(op_names[-2] + ':0')) - -# sess = tf.Session(graph=graph) - -# return sess, inputs, outputs \ No newline at end of file From 95cedd92b0e4aaef7e1f57725936ab6b9ae1f23d Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 11:24:05 -0400 Subject: [PATCH 13/28] fix colormaps for display --- dlclive/analyze.py | 7 +++---- dlclive/dlclive.py | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dlclive/analyze.py b/dlclive/analyze.py index c1f900c..578d449 100644 --- a/dlclive/analyze.py +++ b/dlclive/analyze.py @@ -159,7 +159,7 @@ def analyze(model_path, ### initialize live object - live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius) + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius, display_cmap=cmap) live.init_inference(frame) TFGPUinference = True if len(live.outputs) == 1 else False @@ -320,9 +320,9 @@ def analyze_videos(model_path, display=False, display_lik=0.5, display_radius=3, + cmap='bmy', save_poses=False, - save_video=False, - cmap='bmy'): + save_video=False): """Analyze videos using DeepLabCut-live exported models. Analyze multiple videos and/or multiple options for the size of the video by specifying a resizing factor or the number of pixels to use in the image (keeping aspect ratio constant). @@ -447,7 +447,6 @@ def main(): parser.add_argument('--save-video', action='store_true') args = parser.parse_args() - analyze_videos(args.model_path, args.video_path, output=args.output, diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 4f7e92c..594a145 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -92,7 +92,8 @@ def __init__(self, processor=None, display=False, display_lik=0.5, - display_radius=3): + display_radius=3, + display_cmap='bmy'): self.path = model_path self.cfg = None @@ -105,7 +106,7 @@ def __init__(self, self.resize = resize self.processor = processor self.convert2rgb = convert2rgb - self.display = Display(lik=display_lik, radius=display_radius) if display else None + self.display = Display(lik=display_lik, radius=display_radius, cmap=display_cmap) if display else None self.sess = None self.inputs = None From 01a7bbe154958d6fb70ec072a0d70f34a330ef33 Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 11:26:59 -0400 Subject: [PATCH 14/28] update readme: add benchmark info --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index d98fd25..d7e69d1 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,34 @@ dlc_live.get_pose() - `resize` = float, optional; factor by which to resize image (resize=0.5 downsizes both width and height of image by half). Can be used to downsize large images for faster inference - `processor` = dlc pose processor object, optional - `display` = bool, optional; display processed image with DeepLabCut points? Can be used to troubleshoot cropping and resizing parameters, but is very slow + + +### Benchmarking/Analyzing Exported DeepLabCut Models + +DeepLabCut-live offers some analysis tools that allow users to peform the following operations on videos, from python or from the command line: +1. Test inference speed across a range of image sizes, downsizing images by specifying the `resize` or `pixels` parameter. Using the `pixels` parameter will resize images to the desired number of `pixels`, without changing the aspect ratio. Results will be saved (along with system info) to a pickle file if you specify an output directory. +``` +# python +dlclive.analyze_videos('/path/to/exported/model', ['/path/to/video1', '/path/to/video2'], output='/path/to/output', resize=[1.0, 0.75, '0.5']) + +# command line +dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 +``` + +2. Display keypoints to visually inspect the accuracy of exported models on different image sizes: +``` +# python +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, display_lik=0.5, display_radius=4, cmap='bmy') + +# command line +dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --display-lik 0.5 --display-radius 4 --cmap bmy +``` + +3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video`. +``` +# python +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, '0.5'], display_lik=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) + +# command line +dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display-lik 0.5 --display-radius 4 --cmap bmy --save_poses --save_video +``` \ No newline at end of file From 9b0ef34e5a924161c3f9b9c3cb6696e23d00a8ef Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 12:45:33 -0400 Subject: [PATCH 15/28] setup: add pandas and tables, needed for analyze_videos --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 99f72bb..37b97e4 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ with open("README.md", "r") as fh: long_description = fh.read() -install_requires = ['numpy', 'ruamel.yaml', 'colorcet', 'pillow', 'py-cpuinfo', 'tqdm'] +install_requires = ['numpy', 'ruamel.yaml', 'colorcet', 'pillow', 'py-cpuinfo', 'tqdm', 'pandas', 'tables'] if find_spec('cv2') is None: install_requires.append('opencv-python') From 4436266836615bef58ec1eecac9baaab394a8480 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Thu, 9 Jul 2020 12:57:23 -0400 Subject: [PATCH 16/28] Update kalmanfilter.py --- dlclive/processor/kalmanfilter.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/dlclive/processor/kalmanfilter.py b/dlclive/processor/kalmanfilter.py index 44182c0..c3ceaf1 100644 --- a/dlclive/processor/kalmanfilter.py +++ b/dlclive/processor/kalmanfilter.py @@ -1,10 +1,6 @@ """ -DeepLabCut2.0 Toolbox (deeplabcut.org) -© A. & M. Mathis Labs -https://github.com/AlexEMG/DeepLabCut - +DeepLabCut-live Toolbox (deeplabcut.org) Please see AUTHORS for contributors. -https://github.com/AlexEMG/DeepLabCut/blob/master/AUTHORS Licensed under GNU Lesser General Public License v3.0 """ @@ -147,4 +143,4 @@ def process(self, pose, **kwargs): future_pose = np.hstack((future_pose, pose[:,2].reshape(self.bp,1))) self.last_pose_time = time.time() - return future_pose \ No newline at end of file + return future_pose From 8ea6b97e65e316ff1a11e088fec8b05eb494f7e7 Mon Sep 17 00:00:00 2001 From: gkane Date: Thu, 9 Jul 2020 13:52:58 -0400 Subject: [PATCH 17/28] change display_lik to pcutoff --- README.md | 6 +++--- dlclive/analyze.py | 18 +++++++++--------- dlclive/bench.py | 12 ++++++------ dlclive/display.py | 8 ++++---- dlclive/dlclive.py | 4 ++-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d7e69d1..2dcf1ba 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /pat 2. Display keypoints to visually inspect the accuracy of exported models on different image sizes: ``` # python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, display_lik=0.5, display_radius=4, cmap='bmy') +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, pcutoff=0.5, display_radius=4, cmap='bmy') # command line dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --display-lik 0.5 --display-radius 4 --cmap bmy @@ -73,8 +73,8 @@ dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --displ 3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video`. ``` # python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, '0.5'], display_lik=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, '0.5'], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) # command line -dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display-lik 0.5 --display-radius 4 --cmap bmy --save_poses --save_video +dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save_poses --save_video ``` \ No newline at end of file diff --git a/dlclive/analyze.py b/dlclive/analyze.py index 578d449..46e54e9 100644 --- a/dlclive/analyze.py +++ b/dlclive/analyze.py @@ -77,7 +77,7 @@ def analyze(model_path, n_frames=1000, print_rate=False, display=False, - display_lik=0.0, + pcutoff=0.0, display_radius=3, cmap='bmy', save_poses=False, @@ -106,7 +106,7 @@ def analyze(model_path, flat to print inference rate frame by frame, by default False display : bool, optional flag to display keypoints on images. Useful for checking the accuracy of exported models. - display_lik : float, optional + pcutoff : float, optional likelihood threshold to display keypoints display_radius : int, optional size (radius in pixels) of keypoint to display @@ -159,7 +159,7 @@ def analyze(model_path, ### initialize live object - live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius, display_cmap=cmap) + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, pcutoff=pcutoff, display_radius=display_radius, display_cmap=cmap) live.init_inference(frame) TFGPUinference = True if len(live.outputs) == 1 else False @@ -200,7 +200,7 @@ def analyze(model_path, this_pose = poses[-1] for j in range(this_pose.shape[0]): - if this_pose[j, 2] > display_lik: + if this_pose[j, 2] > pcutoff: x = int(this_pose[j, 0]) y = int(this_pose[j, 1]) frame = cv2.circle(frame, (x, y), display_radius, colors[j], thickness=-1) @@ -318,7 +318,7 @@ def analyze_videos(model_path, pixels=None, print_rate=False, display=False, - display_lik=0.5, + pcutoff=0.5, display_radius=3, cmap='bmy', save_poses=False, @@ -351,7 +351,7 @@ def analyze_videos(model_path, flat to print inference rate frame by frame, by default False display : bool, optional flag to display keypoints on images. Useful for checking the accuracy of exported models. - display_lik : float, optional + pcutoff : float, optional likelihood threshold to display keypoints display_radius : int, optional size (radius in pixels) of keypoint to display @@ -413,7 +413,7 @@ def analyze_videos(model_path, n_frames=n_frames, print_rate=print_rate, display=display, - display_lik=display_lik, + pcutoff=pcutoff, display_radius=display_radius, cmap=cmap, save_poses=save_poses, @@ -440,7 +440,7 @@ def main(): parser.add_argument('-p', '--pixels', type=float, nargs='+') parser.add_argument('-v', '--print-rate', default=False, action='store_true') parser.add_argument('-d', '--display', default=False, action='store_true') - parser.add_argument('-l', '--display-lik', default=0.5, type=float) + parser.add_argument('-l', '--pcutoff', default=0.5, type=float) parser.add_argument('-s', '--display-radius', default=3, type=int) parser.add_argument('-c', '--cmap', type=str, default='bmy') parser.add_argument('--save-poses', action='store_true') @@ -455,7 +455,7 @@ def main(): n_frames=args.n_frames, print_rate=args.print_rate, display=args.display, - display_lik=args.display_lik, + pcutoff=args.pcutoff, display_radius=args.display_radius, cmap=args.cmap, save_poses=args.save_poses, diff --git a/dlclive/bench.py b/dlclive/bench.py index 8b6e196..428c6dd 100644 --- a/dlclive/bench.py +++ b/dlclive/bench.py @@ -65,7 +65,7 @@ def get_system_info(): return host_name, op_sys, host_python, (dev_type, dev) -def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=None, n_frames=10000, print_rate=False, display=False, display_lik=0.0, display_radius=3): +def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=None, n_frames=10000, print_rate=False, display=False, pcutoff=0.0, display_radius=3): """ Benchmark on inference times for a given DLC model and video Parameters @@ -104,7 +104,7 @@ def run_benchmark(model_path, video_path, tf_config=None, resize=None, pixels=No ### initialize live object - live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, display_lik=display_lik, display_radius=display_radius) + live = DLCLive(model_path, tf_config=tf_config, resize=resize, display=display, pcutoff=pcutoff, display_radius=display_radius) live.init_inference(frame) TFGPUinference = True if len(live.outputs) == 1 else False @@ -200,7 +200,7 @@ def save_benchmark(sys_info, inf_times, pixels, TFGPUinference, model=None, out_ return True -def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, tf_config=None, resize=None, pixels=None, print_rate=False, display=False, display_lik=0.5, display_radius=3): +def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, tf_config=None, resize=None, pixels=None, print_rate=False, display=False, pcutoff=0.5, display_radius=3): """Benchmark DLC model by image size Parameters @@ -248,7 +248,7 @@ def benchmark_model_by_size(model_path, video_path, output=None, n_frames=10000, n_frames=n_frames, print_rate=print_rate, display=display, - display_lik=display_lik, + pcutoff=pcutoff, display_radius=display_radius) ### save results @@ -268,7 +268,7 @@ def main(): parser.add_argument('-p', '--pixels', type=float, nargs='+') parser.add_argument('-v', '--print_rate', default=False, action='store_true') parser.add_argument('-d', '--display', default=False, action='store_true') - parser.add_argument('-l', '--display-lik', default=0.5, type=float) + parser.add_argument('-l', '--pcutoff', default=0.5, type=float) parser.add_argument('-s', '--display-radius', default=3, type=int) args = parser.parse_args() @@ -281,7 +281,7 @@ def main(): n_frames=args.n_frames, print_rate=args.print_rate, display=args.display, - display_lik=args.display_lik, + pcutoff=args.pcutoff, display_radius=args.display_radius) diff --git a/dlclive/display.py b/dlclive/display.py index 2a551bd..534cb90 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -19,19 +19,19 @@ class Display(object): ----------- cmap : string string indicating the Matoplotlib colormap to use. - lik : float + pcutoff : float likelihood threshold to display points ''' - def __init__(self, cmap='bmy', radius=3, lik=0.5): + def __init__(self, cmap='bmy', radius=3, pcutoff=0.5): """ Constructor method """ self.cmap = cmap self.colors = None self.radius = radius - self.lik = lik + self.pcutoff = pcutoff self.window = None @@ -79,7 +79,7 @@ def display_frame(self, frame, pose=None): draw = ImageDraw.Draw(img) for i in range(pose.shape[0]): - if pose[i,2] > self.lik: + if pose[i,2] > self.pcutoff: try: x0 = pose[i,0] - self.radius if pose[i,0] - self.radius > 0 else 0 x1 = pose[i,0] + self.radius if pose[i,0] + self.radius < im_size[1] else im_size[1] diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 594a145..e526b78 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -91,7 +91,7 @@ def __init__(self, convert2rgb=True, processor=None, display=False, - display_lik=0.5, + pcutoff=0.5, display_radius=3, display_cmap='bmy'): @@ -106,7 +106,7 @@ def __init__(self, self.resize = resize self.processor = processor self.convert2rgb = convert2rgb - self.display = Display(lik=display_lik, radius=display_radius, cmap=display_cmap) if display else None + self.display = Display(pcutoff=pcutoff, radius=display_radius, cmap=display_cmap) if display else None self.sess = None self.inputs = None From 6a6a0d5ccb5a270e220e160ae3faa57d200cc0fe Mon Sep 17 00:00:00 2001 From: Gary Kane Date: Thu, 9 Jul 2020 14:08:51 -0400 Subject: [PATCH 18/28] resolve setup and readme conflicts --- README.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2dcf1ba..f3b9783 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ dlc_live.get_pose() - `processor` = dlc pose processor object, optional - `display` = bool, optional; display processed image with DeepLabCut points? Can be used to troubleshoot cropping and resizing parameters, but is very slow + `DLCLive` **inputs:** + + - `` = path to the folder that has the `.pb` files that you acquire after running `deeplabcut.export_model` + - `` = is a numpy array of each frame + ### Benchmarking/Analyzing Exported DeepLabCut Models diff --git a/setup.py b/setup.py index 37b97e4..07ad80b 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ with open("README.md", "r") as fh: long_description = fh.read() -install_requires = ['numpy', 'ruamel.yaml', 'colorcet', 'pillow', 'py-cpuinfo', 'tqdm', 'pandas', 'tables'] +install_requires = ['numpy', 'ruamel.yaml', 'colorcet', 'pillow', 'py-cpuinfo==5.0.0', 'tqdm', 'pandas', 'tables'] if find_spec('cv2') is None: install_requires.append('opencv-python') From de06ad2683e5094573af161065282108eb1965b5 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Thu, 9 Jul 2020 19:30:52 -0400 Subject: [PATCH 19/28] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 641d378..9e10536 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /pat dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, pcutoff=0.5, display_radius=4, cmap='bmy') # command line -dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --display-lik 0.5 --display-radius 4 --cmap bmy +dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy ``` 3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video`. @@ -82,4 +82,4 @@ dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, # command line dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save_poses --save_video -``` \ No newline at end of file +``` From 2aa7dc896ce8a33f2d7e90d7e869c792cee17b60 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Thu, 9 Jul 2020 19:32:54 -0400 Subject: [PATCH 20/28] fixed resize in python, check cmd --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9e10536..f0bf69b 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /pat 2. Display keypoints to visually inspect the accuracy of exported models on different image sizes: ``` # python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, pcutoff=0.5, display_radius=4, cmap='bmy') +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[0.5], display=True, pcutoff=0.5, display_radius=4, cmap='bmy') # command line dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy @@ -78,7 +78,7 @@ dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --pcuto 3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video`. ``` # python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, '0.5'], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) +dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, 0.5], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) # command line dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save_poses --save_video From 2fff10f3af8543a7a9db6d21ce66a813ec8a9a5d Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Thu, 9 Jul 2020 19:34:39 -0400 Subject: [PATCH 21/28] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0bf69b..c48862f 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ dlclive.analyze_videos('/path/to/exported/model', ['/path/to/video1', '/path/to/ dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 ``` -2. Display keypoints to visually inspect the accuracy of exported models on different image sizes: +2. Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow): ``` # python dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[0.5], display=True, pcutoff=0.5, display_radius=4, cmap='bmy') From faade24b0f0f5562ab57f94ccebc498c62f2956f Mon Sep 17 00:00:00 2001 From: Gary Kane Date: Fri, 10 Jul 2020 08:41:42 -0400 Subject: [PATCH 22/28] analyze: allow pixels/resize to be scalar or list --- dlclive/analyze.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dlclive/analyze.py b/dlclive/analyze.py index 46e54e9..a060e23 100644 --- a/dlclive/analyze.py +++ b/dlclive/analyze.py @@ -385,8 +385,10 @@ def analyze_videos(model_path, ### fix resize if pixels: + pixels = pixels if type(pixels) is list else [pixels] resize = [None for p in pixels] elif resize: + resize = resize if type(resize) is list else [resize] pixels = [None for r in resize] else: resize = [None] From 54fed1de7aacda465569f5ab87b7cfac1bf30a1b Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Fri, 10 Jul 2020 10:30:49 -0400 Subject: [PATCH 23/28] fix TF warnings --- dlclive/graph.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dlclive/graph.py b/dlclive/graph.py index 17716af..d2d16e3 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -20,12 +20,12 @@ def read_graph(file): Returns -------- - graph_def :class:`tensorflow.GraphDef` + graph_def :class:`tensorflow.tf.compat.v1.GraphDef` The graph definition of the DeepLabCut model found at the object's path ''' - with tf.gfile.GFile(file, 'rb') as f: - graph_def = tf.GraphDef() + with tf.io.gfile.GFile(file, 'rb') as f: + graph_def = tf.compat.v1.GraphDef() graph_def.ParseFromString(f.read()) return graph_def @@ -36,12 +36,12 @@ def finalize_graph(graph_def): Parameters ----------- - graph_def :class:`tensorflow.GraphDef` + graph_def :class:`tensorflow.compat.v1.GraphDef` The graph of the DeepLabCut model, read using the :func:`read_graph` method Returns -------- - graph :class:`tensorflow.Graph` + graph :class:`tensorflow.compat.v1.GraphDef` The finalized graph of the DeepLabCut model inputs :class:`tensorflow.Tensor` Input tensor(s) for the model @@ -125,7 +125,7 @@ def extract_graph(graph, tf_config=None): input_tensor = get_input_tensor(graph) output_tensor = get_output_tensors(graph) - sess = tf.Session(graph=graph, config=tf_config) + sess = tf.compat.v1.Session(graph=graph, config=tf_config) inputs = graph.get_tensor_by_name(input_tensor) outputs = [graph.get_tensor_by_name(out) for out in output_tensor] From a739cf6d089455c80374b336ebcf06cd7c3cc262 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Fri, 10 Jul 2020 10:35:10 -0400 Subject: [PATCH 24/28] changed name to benchmark_videos --- dlclive/analyze.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dlclive/analyze.py b/dlclive/analyze.py index a060e23..3208c68 100644 --- a/dlclive/analyze.py +++ b/dlclive/analyze.py @@ -309,7 +309,7 @@ def save_inf_times(sys_info, return True -def analyze_videos(model_path, +def benchmark_videos(model_path, video_path, output=None, n_frames=1000, @@ -365,17 +365,17 @@ def analyze_videos(model_path, Example ------- Return a vector of inference times for 10000 frames on one video or two videos: - dlclive.analyze_videos('/my/exported/model', 'my_video.avi', n_frames=10000) - dlclive.analyze_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000) + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', n_frames=10000) + dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000) Return a vector of inference times, testing full size and resizing images to half the width and height for inference, for two videos - dlclive.analyze_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000, resize=[1.0, 0.5]) + dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000, resize=[1.0, 0.5]) Display keypoints to check the accuracy of an exported model - dlclive.analyze_videos('/my/exported/model', 'my_video.avi', display=True) + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', display=True) Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.analyze_videos` and :function:`create_labeled_video` - dlclive.analyze_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) """ ### convert video_paths to list @@ -430,7 +430,7 @@ def analyze_videos(model_path, def main(): - """Provides a command line interface :function:`analyze_videos` + """Provides a command line interface :function:`benchmark_videos` """ parser = argparse.ArgumentParser() @@ -449,7 +449,7 @@ def main(): parser.add_argument('--save-video', action='store_true') args = parser.parse_args() - analyze_videos(args.model_path, + benchmark_videos(args.model_path, args.video_path, output=args.output, resize=args.resize, From a7c50e7ffdc7a7a79250341ac9512fca8d9199a2 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Fri, 10 Jul 2020 10:49:53 -0400 Subject: [PATCH 25/28] Update README.md --- README.md | 62 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c48862f..aaee4b7 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,15 @@ -# DeepLabCut-live DLC LIVE! +# DeepLabCut-live SDKDLC LIVE! -This package contains a DeepLabCut inference pipeline that has minimal (software) dependencies. Thus, it is as easy to install as possible (in particular, on atypical systems like NVIDIA Jetson boards). +![PyPI - Python Version](https://img.shields.io/pypi/v/deeplabcut-live) +[![License](https://img.shields.io/pypi/l/deeplabcutcore.svg)](https://github.com/DeepLabCut/deeplabcutlive/raw/master/LICENSE) +![PyPI - Downloads](https://img.shields.io/pypi/dm/deeplabcut-live?color=purple) +![Python package](https://github.com/DeepLabCut/DeepLabCut-live/workflows/Python%20package/badge.svg) -This package provides a `DLCLive` class which enables pose estimation online to provide feedback. This object loads and prepares a DeepLabCut network for inference, and will return the predicted pose for single images. +This package contains a DeepLabCut inference pipeline for real-time applications that has minimal (software) dependencies. Thus, it is as easy to install as possible (in particular, on atypical systems like NVIDIA Jetson boards). + +**Performance:** If you would like to see estimates on how your model might perform given a video size, neural network type, and hardware, please see: https://deeplabcut.github.io/DLC-inferencespeed-benchmark/ And, consider submitting your results too! https://github.com/DeepLabCut/DLC-inferencespeed-benchmark + +**What this SDK provides:** This package provides a `DLCLive` class which enables pose estimation online to provide feedback. This object loads and prepares a DeepLabCut network for inference, and will return the predicted pose for single images. To perform processing on poses (such as predicting the future pose of an animal given it's current pose, or to trigger external hardware like send TTL pulses to a laser for optogenetic stimulation), this object takes in a `Processor` object. Processor objects must contain two methods: process and save. @@ -10,13 +17,15 @@ To perform processing on poses (such as predicting the future pose of an animal - The `save` method saves any valuable data created by or used by the processor For examples, please see the [processor directory](processor) -###### Note :: alone, this object does not record video or capture images from a camera. This must be done separately, i.e. see our DeepLabCut-live GUI. +###### Note :: alone, this object does not record video or capture images from a camera. This must be done separately, i.e. see our [DeepLabCut-live GUI](https://github.com/gkane26/DeepLabCut-live-GUI). ### Installation: Please see our instruction manual to install on a [Windows or Linux machine](docs/install_desktop.md) or on a [NVIDIA Jetson Development Board](docs/install_jetson.md) +- available on pypi as: `pip install deeplabcut-live` + ### Quick Start: instructions for use: @@ -58,28 +67,45 @@ dlc_live.get_pose() DeepLabCut-live offers some analysis tools that allow users to peform the following operations on videos, from python or from the command line: 1. Test inference speed across a range of image sizes, downsizing images by specifying the `resize` or `pixels` parameter. Using the `pixels` parameter will resize images to the desired number of `pixels`, without changing the aspect ratio. Results will be saved (along with system info) to a pickle file if you specify an output directory. +##### python +```python +dlclive.benchmark_videos('/path/to/exported/model', ['/path/to/video1', '/path/to/video2'], output='/path/to/output', resize=[1.0, 0.75, '0.5']) +``` +##### command line ``` -# python -dlclive.analyze_videos('/path/to/exported/model', ['/path/to/video1', '/path/to/video2'], output='/path/to/output', resize=[1.0, 0.75, '0.5']) - -# command line dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 ``` -2. Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow): -``` -# python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[0.5], display=True, pcutoff=0.5, display_radius=4, cmap='bmy') +2. Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow and only for testing purposes): -# command line +##### python +```python +dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=[0.5], display=True, pcutoff=0.5, display_radius=4, cmap='bmy') +``` +##### command line +``` dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy ``` -3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video`. -``` -# python -dlclive.analyze_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, 0.5], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) +3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video` (note, this is slow and only for testing purposes). -# command line +##### python +```python +dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, 0.5], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) +``` +##### command line +``` dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save_poses --save_video ``` + +### Citation: + +If you find our code helpful, please consider citing: +``` +@Article{Kane2020dlclive, + author = {Kane, Gary and Lopes, Gonçalo and Sanders, Jonny and Mathis, Alexander and Mathis, Mackenzie}, + title = {Real-time DeepLabCut for closed-loop feedback based on posture}, + journal = {BioRxiv}, + year = {2020}, +} +``` From 84aa1a60287bf44ceb2daa914d397ea2e9117b4b Mon Sep 17 00:00:00 2001 From: gkane Date: Fri, 10 Jul 2020 17:34:12 -0400 Subject: [PATCH 26/28] dlclive: tfgpu flip x and y --- dlclive/dlclive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index e526b78..3059b70 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -356,7 +356,8 @@ def get_pose(self, frame=None, **kwargs): else: self.pose = argmax_pose_predict(scmap, locref, self.cfg['stride']) else: - self.pose = pose_output[0] + pose = np.array(pose_output[0]) + self.pose = pose[:, [1,0,2]] # display image if display=True before correcting pose for cropping/resizing From 768b8fef4856013e8b725ae23971dc30e08a3ceb Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Fri, 10 Jul 2020 18:30:54 -0400 Subject: [PATCH 27/28] name update --- dlclive/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 885e23b..5c19b20 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -9,4 +9,4 @@ from dlclive.dlclive import DLCLive from dlclive.processor import Processor from dlclive.bench import benchmark_model_by_size -from dlclive.analyze import analyze, analyze_videos +from dlclive.analyze import analyze, benchmark_videos From ab35d11ea6938f3e7f899ea109a00d41146f3a4a Mon Sep 17 00:00:00 2001 From: Gary Kane Date: Mon, 13 Jul 2020 14:27:44 -0400 Subject: [PATCH 28/28] complete convert analyze to benchmarking --- README.md | 10 +- dlclive/__init__.py | 2 +- dlclive/{analyze.py => benchmark.py} | 146 ++++++++++++++++----------- setup.py | 2 +- 4 files changed, 92 insertions(+), 68 deletions(-) rename dlclive/{analyze.py => benchmark.py} (81%) diff --git a/README.md b/README.md index aaee4b7..756f7bc 100644 --- a/README.md +++ b/README.md @@ -73,21 +73,21 @@ dlclive.benchmark_videos('/path/to/exported/model', ['/path/to/video1', '/path/t ``` ##### command line ``` -dlc-live-analyze /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 +dlc-live-benchmark /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 ``` 2. Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow and only for testing purposes): ##### python ```python -dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=[0.5], display=True, pcutoff=0.5, display_radius=4, cmap='bmy') +dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, pcutoff=0.5, display_radius=4, cmap='bmy') ``` ##### command line ``` -dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy +dlc-live-benchmark /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy ``` -3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.analyze_videos` and `deeplabcut.create_labeled_video` (note, this is slow and only for testing purposes). +3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.benchmark_videos` and `deeplabcut.create_labeled_video` (note, this is slow and only for testing purposes). ##### python ```python @@ -95,7 +95,7 @@ dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=[1. ``` ##### command line ``` -dlc-live-analyze /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save_poses --save_video +dlc-live-benchmark /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save-poses --save-video ``` ### Citation: diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 5c19b20..98096ac 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -9,4 +9,4 @@ from dlclive.dlclive import DLCLive from dlclive.processor import Processor from dlclive.bench import benchmark_model_by_size -from dlclive.analyze import analyze, benchmark_videos +from dlclive.benchmark import benchmark, benchmark_videos diff --git a/dlclive/analyze.py b/dlclive/benchmark.py similarity index 81% rename from dlclive/analyze.py rename to dlclive/benchmark.py index 3208c68..a18b8d1 100644 --- a/dlclive/analyze.py +++ b/dlclive/benchmark.py @@ -11,49 +11,69 @@ import time import sys import warnings -import argparse +import subprocess +import typing import pickle import colorcet as cc from PIL import ImageColor import ruamel import pandas as pd -import cpuinfo +try: + from pip._internal.operations import freeze +except ImportError: + from pip.operations import freeze + from tqdm import tqdm import numpy as np import tensorflow as tf import cv2 from dlclive import DLCLive +from dlclive import VERSION +from dlclive import __file__ as dlcfile -def get_system_info(): +def get_system_info() -> dict: """ Return summary info for system running benchmark - Returns ------- - str - name of machine - str - operating system - str - path to python (which conda/virtual environment) - str - device name + dict + Dictionary containing the following system information: + * ``host_name`` (str): name of machine + * ``op_sys`` (str): operating system + * ``python`` (str): path to python (which conda/virtual environment) + * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) + * ``freeze`` (list): list of installed packages and versions + * ``python_version`` (str): python version + * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit + * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` """ ### get os op_sys = platform.platform() - host_name = platform.node() + host_name = platform.node().replace(' ', '') + + # A string giving the absolute path of the executable binary for the Python interpreter, on systems where this makes sense. if platform.system() == 'Windows': host_python = sys.executable.split(os.path.sep)[-2] else: host_python = sys.executable.split(os.path.sep)[-3] - ### get device info (GPU or CPU) - + # try to get git hash if possible + dlc_basedir = os.path.dirname(os.path.dirname(dlcfile)) + git_hash = None + try: + git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=dlc_basedir) + git_hash = git_hash.decode('utf-8').rstrip('\n') + except subprocess.CalledProcessError: + # not installed from git repo, eg. pypi + # fine, pass quietly + pass + + # get device info (GPU or CPU) dev = None if tf.test.is_gpu_available(): gpu_name = tf.test.gpu_device_name() @@ -66,10 +86,19 @@ def get_system_info(): dev = [get_cpu_info()['brand']] dev_type = "CPU" - return host_name, op_sys, host_python, (dev_type, dev) - - -def analyze(model_path, + return { + 'host_name': host_name, + 'op_sys' : op_sys, + 'python': host_python, + 'device_type': dev_type, + 'device': dev, + 'freeze': list(freeze.freeze()), # pip freeze to get versions of all packages + 'python_version': sys.version, + 'git_hash': git_hash, + 'dlclive_version': VERSION + } + +def benchmark(model_path, video_path, tf_config=None, resize=None, @@ -82,7 +111,7 @@ def analyze(model_path, cmap='bmy', save_poses=False, save_video=False, - output=None): + output=None) -> typing.Tuple[np.ndarray, int, bool]: """ Analyze DeepLabCut-live exported model on a video: Calculate inference time, display keypoints, or @@ -113,7 +142,7 @@ def analyze(model_path, cmap : str, optional a string indicating the :package:`colorcet` colormap, `options here `, by default "bmy" save_poses : bool, optional - flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.analyze_videos`, by default False + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False save_video : bool, optional flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False output : str, optional @@ -129,16 +158,16 @@ def analyze(model_path, Example ------- Return a vector of inference times for 10000 frames: - dlclive.analyze('/my/exported/model', 'my_video.avi', n_frames=10000) + dlclive.benchmark('/my/exported/model', 'my_video.avi', n_frames=10000) Return a vector of inference times, resizing images to half the width and height for inference - dlclive.analyze('/my/exported/model', 'my_video.avi', n_frames=10000, resize=0.5) + dlclive.benchmark('/my/exported/model', 'my_video.avi', n_frames=10000, resize=0.5) Display keypoints to check the accuracy of an exported model - dlclive.analyze('/my/exported/model', 'my_video.avi', display=True) + dlclive.benchmark('/my/exported/model', 'my_video.avi', display=True) - Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.analyze_videos` and :function:`create_labeled_video` - dlclive.analyze('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` + dlclive.benchmark('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) """ ### load video @@ -248,15 +277,15 @@ def save_inf_times(sys_info, TFGPUinference, model=None, output=None): - """ Save inference time data collected using :function:`analyze` with system information to a pickle file. - This is primarily used through :function:`analyze_videos` + """ Save inference time data collected using :function:`benchmark` with system information to a pickle file. + This is primarily used through :function:`benchmark_videos` Parameters ---------- sys_info : tuple system information generated by :func:`get_system_info` inf_times : :class:`numpy.ndarray` - array of inference times generated by :func:`analyze` + array of inference times generated by :func:`benchmark` pixels : float or :class:`numpy.ndarray` number of pixels for each benchmark run. If an array, each index corresponds to a row in inf_times TFGPUinference: bool @@ -273,8 +302,6 @@ def save_inf_times(sys_info, """ output = output if output is not None else os.getcwd() - host_name, op_sys, host_python, dev = sys_info - host_name = host_name.replace(" ", "") model_type = None if model is not None: if 'resnet' in model: @@ -285,24 +312,19 @@ def save_inf_times(sys_info, model_type = None fn_ind = 0 - base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind) + base_name = f"benchmark_{sys_info['host_name']}_{sys_info['device'][0]}_{fn_ind}.pickle" while os.path.isfile(os.path.normpath(output + '/' + base_name)): fn_ind += 1 - base_name = "benchmark_{}_{}_{}.pickle".format(host_name, dev[0], fn_ind) - - fn = os.path.normpath(output) + base_name = f"benchmark_{sys_info['host_name']}_{sys_info['device'][0]}_{fn_ind}.pickle" - data = {'host_name' : host_name, - 'op_sys' : op_sys, - 'python' : host_python, - 'device_type' : dev[0], - 'device' : dev[1], - 'model' : model, + data = {'model' : model, 'model_type' : model_type, 'TFGPUinference' : TFGPUinference, 'pixels' : pixels, 'inference_times' : inf_times} + data.update(sys_info) + os.makedirs(os.path.normpath(output), exist_ok=True) pickle.dump(data, open(os.path.normpath(f"{output}/{base_name}"), 'wb')) @@ -328,7 +350,7 @@ def benchmark_videos(model_path, by specifying a resizing factor or the number of pixels to use in the image (keeping aspect ratio constant). Options to record inference times (to examine inference speed), display keypoints to visually check the accuracy, - or save poses to an hdf5 file as in :function:`deeplabcut.analyze_videos` and + or save poses to an hdf5 file as in :function:`deeplabcut.benchmark_videos` and create a labeled video as in :function:`deeplabcut.create_labeled_video`. Parameters @@ -358,7 +380,7 @@ def benchmark_videos(model_path, cmap : str, optional a string indicating the :package:`colorcet` colormap, `options here `, by default "bmy" save_poses : bool, optional - flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.analyze_videos`, by default False + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False save_video : bool, optional flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False @@ -374,7 +396,7 @@ def benchmark_videos(model_path, Display keypoints to check the accuracy of an exported model dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', display=True) - Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.analyze_videos` and :function:`create_labeled_video` + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) """ @@ -405,22 +427,22 @@ def benchmark_videos(model_path, for i in range(len(resize)): - print("\nRun {:d} / {:d}\n".format(i+1, len(resize))) - - inf_times[i], pixels_out[i], TFGPUinference = analyze(model_path, - v, - tf_config=tf_config, - resize=resize[i], - pixels=pixels[i], - n_frames=n_frames, - print_rate=print_rate, - display=display, - pcutoff=pcutoff, - display_radius=display_radius, - cmap=cmap, - save_poses=save_poses, - save_video=save_video, - output=output) + print(f"\nRun {i+1} / {len(resize)}\n") + + inf_times[i], pixels_out[i], TFGPUinference = benchmark(model_path, + v, + tf_config=tf_config, + resize=resize[i], + pixels=pixels[i], + n_frames=n_frames, + print_rate=print_rate, + display=display, + pcutoff=pcutoff, + display_radius=display_radius, + cmap=cmap, + save_poses=save_poses, + save_video=save_video, + output=output) ### save results @@ -433,11 +455,13 @@ def main(): """Provides a command line interface :function:`benchmark_videos` """ + import argparse + parser = argparse.ArgumentParser() parser.add_argument('model_path', type=str) parser.add_argument('video_path', type=str, nargs='+') parser.add_argument('-o', '--output', type=str, default=None) - parser.add_argument('-n', '--n-frames', type=int, default=10000) + parser.add_argument('-n', '--n-frames', type=int, default=1000) parser.add_argument('-r', '--resize', type=float, nargs='+') parser.add_argument('-p', '--pixels', type=float, nargs='+') parser.add_argument('-v', '--print-rate', default=False, action='store_true') diff --git a/setup.py b/setup.py index f8f5b04..dddf352 100644 --- a/setup.py +++ b/setup.py @@ -42,5 +42,5 @@ "Operating System :: OS Independent", ), entry_points = {'console_scripts' : ['dlc-live-bench=dlclive.bench:main', - 'dlc-live-analyze=dlclive.analyze:main']} + 'dlc-live-benchmark=dlclive.benchmark:main']} )