Updates to benchmarking (#29)

* remove old bench.py

* Add dynamic cropping to benchmark (#28)

* add dynamic cropping to benchmark

* add cropping to benchmark

* dlclive: make sure dynamic cropping indices are integers

* Update README.md

* dowloading data, streamlined process

* typo fixed

Co-authored-by: Mackenzie Mathis <mathis@rowland.harvard.edu>
Co-authored-by: alex <alexander@deeplabcut.org>

Author: 3 people

.gitignore

@@ -1,5 +1,9 @@
 # DeepLabCut-live
 
+# Data related to benchmark!
+benchmarking/Data*
+benchmarking/results*
+
 *test*
 **DS_Store*
 *vscode*

README.md

@@ -1,4 +1,4 @@
-# DeepLabCut-live SDK<img src="https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1572296495650-Y4ZTJ2XP2Z9XF1AD74VW/ke17ZwdGBToddI8pDm48kMulEJPOrz9Y8HeI7oJuXxR7gQa3H78H3Y0txjaiv_0fDoOvxcdMmMKkDsyUqMSsMWxHk725yiiHCCLfrh8O1z5QPOohDIaIeljMHgDF5CVlOqpeNLcJ80NK65_fV7S1UZiU3J6AN9rgO1lHw9nGbkYQrCLTag1XBHRgOrY8YAdXW07ycm2Trb21kYhaLJjddA/DLC_logo_blk-01.png?format=1000w" width="350" title="DLC-live" alt="DLC LIVE!" align="right" vspace = "50">
+# DeepLabCut-live! SDK<img src="https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1606082050387-M8M1CFI5DFUZCBAAUI0W/ke17ZwdGBToddI8pDm48kLuMKy7Ws6mFofiFehYynfdZw-zPPgdn4jUwVcJE1ZvWQUxwkmyExglNqGp0IvTJZUJFbgE-7XRK3dMEBRBhUpzp2tFVMcEgqZM8QO7VXXQogrsLnYKC4n4YnYuHC1HMRWygQlqMNAoTF9HaycikLeg/DLClive.png?format=750w" width="350" title="DLC-live" alt="DLC LIVE!" align="right" vspace = "50">
 
 <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
 ![PyPI - Python Version](https://img.shields.io/pypi/v/deeplabcut-live)
@@ -40,6 +40,8 @@ Note, you can then test your installation by running:
 
 If installed properly, this script will i) create a temporary folder ii) download the full_dog model from the [DeepLabCut Model Zoo](http://www.mousemotorlab.org/dlc-modelzoo), iii) download a short video clip of a dog, and iv) run inference while displaying keypoints. v) remove the temporary folder.
 
+<img src="https://images.squarespace-cdn.com/content/v1/57f6d51c9f74566f55ecf271/1606081086014-TG9GWH63ZGGOO7K779G3/ke17ZwdGBToddI8pDm48kHiSoSToKfKUI9t99vKErWoUqsxRUqqbr1mOJYKfIPR7LoDQ9mXPOjoJoqy81S2I8N_N4V1vUb5AoIIIbLZhVYxCRW4BPu10St3TBAUQYVKcOoIGycwr1shdgJWzLuxyzjLbSRGBFFxjYMBr42yCvRK5HHsLZWtMlAHzDU294nCd/dlclivetest.png?format=1000w" width="650" title="DLC-live-test" alt="DLC LIVE TEST" align="center" vspace = "50">
+
 ### Quick Start: instructions for use:
 
 1. Initialize `Processor` (if desired)
@@ -76,7 +78,7 @@ dlc_live.get_pose(<your image>)
   - `<your image>` = is a numpy array of each frame
 
 
-### Benchmarking/Analyzing Exported DeepLabCut Models
+### Benchmarking/Analyzing your exported DeepLabCut models
 
 DeepLabCut-live offers some analysis tools that allow users to peform the following operations on videos, from python or from the command line:
 

benchmarking/run.py

@@ -10,7 +10,7 @@
 import os
 import glob
 
-from dlclive import benchmark_model_by_size
+from dlclive import benchmark_videos
 
 # Update the datafolder to where the data is:
 datafolder = "/your/path/to/data/here"
@@ -27,21 +27,15 @@
 # storing results in /benchmarking/results: (for your PR)
 out_dir = os.path.normpath(this_dir + "/results")
 
+if not os.path.isdir(out_dir):
+    os.path.mkdir(out_dir)
+
 for ind_m, m in enumerate(dog_models):
     print("\n\nMODEL {:d} / 8".format(ind_m))
-    benchmark_model_by_size(
-        m, dog_video, ind_m, out_dir=out_dir, n_frames=n_frames, pixels=pixels
-    )
+    benchmark_videos(m, dog_video, output=out_dir, n_frames=n_frames, pixels=pixels)
 
 offset = ind_m + 1
 
 for ind_m, m in enumerate(mouse_models):
     print("\n\nMODEL {:d} / 8".format(ind_m))
-    benchmark_model_by_size(
-        m,
-        mouse_video,
-        ind_m + offset,
-        out_dir=out_dir,
-        n_frames=n_frames,
-        pixels=pixels,
-    )
+    benchmark_videos(m, mouse_video, output=out_dir, n_frames=n_frames, pixels=pixels)

benchmarking/run_dlclive_benchmark.py

@@ -0,0 +1,43 @@
+"""
+DeepLabCut Toolbox (deeplabcut.org)
+© A. & M. Mathis Labs
+
+Licensed under GNU Lesser General Public License v3.0
+"""
+
+# Script for running the official benchmark from Kane et al, 2020.
+# Please share your results at https://github.com/DeepLabCut/DLC-inferencespeed-benchmark
+
+import os, pathlib
+import glob
+
+from dlclive import benchmark_videos, download_benchmarking_data
+
+datafolder = os.path.join(
+    pathlib.Path(__file__).parent.absolute(), "Data-DLC-live-benchmark"
+)
+
+if not os.path.isdir(datafolder):  # only download if data doesn't exist!
+    # Downloading data.... this takes a while (see terminal)
+    download_benchmarking_data(datafolder)
+
+n_frames = 10000  # change to 10000 for testing on a GPU!
+pixels = [2500, 10000, 40000, 160000, 320000, 640000]
+
+dog_models = glob.glob(datafolder + "/dog/*[!avi]")
+dog_video = glob.glob(datafolder + "/dog/*.avi")[0]
+mouse_models = glob.glob(datafolder + "/mouse_lick/*[!avi]")
+mouse_video = glob.glob(datafolder + "/mouse_lick/*.avi")[0]
+
+this_dir = os.path.dirname(os.path.realpath(__file__))
+# storing results in /benchmarking/results: (for your PR)
+out_dir = os.path.normpath(this_dir + "/results")
+
+if not os.path.isdir(out_dir):
+    os.mkdir(out_dir)
+
+for m in dog_models:
+    benchmark_videos(m, dog_video, output=out_dir, n_frames=n_frames, pixels=pixels)
+
+for m in mouse_models:
+    benchmark_videos(m, mouse_video, output=out_dir, n_frames=n_frames, pixels=pixels)

dlclive/__init__.py

@@ -8,5 +8,4 @@
 from dlclive.version import __version__, VERSION
 from dlclive.dlclive import DLCLive
 from dlclive.processor import Processor
-from dlclive.bench import benchmark_model_by_size
-from dlclive.benchmark import benchmark, benchmark_videos
+from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data