From 144194730c0b458d1ecc86e2b09dadf99742a035 Mon Sep 17 00:00:00 2001 From: Zach Borboa Date: Sat, 1 Jul 2023 06:48:58 -0700 Subject: [PATCH] Add tqdm and curses examples for displaying progress of MultiCurl downloads --- examples/multi_curl_progress_advanced.php | 137 ++++++++++++++++++ ...lti_curl_progress_advanced_watch_curses.py | 97 +++++++++++++ ...multi_curl_progress_advanced_watch_tqdm.py | 84 +++++++++++ 3 files changed, 318 insertions(+) create mode 100644 examples/multi_curl_progress_advanced.php create mode 100644 examples/multi_curl_progress_advanced_watch_curses.py create mode 100644 examples/multi_curl_progress_advanced_watch_tqdm.py diff --git a/examples/multi_curl_progress_advanced.php b/examples/multi_curl_progress_advanced.php new file mode 100644 index 0000000..663e29a --- /dev/null +++ b/examples/multi_curl_progress_advanced.php @@ -0,0 +1,137 @@ + ] +// 23% [========> ] +// 15% [=====> ] +// +// Note: The server needs to send a content-length header for progress updates to work. + +use Curl\MultiCurl; + +// Keep track of download progress for each of the downloads. +$download_status = []; + +// Keep track of when the screen was last updated so it not updated too frequently. +$last_updated_time = 0; + +$multi_curl = new MultiCurl(); + +$urls_to_download = [ + 'https://www.php.net/distributions/manual/php_manual_en.html.gz', + 'https://www.php.net/distributions/manual/php_manual_en.tar.gz', + 'https://www.php.net/distributions/manual/php_manual_en.chm', +]; + +$i = 0; +foreach ($urls_to_download as $url) { + $filename = basename($url); + echo 'will be downloading ' . $url . ' and saving as "' . $filename . '"' . "\n"; + + $download_status[$i] = [ + 'position' => $i, + 'complete' => false, + 'filename' => $filename, + 'size' => 0, + 'downloaded' => 0, + ]; + + $curl = $multi_curl->addDownload($url, $filename); + + // Increase timeout to avoid error: + // "Operation timed out after 30000 milliseconds with ... out of ... bytes + // received". + $curl->setTimeout(500); + + // Slow the download. Comment the following lines to remove the download + // throttling. + $curl->setOpt(CURLOPT_MAX_RECV_SPEED_LARGE, 500000); + $curl->setOpt(CURLOPT_BUFFERSIZE, 1024); + + $curl->progress(function ( + $client, + $download_size, + $downloaded, + $upload_size, + $uploaded + ) use ( + $i, + &$download_status, + &$last_updated_time + ) { + if ($download_size === 0) { + return 0; + } + + $download_completed = $downloaded === $download_size; + $current_time = time(); + + // Avoid sending an update if we're within the same second and the + // download has not yet completed. + if (!$download_completed && $current_time === $last_updated_time) { + return 0; + } + + $last_updated_time = $current_time; + + // Update progress of this download. + $download_status[$i]['complete'] = $download_completed; + $download_status[$i]['size'] = $download_size; + $download_status[$i]['downloaded'] = $downloaded; + + // Generate response including completion status of all downloads and + // status of each individual download. + $response = [ + 'status' => '', + 'downloads' => [], + ]; + $all_downloads_completed = true; + foreach ($download_status as $key => $value) { + $response['downloads'][] = $value; + $all_downloads_completed = $all_downloads_completed && $value['complete']; + } + $response['status'] = $all_downloads_completed ? 'done' : 'active'; + $json_response = json_encode($response); + + $out = fopen('/tmp/myfifo', 'w'); + + // TODO: Catch broken pipe: + // PHP Notice: fwrite(): Write of 52 bytes failed with errno=32 + // Broken pipe in ./multi_curl_progress_advanced.php on line [...] + fwrite($out, $json_response . "\n"); + + fclose($out); + + // Comment the following line to hide the download progress updates + // being sent to the named pipe. + echo $json_response . "\n"; + + return 0; + }); + + $i += 1; +} + +echo 'starting download' . "\n"; +$multi_curl->start(); + +echo 'all done' . "\n"; diff --git a/examples/multi_curl_progress_advanced_watch_curses.py b/examples/multi_curl_progress_advanced_watch_curses.py new file mode 100644 index 0000000..59b8adb --- /dev/null +++ b/examples/multi_curl_progress_advanced_watch_curses.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +# Reads a named pipe file and displays progress bars using curses. +# +# See multi_curl_progress_advanced.php +# +# $ ipython multi_curl_progress_advanced_watch_curses.py +# 56% [=====================> ] +# 23% [========> ] +# 15% [=====> ] + +import curses +import json +import os + + +FIFO = "/tmp/myfifo" + + +def main(stdscr): + # Create named pipe file if it doesn't exist. + if not os.path.exists(FIFO): + os.mkfifo(FIFO) + + curses.noecho() + curses.cbreak() + + try: + while True: + display_progress_bars(stdscr) + except KeyboardInterrupt: + # Handle Control-C pressed. + pass + finally: + curses.echo() + curses.nocbreak() + curses.endwin() + + # Return exit code 0. + return 0 + + +def display_progress_bars(stdscr): + display_notice = True + + while True: + if display_notice: + display_notice = False + stdscr.clear() + stdscr.addstr(0, 0, "waiting for input") + stdscr.refresh() + + # Read named pipe file. + with open(FIFO) as f: + for line in f: + response = json.loads(line) + + # Update progress for each of the files being downloaded. + for entry in response.get("downloads", []): + # Display a progress bar: xxx% [=======> ] + progress_size = 40 + try: + fraction_downloaded = entry["downloaded"] / entry["size"] + except ZeroDivisionError: + fraction_downloaded = 0 + dots = round(fraction_downloaded * progress_size) + task_progress = "%3.0f%% [" % (fraction_downloaded * 100) + + i = 0 + while i < dots - 1: + task_progress += "=" + i += 1 + + task_progress += ">" + + while i < progress_size - 1: + task_progress += " " + i += 1 + + task_progress += "]" + + stdscr.addstr(entry["position"], 0, task_progress) + + # Refresh display of the progress bars. + stdscr.refresh() + + # Exit only after progress bars have been updated to end with + # each displaying 100%. + if response.get("status", "") == "done": + return + + +if __name__ == "__main__": + # Avoid getting the terminal in an unmanagable state by using the curses + # wrapper. The wrapper restores the terminal to its previous state even when + # there is an uncaught exception. + curses.wrapper(main) diff --git a/examples/multi_curl_progress_advanced_watch_tqdm.py b/examples/multi_curl_progress_advanced_watch_tqdm.py new file mode 100644 index 0000000..c3bc814 --- /dev/null +++ b/examples/multi_curl_progress_advanced_watch_tqdm.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# Reads a named pipe file and displays progress bars using tqdm. +# +# See multi_curl_progress_advanced.php +# +# $ ipython multi_curl_progress_advanced_watch_tqdm.py +# php_manual_en.html.gz: 56%|████████████████████████ | 2.99M/5.34M [00:02<00:02, 1.14MB/s] +# php_manual_en.tar.gz: 23%|██████████▎ | 2.37M/10.4M [00:02<00:08, 901kB/s] +# php_manual_en.chm: 15%|███████▏ | 2.06M/13.7M [00:02<00:14, 783kB/s] + +import json +import os +import sys + +from tqdm import tqdm + + +FIFO = "/tmp/myfifo" + + +def main(): + # Create named pipe file if it doesn't exist. + if not os.path.exists(FIFO): + os.mkfifo(FIFO) + + try: + while True: + display_progress_bars() + except KeyboardInterrupt: + # Handle Control-C pressed. + pass + + # Return exit code 0. + return 0 + + +def display_progress_bars(): + display_notice = True + progress_bars = {} + + while True: + if display_notice: + display_notice = False + print("waiting for input") + + # Read named pipe file. + with open(FIFO) as f: + for line in f: + response = json.loads(line) + + # Update progress for each of the files being downloaded. + for entry in response.get("downloads", []): + # Get or create progress bar. + progress_bar = progress_bars.get(entry["position"]) + if progress_bar is None: + progress_bar = tqdm( + desc=entry["filename"], + total=entry["size"], + position=entry["position"], + unit_scale=True, + unit_divisor=1000, + unit="B", + ) + progress_bars[entry["position"]] = progress_bar + + # Update download progress. + progress_bar.n = entry["downloaded"] + + # Update progress bar's total size. Initial size may have + # been sent as 0 until content-length was determined. + progress_bar.total = entry["size"] + + # Refresh display of the progress bar. + progress_bar.refresh() + + # Exit only after progress bars have been updated to end with + # each displaying 100%. + if response.get("status", "") == "done": + return + + +if __name__ == "__main__": + sys.exit(main())