Merge pull request #716 from zachborboa/master

Attempt to stop active requests when MultiCurl::stop() is called

README.md

@@ -290,11 +290,12 @@ Curl::setRange($range)
 Curl::setReferer($referer)
 Curl::setReferrer($referrer)
 Curl::setRetry($mixed)
-Curl::setStop($callback)
+Curl::setStop($callback = null)
 Curl::setTimeout($seconds)
 Curl::setUrl($url, $mixed_data = '')
 Curl::setUserAgent($user_agent)
 Curl::setXmlDecoder($mixed)
+Curl::stop()
 Curl::success($callback)
 Curl::unsetHeader($key)
 Curl::unsetProxy()
@@ -349,6 +350,7 @@ MultiCurl::setRange($range)
 MultiCurl::setRateLimit($rate_limit)
 MultiCurl::setReferer($referer)
 MultiCurl::setReferrer($referrer)
+MultiCurl::setRequestTimeAccuracy()
 MultiCurl::setRetry($mixed)
 MultiCurl::setTimeout($seconds)
 MultiCurl::setUrl($url, $mixed_data = '')

examples/multi_curl_stop.php

@@ -0,0 +1,29 @@
+<?php
+require __DIR__ . '/../vendor/autoload.php';
+
+use Curl\MultiCurl;
+
+$multi_curl = new MultiCurl();
+
+// Count the number of completed requests.
+$request_count = 0;
+$multi_curl->complete(function ($instance) use (&$request_count) {
+    $request_count += 1;
+});
+
+$multi_curl->success(function ($instance) use (&$request_count, $multi_curl) {
+    echo 'call to "' . $instance->url . '" was successful.' . "\n";
+
+    // Stop pending requests and attempt to stop active requests after the first
+    // successful request.
+    $multi_curl->stop();
+});
+
+$multi_curl->addGet('https://httpbin.org/delay/4');
+$multi_curl->addGet('https://httpbin.org/delay/1');
+$multi_curl->addGet('https://httpbin.org/delay/3');
+$multi_curl->addGet('https://httpbin.org/delay/2');
+
+$multi_curl->start();
+
+assert($request_count === 1);

src/Curl/Curl.php

@@ -223,6 +223,7 @@ class Curl
         $this->jsonDecoderArgs = null;
         $this->xmlDecoder = null;
         $this->xmlDecoderArgs = null;
+        $this->headerCallbackData = null;
         $this->defaultDecoder = null;
     }
 
@@ -2067,6 +2068,7 @@ class Curl
         $header_callback_data->stopRequestDecider = null;
         $header_callback_data->stopRequest = false;
         $this->headerCallbackData = $header_callback_data;
+        $this->setStop();
         $this->setOpt(CURLOPT_HEADERFUNCTION, createHeaderCallback($header_callback_data));
 
         $this->setOpt(CURLOPT_RETURNTRANSFER, true);
@@ -2090,27 +2092,34 @@ class Curl
      * The callable must return a truthy value for the request to be stopped
      * early.
      *
+     * The callable may be set to null to avoid calling the stop request decider
+     * callback and instead just check the value of stopRequest for attempting
+     * to stop the request as used by Curl::stop().
+     *
      * @access public
-     * @param  $callback callable
+     * @param  $callback callable|null
      */
-    public function setStop($callback)
+    public function setStop($callback = null)
     {
         $this->headerCallbackData->stopRequestDecider = $callback;
         $this->headerCallbackData->stopRequest = false;
 
         $header_callback_data = $this->headerCallbackData;
-        $this->progress(function (
-            $resource,
-            $download_size,
-            $downloaded,
-            $upload_size,
-            $uploaded
-        ) use (
-            $header_callback_data
-        ) {
-            // Abort the transfer when the stop request flag has been set by returning a non-zero value.
-            return $header_callback_data->stopRequest ? 1 : 0;
-        });
+        $this->progress(createStopRequestFunction($header_callback_data));
+    }
+
+    /**
+     * Stop
+     *
+     * Attempt to stop request.
+     *
+     * Used by MultiCurl::stop() when making multiple parallel requests.
+     *
+     * @access public
+     */
+    public function stop()
+    {
+        $this->headerCallbackData->stopRequest = true;
     }
 }
 
@@ -2142,3 +2151,29 @@ function createHeaderCallback($header_callback_data) {
         return strlen($header);
     };
 }
+
+/**
+ * Create Stop Request Function
+ *
+ * Create a function for Curl::progress() that stops a request early when the
+ * stopRequest flag is on. Keep this function separate from the class to prevent
+ * a memory leak.
+ *
+ * @param  $header_callback_data
+ *
+ * @return callable
+ */
+function createStopRequestFunction($header_callback_data) {
+    return function (
+        $resource,
+        $download_size,
+        $downloaded,
+        $upload_size,
+        $uploaded
+    ) use (
+        $header_callback_data
+    ) {
+        // Abort the transfer when the stop request flag has been set by returning a non-zero value.
+        return $header_callback_data->stopRequest ? 1 : 0;
+    };
+}

src/Curl/MultiCurl.php

@@ -20,6 +20,7 @@ class MultiCurl
     private $currentRequestCount = 0;
     private $concurrency = 25;
     private $nextCurlId = 0;
+    private $preferRequestTimeAccuracy = false;
 
     private $rateLimit = null;
     private $rateLimitEnabled = false;
@@ -959,17 +960,40 @@ class MultiCurl
                 $this->waitUntilRequestQuotaAvailable();
             }
 
-            // Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.
-            // https://bugs.php.net/bug.php?id=63411
-            //
-            // Also, use a shorter curl_multi_select() timeout instead the default of one second. This allows pending
-            // requests to have more accurate start times. Without a shorter timeout, it can be nearly a full second
-            // before available request quota is rechecked and pending requests can be initialized.
-            if (curl_multi_select($this->multiCurl, 0.2) === -1) {
-                usleep(100000);
-            }
+            if ($this->preferRequestTimeAccuracy) {
+                // Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly
+                // block.
+                // https://bugs.php.net/bug.php?id=63411
+                //
+                // Also, use a shorter curl_multi_select() timeout instead the default of one second. This allows
+                // pending requests to have more accurate start times. Without a shorter timeout, it can be nearly a
+                // full second before available request quota is rechecked and pending requests can be initialized.
+                if (curl_multi_select($this->multiCurl, 0.2) === -1) {
+                    usleep(100000);
+                }
 
-            curl_multi_exec($this->multiCurl, $active);
+                curl_multi_exec($this->multiCurl, $active);
+            } else {
+                // Use multiple loops to get data off of the multi handler. Without this, the following error may appear
+                // intermittently on certain versions of PHP:
+                //   curl_multi_exec(): supplied resource is not a valid cURL handle resource
+
+                // Clear out the curl buffer.
+                do {
+                    $status = curl_multi_exec($this->multiCurl, $active);
+                } while ($status === CURLM_CALL_MULTI_PERFORM);
+
+                // Wait for more information and then get that information.
+                while ($active && $status === CURLM_OK) {
+                    // Check if the network socket has some data.
+                    if (curl_multi_select($this->multiCurl) !== -1) {
+                        // Process the data for as long as the system tells us to keep getting it.
+                        do {
+                            $status = curl_multi_exec($this->multiCurl, $active);
+                        } while ($status === CURLM_CALL_MULTI_PERFORM);
+                    }
+                }
+            }
 
             while (($info_array = curl_multi_info_read($this->multiCurl)) !== false) {
                 if ($info_array['msg'] === CURLMSG_DONE) {
@@ -1022,10 +1046,17 @@ class MultiCurl
      */
     public function stop()
     {
+        // Remove any queued curl requests.
         while (count($this->queuedCurls)) {
             $curl = array_pop($this->queuedCurls);
             $curl->close();
         }
+
+        // Attempt to stop active curl requests.
+        while (count($this->activeCurls)) {
+            $curl = array_pop($this->activeCurls);
+            $curl->stop();
+        }
     }
 
     /**
@@ -1134,6 +1165,16 @@ class MultiCurl
         $this->setOpt(CURLOPT_MAXREDIRS, $maximum_redirects);
     }
 
+    /**
+     * Set request time accuracy
+     *
+     * @access public
+     */
+    public function setRequestTimeAccuracy()
+    {
+        $this->preferRequestTimeAccuracy = true;
+    }
+
     /**
      * Destruct
      *

tests/PHPCurlClass/PHPCurlClassTest.php

@@ -3256,7 +3256,12 @@ class CurlTest extends \PHPUnit\Framework\TestCase
             }
             echo '{"before":' . memory_get_usage() . ',';
             $curl = new Curl();
+
+            // Unset the $curl object instead of calling $curl->close(). Calling
+            // unset($curl) should trigger the clean up: __destruct() which
+            // calls $curl->close().
             unset($curl);
+
             echo '"after":' . memory_get_usage() . '}';
             sleep(1);
         }

tests/PHPCurlClass/PHPMultiCurlClassTest.php

@@ -3745,6 +3745,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -3818,6 +3819,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -3889,6 +3891,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -3960,6 +3963,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4032,6 +4036,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4104,6 +4109,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4176,6 +4182,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4256,6 +4263,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4336,6 +4344,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('2/5s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4408,6 +4417,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setRateLimit('1/1s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
             $request_stats[$instance->id] = [];
@@ -4446,6 +4456,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('5/30s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4480,6 +4491,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('1/1m');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4514,6 +4526,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('3/1m');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4546,6 +4559,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('3/65s');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4578,6 +4592,7 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $request_stats = [];
 
         $multi_curl = new MultiCurl();
+        $multi_curl->setRequestTimeAccuracy();
         $multi_curl->setHeader('X-DEBUG-TEST', 'timeout');
         $multi_curl->setRateLimit('10/2m');
         $multi_curl->beforeSend(function ($instance) use (&$request_stats) {
@@ -4942,7 +4957,25 @@ class MultiCurlTest extends \PHPUnit\Framework\TestCase
         $multi_curl->start();
     }
 
-    public function testCurlStop()
+    public function testCurlStopActiveConcurrencyDefault()
+    {
+        $multi_curl = new MultiCurl();
+        $request_count = 0;
+        $multi_curl->complete(function ($instance) use (&$request_count, $multi_curl) {
+            $request_count += 1;
+            $multi_curl->stop();
+        });
+
+        $multi_curl->addGet(Test::TEST_URL);
+        $multi_curl->addGet(Test::TEST_URL);
+        $multi_curl->addGet(Test::TEST_URL);
+
+        $multi_curl->start();
+
+        $this->assertEquals(1, $request_count);
+    }
+
+    public function testCurlStopActiveConcurrencyOne()
     {
         $multi_curl = new MultiCurl();
         $multi_curl->setConcurrency(1);