geckon01/php-curl-class
1
0
Fork 0
mirror of https://github.com/php-curl-class/php-curl-class.git synced 2026-08-30 12:14:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement rate limiting requests via MultiCurl::setRateLimit()

Browse Source
This commit is contained in:
Zach Borboa
2020-05-16 22:01:15 -07:00
parent 9c00119ab5
commit 976008d094
8 changed files with 1333 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -333,6 +333,7 @@ MultiCurl::setProxyAuth($auth)
MultiCurl::setProxyTunnel($tunnel = true)
MultiCurl::setProxyType($type)
MultiCurl::setRange($range)
MultiCurl::setRateLimit($rate_limit)
MultiCurl::setReferer($referer)
MultiCurl::setReferrer($referrer)
MultiCurl::setRetry($mixed)
src/Curl/MultiCurl.php
+138 -21
View File
@@ -15,6 +15,16 @@ class MultiCurl
private $concurrency = 25;
private $nextCurlId = 0;
private $rateLimit = null;
private $rateLimitEnabled = false;
private $rateLimitReached = false;
private $maxRequests = null;
private $interval = null;
private $intervalSeconds = null;
private $unit = null;
private $currentStartTime = null;
private $currentRequestCount = 0;
private $beforeSendCallback = null;
private $successCallback = null;
private $errorCallback = null;
@@ -713,6 +723,57 @@ class MultiCurl
$this->setOpt(CURLOPT_RANGE, $range);
}
/**
* Set Rate Limit
*
* @access public
* @param $rate_limit string (e.g. "60/1m").
*/
public function setRateLimit($rate_limit)
{
$rate_limit_pattern =
'/' . // delimiter
'^' . // assert start
'(\d+)' . // digit(s)
'\/' . // slash
'(\d+)?' . // digit(s), optional
'(s|m|h)' . // unit, s for seconds, m for minutes, h for hours
'$' . // assert end
'/' . // delimiter
'i' . // case-insensitive matches
'';
if (!preg_match($rate_limit_pattern, $rate_limit, $matches)) {
throw new \UnexpectedValueException(
'rate limit must be formatted as $max_requests/$interval(s|m|h) ' .
'(e.g. "60/1m" for a maximum of 60 requests per 1 minute)'
);
}
$max_requests = (int)$matches['1'];
if ($matches['2'] === '') {
$interval = 1;
} else {
$interval = (int)$matches['2'];
}
$unit = strtolower($matches['3']);
// Convert interval to seconds based on unit.
if ($unit === 's') {
$interval_seconds = $interval * 1;
} elseif ($unit === 'm') {
$interval_seconds = $interval * 60;
} elseif ($unit === 'h') {
$interval_seconds = $interval * 3600;
}
$this->rateLimit = $max_requests . '/' . $interval . $unit;
$this->rateLimitEnabled = true;
$this->maxRequests = $max_requests;
$this->interval = $interval;
$this->intervalSeconds = $interval_seconds;
$this->unit = $unit;
}
/**
* Set Referer
*
@@ -823,17 +884,21 @@ class MultiCurl
}
$this->isStarted = true;
$concurrency = $this->concurrency;
if ($concurrency > count($this->curls)) {
$concurrency = count($this->curls);
}
for ($i = 0; $i < $concurrency; $i++) {
$this->initHandle(array_shift($this->curls));
}
$this->currentStartTime = microtime(true);
$this->currentRequestCount = 0;
do {
while (count($this->curls) &&
count($this->activeCurls) < $this->concurrency &&
(!$this->rateLimitEnabled || $this->hasRequestQuota())
) {
$this->initHandle();
}
if ($this->rateLimitEnabled && !count($this->activeCurls) && !$this->hasRequestQuota()) {
$this->waitUntilRequestQuotaAvailable();
}
// Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.
// https://bugs.php.net/bug.php?id=63411
if (curl_multi_select($this->multiCurl) === -1) {
@@ -842,7 +907,7 @@ class MultiCurl
curl_multi_exec($this->multiCurl, $active);
while (!($info_array = curl_multi_info_read($this->multiCurl)) === false) {
while (($info_array = curl_multi_info_read($this->multiCurl)) !== false) {
if ($info_array['msg'] === CURLMSG_DONE) {
foreach ($this->activeCurls as $key => $curl) {
if ($curl->curl === $info_array['handle']) {
@@ -868,10 +933,7 @@ class MultiCurl
// Remove completed instance from active curls.
unset($this->activeCurls[$key]);
// Start new requests before removing the handle of the completed one.
while (count($this->curls) >= 1 && count($this->activeCurls) < $this->concurrency) {
$this->initHandle(array_shift($this->curls));
}
// Remove handle of the completed instance.
curl_multi_remove_handle($this->multiCurl, $curl->curl);
// Clean up completed instance.
@@ -883,11 +945,7 @@ class MultiCurl
}
}
}
if (!$active) {
$active = count($this->activeCurls);
}
} while ($active > 0);
} while ($active || count($this->activeCurls) || count($this->curls));
$this->isStarted = false;
}
@@ -993,8 +1051,17 @@ class MultiCurl
* @param $curl
* @throws \ErrorException
*/
private function initHandle($curl)
private function initHandle()
{
$curl = array_shift($this->curls);
if ($curl === null) {
return;
}
// Add instance to list of active curls.
$this->currentRequestCount += 1;
$this->activeCurls[$curl->id] = $curl;
// Set callbacks if not already individually set.
if ($curl->beforeSendCallback === null) {
$curl->beforeSend($this->beforeSendCallback);
@@ -1033,7 +1100,57 @@ class MultiCurl
throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code));
}
$this->activeCurls[$curl->id] = $curl;
$curl->call($curl->beforeSendCallback);
}
/**
* Has Request Quota
*
* Checks if there is any available quota to make additional requests while
* rate limiting is enabled.
*
* @access private
*/
private function hasRequestQuota()
{
// Calculate if there's request quota since ratelimiting is enabled.
if ($this->rateLimitEnabled) {
// Determine if the limit of requests per interval has been reached.
if ($this->currentRequestCount >= $this->maxRequests) {
$elapsed_seconds = microtime(true) - $this->currentStartTime;
if ($elapsed_seconds <= $this->intervalSeconds) {
$this->rateLimitReached = true;
return false;
} elseif ($this->rateLimitReached) {
$this->rateLimitReached = false;
$this->currentStartTime = microtime(true);
$this->currentRequestCount = 0;
}
}
return true;
} else {
return true;
}
}
/**
* Wait Until Request Quota Available
*
* Waits until there is available request quota available based on the rate limit.
*
* @access private
*/
private function waitUntilRequestQuotaAvailable()
{
$sleep_until = $this->currentStartTime + $this->intervalSeconds;
$sleep_until_relative = $sleep_until - $this->currentStartTime;
$sleep_seconds = $sleep_until - microtime(true);
// Avoid using time_sleep_until() as it appears to be less precise and not sleep long enough.
usleep($sleep_seconds * 1000000);
$this->currentStartTime = microtime(true);
$this->currentRequestCount = 0;
}
}
tests/PHPCurlClass/Helper.php
+18
View File
@@ -50,6 +50,16 @@ class Test
$this->chainedRequest($first, $data);
$this->chainedRequest($second, $data);
}
public static function getTestUrl($port)
{
if (getenv('PHP_CURL_CLASS_LOCAL_TEST') === 'yes' ||
in_array(getenv('TRAVIS_PHP_VERSION'), array('7.0', '7.1', '7.2', '7.3', '7.4', 'nightly'))) {
return 'http://127.0.0.1:' . $port . '/';
} else {
return self::TEST_URL;
}
}
}
function create_png()
@@ -127,3 +137,11 @@ function remove_file_from_server($uploaded_file_path) {
)));
assert(file_exists($uploaded_file_path) === false);
}
function get_multi_curl_property_value($instance, $property_name)
{
$reflector = new \ReflectionClass('\Curl\MultiCurl');
$property = $reflector->getProperty($property_name);
$property->setAccessible(true);
return $property->getValue($instance);
}
tests/PHPCurlClass/PHPMultiCurlClassTest.php
+1134
View File
File diff suppressed because it is too large Load Diff
tests/PHPCurlClass/server.php
+7 -1
View File
@@ -1,4 +1,6 @@
<?php
$server_start = microtime(true);
// Prevent direct access unless testing.
if (getenv('PHP_CURL_CLASS_TEST_MODE_ENABLED') !== 'yes' &&
@$_SERVER['PHP_CURL_CLASS_TEST_MODE_ENABLED'] !== 'yes') {
@@ -291,8 +293,12 @@ if ($test === 'http_basic_auth') {
break;
}
}
echo '",' . "\n";
echo ' "elapsed_seconds": "' . $elapsed . '"' . "\n";
echo ' "elapsed_seconds": "' . $elapsed . '",' . "\n";
echo ' "server_port": "' . ((int)$_SERVER['SERVER_PORT']) . '",' . "\n";
echo ' "server_start": "' . $server_start . '",' . "\n";
echo ' "server_stop": "' . microtime(true) . '"' . "\n";
echo '}' . "\n";
exit;
} elseif ($test === 'error_message') {
tests/before_script.sh
+14 -6
View File
@@ -85,6 +85,14 @@ phpunit_v7_5_shim() {
remove_expectWarning
}
start_php_servers() {
for i in $(seq 0 6); do
port=8000
(( port += $i ))
php -S "127.0.0.1:${port}" -t tests/PHPCurlClass/ &
done
}
set -x
echo "TRAVIS_PHP_VERSION: ${TRAVIS_PHP_VERSION}"
php -r "var_dump(phpversion());"
@@ -160,16 +168,16 @@ elif [[ "${TRAVIS_PHP_VERSION}" == "5.6" ]]; then
phpunit_shim
elif [[ "${TRAVIS_PHP_VERSION}" == "7.0" ]]; then
phpunit_v6_5_shim
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
elif [[ "${TRAVIS_PHP_VERSION}" == "7.1" ]]; then
phpunit_v7_5_shim
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
elif [[ "${TRAVIS_PHP_VERSION}" == "7.2" ]]; then
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
elif [[ "${TRAVIS_PHP_VERSION}" == "7.3" ]]; then
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
elif [[ "${TRAVIS_PHP_VERSION}" == "7.4" ]]; then
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
elif [[ "${TRAVIS_PHP_VERSION}" == "hhvm" || "${TRAVIS_PHP_VERSION}" == "hhvm-nightly" ]]; then
curl "https://nginx.org/keys/nginx_signing.key" | sudo apt-key add -
echo "deb https://nginx.org/packages/mainline/ubuntu/ trusty nginx" | sudo tee -a /etc/apt/sources.list
@@ -210,5 +218,5 @@ EOF
composer require phpunit/phpunit:5.7.*
fi
elif [[ "${TRAVIS_PHP_VERSION}" == "nightly" ]]; then
php -S 127.0.0.1:8000 -t tests/PHPCurlClass/ &
start_php_servers
fi
tests/run.sh
+21 -4
View File
@@ -6,9 +6,22 @@ set -x
# Let test server know we should allow testing.
export PHP_CURL_CLASS_TEST_MODE_ENABLED="yes"
# Start test server.
php -S 127.0.0.1:8000 -t PHPCurlClass/ &> /dev/null &
pid="${!}"
# Let test server know this is a local test.
export PHP_CURL_CLASS_LOCAL_TEST="yes"
# Start test servers. Run servers on different ports to allow simultaneous
# requests without blocking.
server_count=7
declare -A pids
for i in $(seq 0 $(("${server_count}" - 1))); do
port=8000
(( port += $i ))
php -S "127.0.0.1:${port}" -t PHPCurlClass/ &> /dev/null &
pid="${!}"
pids["${i}"]="${pid}"
done
# Determine which phpunit to use.
if [[ -f "../vendor/phpunit/phpunit/phpunit" ]]; then
@@ -24,4 +37,8 @@ extra_args="${@}"
--debug \
--verbose \
${extra_args}
kill "${pid}"
# Stop test servers.
for pid in "${pids[@]}"; do
kill "${pid}"
done
tests/start_servers.sh
-24
View File
@@ -1,24 +0,0 @@
screen_name="my_screen"
server_count=5
# screen_binary="screen"
screen_binary="byobu-screen"
# Start screen in detached mode with a session name.
screen -S "${screen_name}" -t "master" -d -m
# Wait for screen to be ready before opening new sessions.
sleep 1
# Create tabs and send commands to each.
for i in $(seq 1 "${server_count}"); do
# Create tab.
screen -S "${screen_name}" -X "screen" -t "my_screen_${i}"
# Start development server in tab.
port=8000
(( port += $i ))
command="php -S 127.0.0.1:${port} -t PHPCurlClass/"
screen -S "${screen_name}" -p "my_screen_${i}" -X stuff "${command}"$'\n'
done
screen -r "${screen_name}"