idanoo/php-resque
1
0
Fork 0
mirror of https://github.com/idanoo/php-resque.git synced 2024-11-22 08:15:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #63 from ruudk/blocking-list-pop

Browse Source 
Basic support for blocking list pop
This commit is contained in:
Chris Boulton 2013-06-12 01:09:11 -07:00
commit 59f617e639
16 changed files with 704 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/resque Normal file → Executable file
View File

@ -59,6 +59,8 @@ if($APP_INCLUDE) {
require_once $APP_INCLUDE; require_once $APP_INCLUDE;
} }
$BLOCKING = getenv('BLOCKING') !== FALSE;
$interval = 5; $interval = 5;
$INTERVAL = getenv('INTERVAL'); $INTERVAL = getenv('INTERVAL');
if(!empty($INTERVAL)) { if(!empty($INTERVAL)) {
@ -89,7 +91,7 @@ if($count > 1) {
$worker = new Resque_Worker($queues); $worker = new Resque_Worker($queues);
$worker->logLevel = $logLevel; $worker->logLevel = $logLevel;
fwrite(STDOUT, '*** Starting worker '.$worker."\n"); fwrite(STDOUT, '*** Starting worker '.$worker."\n");
$worker->work($interval); $worker->work($interval, $BLOCKING);
break; break;
} }
} }
@ -107,6 +109,6 @@ else {
} }
fwrite(STDOUT, '*** Starting worker '.$worker."\n"); fwrite(STDOUT, '*** Starting worker '.$worker."\n");
$worker->work($interval); $worker->work($interval, $BLOCKING);
} }
?> ?>

2
composer.json
View File

@ -20,7 +20,7 @@
"require": { "require": {
"php": ">=5.3.0", "php": ">=5.3.0",
"ext-pcntl": "*", "ext-pcntl": "*",
"colinmollenhour/credis": "dev-master" "colinmollenhour/credis": "1.2.*"
}, },
"suggest": { "suggest": {
"ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.", "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.",

439
composer.lock generated
View File

@ -1,24 +1,27 @@
{ {
"hash": "d37909ad0ffc11ed4d1e67dcaabe00b2", "_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file"
],
"hash": "1f551c3cdade1b7ff7d9e32a44eeb3dc",
"packages": [ "packages": [
{ {
"name": "colinmollenhour/credis", "name": "colinmollenhour/credis",
"version": "dev-master", "version": "1.2",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/chrisboulton/credis", "url": "https://github.com/colinmollenhour/credis.git",
"reference": "62c73dd16e08069e3fd8f224cb4a5ddd73db8095" "reference": "1.2"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/chrisboulton/credis/zipball/62c73dd16e08069e3fd8f224cb4a5ddd73db8095", "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/1.2",
"reference": "62c73dd16e08069e3fd8f224cb4a5ddd73db8095", "reference": "1.2",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.3.0" "php": ">=5.3.0"
}, },
"time": "2013-01-12 10:15:31",
"type": "library", "type": "library",
"autoload": { "autoload": {
"classmap": [ "classmap": [
@ -26,6 +29,7 @@
"Cluster.php" "Cluster.php"
] ]
}, },
"notification-url": "https://packagist.org/downloads/",
"license": [ "license": [
"MIT" "MIT"
], ],
@ -37,17 +41,424 @@
], ],
"description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.",
"homepage": "https://github.com/colinmollenhour/credis", "homepage": "https://github.com/colinmollenhour/credis",
"support": { "time": "2013-03-19 02:57:04"
"source": "https://github.com/chrisboulton/credis/tree/master" }
} ],
"packages-dev": [
{
"name": "phpunit/php-code-coverage",
"version": "1.2.9",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/php-code-coverage.git",
"reference": "1.2.9"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/1.2.9",
"reference": "1.2.9",
"shasum": ""
},
"require": {
"php": ">=5.3.3",
"phpunit/php-file-iterator": ">=1.3.0@stable",
"phpunit/php-text-template": ">=1.1.1@stable",
"phpunit/php-token-stream": ">=1.1.3@stable"
},
"suggest": {
"ext-dom": "*",
"ext-xdebug": ">=2.0.5"
},
"type": "library",
"autoload": {
"classmap": [
"PHP/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.",
"homepage": "https://github.com/sebastianbergmann/php-code-coverage",
"keywords": [
"coverage",
"testing",
"xunit"
],
"time": "2013-02-26 18:55:56"
},
{
"name": "phpunit/php-file-iterator",
"version": "1.3.3",
"source": {
"type": "git",
"url": "git://github.com/sebastianbergmann/php-file-iterator.git",
"reference": "1.3.3"
},
"dist": {
"type": "zip",
"url": "https://github.com/sebastianbergmann/php-file-iterator/zipball/1.3.3",
"reference": "1.3.3",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"type": "library",
"autoload": {
"classmap": [
"File/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "FilterIterator implementation that filters files based on a list of suffixes.",
"homepage": "http://www.phpunit.de/",
"keywords": [
"filesystem",
"iterator"
],
"time": "2012-10-11 04:44:38"
},
{
"name": "phpunit/php-text-template",
"version": "1.1.4",
"source": {
"type": "git",
"url": "git://github.com/sebastianbergmann/php-text-template.git",
"reference": "1.1.4"
},
"dist": {
"type": "zip",
"url": "https://github.com/sebastianbergmann/php-text-template/zipball/1.1.4",
"reference": "1.1.4",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"type": "library",
"autoload": {
"classmap": [
"Text/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "Simple template engine.",
"homepage": "https://github.com/sebastianbergmann/php-text-template/",
"keywords": [
"template"
],
"time": "2012-10-31 11:15:28"
},
{
"name": "phpunit/php-timer",
"version": "1.0.4",
"source": {
"type": "git",
"url": "git://github.com/sebastianbergmann/php-timer.git",
"reference": "1.0.4"
},
"dist": {
"type": "zip",
"url": "https://github.com/sebastianbergmann/php-timer/zipball/1.0.4",
"reference": "1.0.4",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"type": "library",
"autoload": {
"classmap": [
"PHP/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "Utility class for timing",
"homepage": "http://www.phpunit.de/",
"keywords": [
"timer"
],
"time": "2012-10-11 04:45:58"
},
{
"name": "phpunit/php-token-stream",
"version": "1.1.5",
"source": {
"type": "git",
"url": "git://github.com/sebastianbergmann/php-token-stream.git",
"reference": "1.1.5"
},
"dist": {
"type": "zip",
"url": "https://github.com/sebastianbergmann/php-token-stream/zipball/1.1.5",
"reference": "1.1.5",
"shasum": ""
},
"require": {
"ext-tokenizer": "*",
"php": ">=5.3.3"
},
"type": "library",
"autoload": {
"classmap": [
"PHP/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "Wrapper around PHP's tokenizer extension.",
"homepage": "http://www.phpunit.de/",
"keywords": [
"tokenizer"
],
"time": "2012-10-11 04:47:14"
},
{
"name": "phpunit/phpunit",
"version": "3.7.18",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/phpunit.git",
"reference": "3.7.18"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3.7.18",
"reference": "3.7.18",
"shasum": ""
},
"require": {
"ext-dom": "*",
"ext-pcre": "*",
"ext-reflection": "*",
"ext-spl": "*",
"php": ">=5.3.3",
"phpunit/php-code-coverage": ">=1.2.1,<1.3.0",
"phpunit/php-file-iterator": ">=1.3.1",
"phpunit/php-text-template": ">=1.1.1",
"phpunit/php-timer": ">=1.0.2,<1.1.0",
"phpunit/phpunit-mock-objects": ">=1.2.0,<1.3.0",
"symfony/yaml": ">=2.2.0"
},
"require-dev": {
"pear-pear/pear": "1.9.4"
},
"suggest": {
"ext-json": "*",
"ext-simplexml": "*",
"ext-tokenizer": "*",
"phpunit/php-invoker": ">=1.1.0,<1.2.0"
},
"bin": [
"composer/bin/phpunit"
],
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.7.x-dev"
}
},
"autoload": {
"classmap": [
"PHPUnit/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"",
"../../symfony/yaml/"
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sebastian@phpunit.de",
"role": "lead"
}
],
"description": "The PHP Unit Testing framework.",
"homepage": "http://www.phpunit.de/",
"keywords": [
"phpunit",
"testing",
"xunit"
],
"time": "2013-03-07 21:45:39"
},
{
"name": "phpunit/phpunit-mock-objects",
"version": "1.2.3",
"source": {
"type": "git",
"url": "git://github.com/sebastianbergmann/phpunit-mock-objects.git",
"reference": "1.2.3"
},
"dist": {
"type": "zip",
"url": "https://github.com/sebastianbergmann/phpunit-mock-objects/archive/1.2.3.zip",
"reference": "1.2.3",
"shasum": ""
},
"require": {
"php": ">=5.3.3",
"phpunit/php-text-template": ">=1.1.1@stable"
},
"suggest": {
"ext-soap": "*"
},
"type": "library",
"autoload": {
"classmap": [
"PHPUnit/"
]
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
""
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Sebastian Bergmann",
"email": "sb@sebastian-bergmann.de",
"role": "lead"
}
],
"description": "Mock Object library for PHPUnit",
"homepage": "https://github.com/sebastianbergmann/phpunit-mock-objects/",
"keywords": [
"mock",
"xunit"
],
"time": "2013-01-13 10:24:48"
},
{
"name": "symfony/yaml",
"version": "v2.2.0",
"target-dir": "Symfony/Component/Yaml",
"source": {
"type": "git",
"url": "https://github.com/symfony/Yaml.git",
"reference": "v2.2.0-RC3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/Yaml/zipball/v2.2.0-RC3",
"reference": "v2.2.0-RC3",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.2-dev"
}
},
"autoload": {
"psr-0": {
"Symfony\\Component\\Yaml\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "http://symfony.com/contributors"
}
],
"description": "Symfony Yaml Component",
"homepage": "http://symfony.com",
"time": "2013-01-27 16:49:19"
} }
], ],
"packages-dev": null,
"aliases": [ "aliases": [
], ],
"minimum-stability": "stable", "minimum-stability": "stable",
"stability-flags": { "stability-flags": [
"colinmollenhour/credis": 20
} ],
"platform": {
"php": ">=5.3.0",
"ext-pcntl": "*"
},
"platform-dev": [
]
} }

1
demo/bad_job.php
View File

@ -6,4 +6,3 @@ class Bad_PHP_Job
throw new Exception('Unable to run this job!'); throw new Exception('Unable to run this job!');
} }
} }
?>

1
demo/check_status.php
View File

@ -18,4 +18,3 @@ while(true) {
fwrite(STDOUT, "Status of ".$argv[1]." is: ".$status->get()."\n"); fwrite(STDOUT, "Status of ".$argv[1]." is: ".$status->get()."\n");
sleep(1); sleep(1);
} }
?>

6
demo/job.php
View File

@ -3,8 +3,8 @@ class PHP_Job
{ {
public function perform() public function perform()
{ {
sleep(120); fwrite(STDOUT, 'Start job! -> ');
fwrite(STDOUT, 'Hello!'); sleep(1);
fwrite(STDOUT, 'Job ended!' . PHP_EOL);
} }
} }
?>

1
demo/long_job.php
View File

@ -6,4 +6,3 @@ class Long_PHP_Job
sleep(600); sleep(600);
} }
} }
?>

1
demo/php_error_job.php
View File

@ -6,4 +6,3 @@ class PHP_Error_Job
callToUndefinedFunction(); callToUndefinedFunction();
} }
} }
?>

3
demo/queue.php
View File

@ -14,6 +14,5 @@ $args = array(
), ),
); );
$jobId = Resque::enqueue('default', $argv[1], $args, true); $jobId = Resque::enqueue($argv[1], $argv[2], $args, true);
echo "Queued job ".$jobId."\n\n"; echo "Queued job ".$jobId."\n\n";
?>

3
demo/resque.php
View File

@ -4,5 +4,4 @@ require 'bad_job.php';
require 'job.php'; require 'job.php';
require 'php_error_job.php'; require 'php_error_job.php';
require '../bin/resque'; require '../bin/resque.php';
?>

37
lib/Resque.php
View File

@ -10,6 +10,8 @@ class Resque
{ {
const VERSION = '1.2'; const VERSION = '1.2';
const DEFAULT_INTERVAL = 5;
/** /**
* @var Resque_Redis Instance of Resque_Redis that talks to redis. * @var Resque_Redis Instance of Resque_Redis that talks to redis.
*/ */
@ -115,6 +117,7 @@ class Resque
public static function pop($queue) public static function pop($queue)
{ {
$item = self::redis()->lpop('queue:' . $queue); $item = self::redis()->lpop('queue:' . $queue);
if(!$item) { if(!$item) {
return; return;
} }
@ -122,6 +125,40 @@ class Resque
return json_decode($item, true); return json_decode($item, true);
} }
/**
* Pop an item off the end of the specified queues, using blocking list pop,
* decode it and return it.
*
* @param array $queues
* @param int $timeout
* @return null|array Decoded item from the queue.
*/
public static function blpop(array $queues, $timeout)
{
$list = array();
foreach($queues AS $queue) {
$list[] = 'queue:' . $queue;
}
$item = self::redis()->blpop($list, (int)$timeout);
if(!$item) {
return;
}
/**
* Normally the Resque_Redis class returns queue names without the prefix
* But the blpop is a bit different. It returns the name as prefix:queue:name
* So we need to strip off the prefix:queue: part
*/
$queue = substr($item[0], strlen(self::redis()->getPrefix() . 'queue:'));
return array(
'queue' => $queue,
'payload' => json_decode($item[1], true)
);
}
/** /**
* Return the size (number of pending jobs) of the specified queue. * Return the size (number of pending jobs) of the specified queue.
* *

27
lib/Resque/Job.php
View File

@ -58,13 +58,11 @@ class Resque_Job
); );
} }
$id = md5(uniqid('', true)); $id = md5(uniqid('', true));
if (!Resque::push($queue, array( Resque::push($queue, array(
'class' => $class, 'class' => $class,
'args' => array($args), 'args' => array($args),
'id' => $id, 'id' => $id,
))) { ));
return false;
}
if($monitor) { if($monitor) {
Resque_Job_Status::create($id); Resque_Job_Status::create($id);
@ -90,6 +88,25 @@ class Resque_Job
return new Resque_Job($queue, $payload); return new Resque_Job($queue, $payload);
} }
/**
* Find the next available job from the specified queues using blocking list pop
* and return an instance of Resque_Job for it.
*
* @param array $queues
* @param int $timeout
* @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found.
*/
public static function reserveBlocking(array $queues, $timeout = null)
{
$item = Resque::blpop($queues, $timeout);
if(!is_array($item)) {
return false;
}
return new Resque_Job($item['queue'], $item['payload']);
}
/** /**
* Update the status of the current job. * Update the status of the current job.
* *
@ -153,7 +170,7 @@ class Resque_Job
); );
} }
$this->instance = new $this->payload['class'](); $this->instance = new $this->payload['class'];
$this->instance->job = $this; $this->instance->job = $this;
$this->instance->args = $this->getArguments(); $this->instance->args = $this->getArguments();
$this->instance->queue = $this->queue; $this->instance->queue = $this->queue;

7
lib/Resque/Redis.php
View File

@ -47,6 +47,7 @@ class Resque_Redis
'lset', 'lset',
'lrem', 'lrem',
'lpop', 'lpop',
'blpop',
'rpop', 'rpop',
'sadd', 'sadd',
'srem', 'srem',
@ -142,7 +143,13 @@ class Resque_Redis
*/ */
public function __call($name, $args) { public function __call($name, $args) {
if(in_array($name, $this->keyCommands)) { if(in_array($name, $this->keyCommands)) {
if(is_array($args[0])) {
foreach($args[0] AS $i => $v) {
$args[0][$i] = self::$defaultNamespace . $v;
}
} else {
$args[0] = self::$defaultNamespace . $args[0]; $args[0] = self::$defaultNamespace . $args[0];
}
} }
try { try {
return $this->driver->__call($name, $args); return $this->driver->__call($name, $args);

32
lib/Resque/Worker.php
View File

@ -147,7 +147,7 @@ class Resque_Worker
* *
* @param int $interval How often to check for new jobs across the queues. * @param int $interval How often to check for new jobs across the queues.
*/ */
public function work($interval = 5) public function work($interval = Resque::DEFAULT_INTERVAL, $blocking = false)
{ {
$this->updateProcLine('Starting'); $this->updateProcLine('Starting');
$this->startup(); $this->startup();
@ -160,7 +160,14 @@ class Resque_Worker
// Attempt to find and reserve a job // Attempt to find and reserve a job
$job = false; $job = false;
if(!$this->paused) { if(!$this->paused) {
$job = $this->reserve(); if($blocking === true) {
$this->log('Starting blocking with timeout of ' . $interval, self::LOG_VERBOSE);
$this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with blocking timeout ' . $interval);
} else {
$this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval);
}
$job = $this->reserve($blocking, $interval);
} }
if(!$job) { if(!$job) {
@ -168,6 +175,9 @@ class Resque_Worker
if($interval == 0) { if($interval == 0) {
break; break;
} }
if($blocking === false)
{
// If no job was found, we sleep for $interval before continuing and checking again // If no job was found, we sleep for $interval before continuing and checking again
$this->log('Sleeping for ' . $interval, self::LOG_VERBOSE); $this->log('Sleeping for ' . $interval, self::LOG_VERBOSE);
if($this->paused) { if($this->paused) {
@ -176,7 +186,10 @@ class Resque_Worker
else { else {
$this->updateProcLine('Waiting for ' . implode(',', $this->queues)); $this->updateProcLine('Waiting for ' . implode(',', $this->queues));
} }
usleep($interval * 1000000); usleep($interval * 1000000);
}
continue; continue;
} }
@ -242,16 +255,24 @@ class Resque_Worker
} }
/** /**
* Attempt to find a job from the top of one of the queues for this worker. * @param bool $blocking
* * @param int $timeout
* @return object|boolean Instance of Resque_Job if a job is found, false if not. * @return object|boolean Instance of Resque_Job if a job is found, false if not.
*/ */
public function reserve() public function reserve($blocking = false, $timeout = null)
{ {
$queues = $this->queues(); $queues = $this->queues();
if(!is_array($queues)) { if(!is_array($queues)) {
return; return;
} }
if($blocking === true) {
$job = Resque_Job::reserveBlocking($queues, $timeout);
if($job) {
$this->log('Found job on ' . $job->queue, self::LOG_VERBOSE);
return $job;
}
} else {
foreach($queues as $queue) { foreach($queues as $queue) {
$this->log('Checking ' . $queue, self::LOG_VERBOSE); $this->log('Checking ' . $queue, self::LOG_VERBOSE);
$job = Resque_Job::reserve($queue); $job = Resque_Job::reserve($queue);
@ -260,6 +281,7 @@ class Resque_Worker
return $job; return $job;
} }
} }
}
return false; return false;
} }

6
test/Resque/Tests/JobStatusTest.php
View File

@ -8,6 +8,11 @@
*/ */
class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase
{ {
/**
* @var \Resque_Worker
*/
protected $worker;
public function setUp() public function setUp()
{ {
parent::setUp(); parent::setUp();
@ -36,6 +41,7 @@ class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase
$status = new Resque_Job_Status($token); $status = new Resque_Job_Status($token);
$this->assertEquals(Resque_Job_Status::STATUS_WAITING, $status->get()); $this->assertEquals(Resque_Job_Status::STATUS_WAITING, $status->get());
} }
public function testRunningJobReturnsRunningStatus() public function testRunningJobReturnsRunningStatus()
{ {
$token = Resque::enqueue('jobs', 'Failing_Job', null, true); $token = Resque::enqueue('jobs', 'Failing_Job', null, true);

23
test/Resque/Tests/WorkerTest.php
View File

@ -247,4 +247,27 @@ class Resque_Tests_WorkerTest extends Resque_Tests_TestCase
$this->assertEquals(1, Resque_Stat::get('failed')); $this->assertEquals(1, Resque_Stat::get('failed'));
} }
public function testBlockingListPop()
{
$worker = new Resque_Worker('jobs');
$worker->registerWorker();
Resque::enqueue('jobs', 'Test_Job_1');
Resque::enqueue('jobs', 'Test_Job_2');
$i = 1;
while($job = $worker->reserve(true, 1))
{
$this->assertEquals('Test_Job_' . $i, $job->payload['class']);
if($i == 2) {
break;
}
$i++;
}
$this->assertEquals(2, $i);
}
} }