From 41bf8c1c26708232252d85310402163c563e7e7e Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Fri, 24 Aug 2012 16:51:03 +0200 Subject: [PATCH 01/15] Basic support for blocking list pop --- lib/Resque.php | 17 ++++++++++----- lib/Resque/Job.php | 6 +++--- lib/Resque/Redis.php | 1 + lib/Resque/RedisCluster.php | 1 + lib/Resque/Worker.php | 42 ++++++++++++++++++------------------- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/lib/Resque.php b/lib/Resque.php index 9bc8143..4fe5a16 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -14,6 +14,8 @@ class Resque { const VERSION = '1.0'; + const DEFAULT_INTERVAL = 5; + /** * @var Resque_Redis Instance of Resque_Redis that talks to redis. */ @@ -115,14 +117,19 @@ class Resque * @param string $queue The name of the queue to fetch an item from. * @return array Decoded item from the queue. */ - public static function pop($queue) + public static function pop($queue, $interval = null) { - $item = self::redis()->lpop('queue:' . $queue); + if($interval == null) { + $item = self::redis()->lpop('queue:' . $queue); + } else { + $item = self::redis()->blpop('queue:' . $queue, $interval ?: Resque::DEFAULT_INTERVAL); + } + if(!$item) { return; } - return json_decode($item, true); + return json_decode($interval == 0 ? $item : $item[1], true); } /** @@ -168,10 +175,10 @@ class Resque * @param string $queue Queue to fetch next available job from. * @return Resque_Job Instance of Resque_Job to be processed, false if none or error. */ - public static function reserve($queue) + public static function reserve($queue, $interval = null) { require_once dirname(__FILE__) . '/Resque/Job.php'; - return Resque_Job::reserve($queue); + return Resque_Job::reserve($queue, $interval); } /** diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 0d275a1..6af0219 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -83,9 +83,9 @@ class Resque_Job * @param string $queue The name of the queue to check for a job in. * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. */ - public static function reserve($queue) + public static function reserve($queue, $interval = null) { - $payload = Resque::pop($queue); + $payload = Resque::pop($queue, $interval); if(!is_array($payload)) { return false; } @@ -156,7 +156,7 @@ class Resque_Job ); } - $this->instance = new $this->payload['class'](); + $this->instance = new $this->payload['class']; $this->instance->job = $this; $this->instance->args = $this->getArguments(); $this->instance->queue = $this->queue; diff --git a/lib/Resque/Redis.php b/lib/Resque/Redis.php index b6d7522..e1f077f 100644 --- a/lib/Resque/Redis.php +++ b/lib/Resque/Redis.php @@ -50,6 +50,7 @@ class Resque_Redis extends Redisent 'lset', 'lrem', 'lpop', + 'blpop', 'rpop', 'sadd', 'srem', diff --git a/lib/Resque/RedisCluster.php b/lib/Resque/RedisCluster.php index 39c3f5c..e884653 100644 --- a/lib/Resque/RedisCluster.php +++ b/lib/Resque/RedisCluster.php @@ -50,6 +50,7 @@ class Resque_RedisCluster extends RedisentCluster 'lset', 'lrem', 'lpop', + 'blpop', 'rpop', 'sadd', 'srem', diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 31f2f3c..58571be 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -153,7 +153,7 @@ class Resque_Worker * * @param int $interval How often to check for new jobs across the queues. */ - public function work($interval = 5) + public function work($interval = Resque::DEFAULT_INTERVAL) { $this->updateProcLine('Starting'); $this->startup(); @@ -166,25 +166,25 @@ class Resque_Worker // Attempt to find and reserve a job $job = false; if(!$this->paused) { - $job = $this->reserve(); + $job = $this->reserve($interval); } - if(!$job) { - // For an interval of 0, break now - helps with unit testing etc - if($interval == 0) { - break; - } - // If no job was found, we sleep for $interval before continuing and checking again - $this->log('Sleeping for ' . $interval, true); - if($this->paused) { - $this->updateProcLine('Paused'); - } - else { - $this->updateProcLine('Waiting for ' . implode(',', $this->queues)); - } - usleep($interval * 1000000); - continue; - } + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + + if(!$job) { + // For an interval of 0, break now - helps with unit testing etc + if($interval == 0) { + break; + } + + // If no job was found, we sleep for $interval before continuing and checking again + if($this->paused) { + $this->updateProcLine('Paused'); + usleep($interval * 1000000); //it's paused, so don't hog redis with requests. + } + + continue; + } $this->log('got ' . $job); Resque_Event::trigger('beforeFork', $job); @@ -252,15 +252,15 @@ class Resque_Worker * * @return object|boolean Instance of Resque_Job if a job is found, false if not. */ - public function reserve() + public function reserve($interval = null) { $queues = $this->queues(); if(!is_array($queues)) { return; } foreach($queues as $queue) { - $this->log('Checking ' . $queue, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue); + $this->log('Checking ' . $queue . ' with interval ' . $interval, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue, $interval); if($job) { $this->log('Found job on ' . $queue, self::LOG_VERBOSE); return $job; From 5eff86d3c3c54eedcfeecc515e02805d70d66ac2 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Mon, 10 Sep 2012 10:23:18 +0200 Subject: [PATCH 02/15] Fixed bug --- lib/Resque.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Resque.php b/lib/Resque.php index 4fe5a16..29249f0 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -122,7 +122,7 @@ class Resque if($interval == null) { $item = self::redis()->lpop('queue:' . $queue); } else { - $item = self::redis()->blpop('queue:' . $queue, $interval ?: Resque::DEFAULT_INTERVAL); + $item = self::redis()->blpop('queue:' . $queue, $interval ? $interval : Resque::DEFAULT_INTERVAL); } if(!$item) { From ff0d2bc655339fa7d2cc284e97a36cc5bf96a910 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Tue, 12 Mar 2013 09:55:03 +0100 Subject: [PATCH 03/15] Added test for blpop --- test/Resque/Tests/WorkerTest.php | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/Resque/Tests/WorkerTest.php b/test/Resque/Tests/WorkerTest.php index b2f0e00..b0001c9 100644 --- a/test/Resque/Tests/WorkerTest.php +++ b/test/Resque/Tests/WorkerTest.php @@ -247,4 +247,27 @@ class Resque_Tests_WorkerTest extends Resque_Tests_TestCase $this->assertEquals(1, Resque_Stat::get('failed')); } + + public function testBlockingListPop() + { + $worker = new Resque_Worker('jobs'); + $worker->registerWorker(); + + Resque::enqueue('jobs', 'Test_Job_1'); + Resque::enqueue('jobs', 'Test_Job_2'); + + $i = 1; + while($job = $worker->reserve(60)) + { + $this->assertEquals('Test_Job_' . $i, $job->payload['class']); + + if($i == 2) { + break; + } + + $i++; + } + + $this->assertEquals(2, $i); + } } \ No newline at end of file From 132531e1a2a2d1e0905c353914e5b0511e1bb77b Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Tue, 12 Mar 2013 11:18:37 +0100 Subject: [PATCH 04/15] WIP --- bin/resque | 1 - demo/job.php | 5 +- demo/queue.php | 2 +- lib/Resque.php | 33 +++++++-- lib/Resque/Job.php | 49 ++++++++++---- lib/Resque/Redis.php | 8 ++- lib/Resque/Worker.php | 154 ++++++++++++++++++++++-------------------- 7 files changed, 150 insertions(+), 102 deletions(-) diff --git a/bin/resque b/bin/resque index 78befad..2cb5ffd 100644 --- a/bin/resque +++ b/bin/resque @@ -1,4 +1,3 @@ -#!/usr/bin/env php '); + sleep(1); + fwrite(STDOUT, 'Job ended!' . PHP_EOL); } } ?> \ No newline at end of file diff --git a/demo/queue.php b/demo/queue.php index 95ff5b0..393df87 100644 --- a/demo/queue.php +++ b/demo/queue.php @@ -14,6 +14,6 @@ $args = array( ), ); -$jobId = Resque::enqueue('default', $argv[1], $args, true); +$jobId = Resque::enqueue($argv[1], $argv[2], $args, true); echo "Queued job ".$jobId."\n\n"; ?> \ No newline at end of file diff --git a/lib/Resque.php b/lib/Resque.php index bd29e84..2e459f0 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -110,21 +110,40 @@ class Resque * @param string $queue The name of the queue to fetch an item from. * @return array Decoded item from the queue. */ - public static function pop($queue, $interval = null) + public static function pop($queue) { - if($interval == null) { - $item = self::redis()->lpop('queue:' . $queue); - } else { - $item = self::redis()->blpop('queue:' . $queue, $interval ? $interval : Resque::DEFAULT_INTERVAL); - } + $item = self::redis()->lpop('queue:' . $queue); if(!$item) { return; } - return json_decode($interval == 0 ? $item : $item[1], true); + return json_decode($item, true); } + /** + * Pop an item off the end of the specified queue, decode it and + * return it. + * + * @param string $queue The name of the queue to fetch an item from. + * @return array Decoded item from the queue. + */ + public static function blpop($queues, $interval = null) + { + $list = array(); + foreach($queues AS $queue) { + $list[] = 'queue:' . $queue; + } + + $item = self::redis()->blpop($list, $interval ? (int)$interval : Resque::DEFAULT_INTERVAL); + + if(!$item) { + return; + } + + return json_decode($item[1], true); + } + /** * Return the size (number of pending jobs) of the specified queue. * diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 5066c27..0b1f01e 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -71,22 +71,41 @@ class Resque_Job return $id; } - /** - * Find the next available job from the specified queue and return an - * instance of Resque_Job for it. - * - * @param string $queue The name of the queue to check for a job in. - * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. - */ - public static function reserve($queue, $interval = null) - { - $payload = Resque::pop($queue, $interval); - if(!is_array($payload)) { - return false; - } + /** + * Find the next available job from the specified queue and return an + * instance of Resque_Job for it. + * + * @param string $queue The name of the queue to check for a job in. + * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. + */ + public static function reserve($queue) + { + $payload = Resque::pop($queue); + if(!is_array($payload)) { + return false; + } - return new Resque_Job($queue, $payload); - } + return new Resque_Job($queue, $payload); + } + + /** + * Find the next available job from the specified queue and return an + * instance of Resque_Job for it. + * + * @param string $queue The name of the queue to check for a job in. + * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. + */ + public static function reserveBlocking($queues, $interval = null) + { + $payload = Resque::blpop($queues, $interval); + if(!is_array($payload)) { + return false; + } + + var_dump($payload); + + return new Resque_Job($payload->queue, $payload); + } /** * Update the status of the current job. diff --git a/lib/Resque/Redis.php b/lib/Resque/Redis.php index 4447146..6cef3d2 100644 --- a/lib/Resque/Redis.php +++ b/lib/Resque/Redis.php @@ -143,7 +143,13 @@ class Resque_Redis */ public function __call($name, $args) { if(in_array($name, $this->keyCommands)) { - $args[0] = self::$defaultNamespace . $args[0]; + if(is_array($args[0])) { + foreach($args[0] AS $i => $v) { + $args[0][$i] = self::$defaultNamespace . $v; + } + } else { + $args[0] = self::$defaultNamespace . $args[0]; + } } try { return $this->driver->__call($name, $args); diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 204aae6..03487f5 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -152,70 +152,53 @@ class Resque_Worker $this->updateProcLine('Starting'); $this->startup(); - while(true) { - if($this->shutdown) { - break; - } - - // Attempt to find and reserve a job - $job = false; - if(!$this->paused) { - $job = $this->reserve($interval); - } - + while($job = $this->reserveBlocking($interval)) { $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); - if(!$job) { - // For an interval of 0, break now - helps with unit testing etc - if($interval == 0) { - break; - } + $this->log('got ' . $job); + Resque_Event::trigger('beforeFork', $job); + $this->workingOn($job); - // If no job was found, we sleep for $interval before continuing and checking again - if($this->paused) { - $this->updateProcLine('Paused'); - usleep($interval * 1000000); //it's paused, so don't hog redis with requests. - } + $this->child = Resque::fork(); - continue; + // Forked and we're the child. Run the job. + if ($this->child === 0 || $this->child === false) { + $status = 'Processing ' . $job->queue . ' since ' . strftime('%F %T'); + $this->updateProcLine($status); + $this->log($status, self::LOG_VERBOSE); + $this->perform($job); + if ($this->child === 0) { + exit(0); + } } - $this->log('got ' . $job); - Resque_Event::trigger('beforeFork', $job); - $this->workingOn($job); + if($this->child > 0) { + // Parent process, sit and wait + $status = 'Forked ' . $this->child . ' at ' . strftime('%F %T'); + $this->updateProcLine($status); + $this->log($status, self::LOG_VERBOSE); - $this->child = Resque::fork(); + // Wait until the child process finishes before continuing + pcntl_wait($status); + $exitStatus = pcntl_wexitstatus($status); + if($exitStatus !== 0) { + $job->fail(new Resque_Job_DirtyExitException( + 'Job exited with exit code ' . $exitStatus + )); + } + } - // Forked and we're the child. Run the job. - if ($this->child === 0 || $this->child === false) { - $status = 'Processing ' . $job->queue . ' since ' . strftime('%F %T'); - $this->updateProcLine($status); - $this->log($status, self::LOG_VERBOSE); - $this->perform($job); - if ($this->child === 0) { - exit(0); - } - } + $this->child = null; + $this->doneWorking(); - if($this->child > 0) { - // Parent process, sit and wait - $status = 'Forked ' . $this->child . ' at ' . strftime('%F %T'); - $this->updateProcLine($status); - $this->log($status, self::LOG_VERBOSE); + if($this->shutdown) { + break; + } - // Wait until the child process finishes before continuing - pcntl_wait($status); - $exitStatus = pcntl_wexitstatus($status); - if($exitStatus !== 0) { - $job->fail(new Resque_Job_DirtyExitException( - 'Job exited with exit code ' . $exitStatus - )); - } - } - - $this->child = null; - $this->doneWorking(); - } + if($this->paused) { + break; + } + } $this->unregisterWorker(); } @@ -241,28 +224,49 @@ class Resque_Worker $this->log('done ' . $job); } - /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. - */ - public function reserve($interval = null) - { - $queues = $this->queues(); - if(!is_array($queues)) { - return; - } - foreach($queues as $queue) { - $this->log('Checking ' . $queue . ' with interval ' . $interval, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue, $interval); - if($job) { - $this->log('Found job on ' . $queue, self::LOG_VERBOSE); - return $job; - } - } + /** + * Attempt to find a job from the top of one of the queues for this worker. + * + * @return object|boolean Instance of Resque_Job if a job is found, false if not. + */ + public function reserve() + { + $queues = $this->queues(); + if(!is_array($queues)) { + return; + } + foreach($queues as $queue) { + $this->log('Checking ' . $queue, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue); + if($job) { + $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + return $job; + } + } - return false; - } + return false; + } + + /** + * Attempt to find a job from the top of one of the queues for this worker. + * + * @return object|boolean Instance of Resque_Job if a job is found, false if not. + */ + public function reserveBlocking($interval = null) + { + $queues = $this->queues(); + if(!is_array($queues)) { + return; + } + + $job = Resque_Job::reserveBlocking($queues, $interval); + if($job) { + $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); + return $job; + } + + return false; + } /** * Return an array containing all of the queues that this worker should use From c2c4d06f7b72a3d8c68c61ef443fbb0b00e87200 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Wed, 13 Mar 2013 12:41:32 +0100 Subject: [PATCH 05/15] Working blocking list pop :) --- bin/resque | 106 +------ bin/resque.php | 105 +++++++ composer.lock | 416 +++++++++++++++++++++++++++- demo/bad_job.php | 3 +- demo/check_status.php | 3 +- demo/job.php | 3 +- demo/long_job.php | 3 +- demo/php_error_job.php | 3 +- demo/queue.php | 3 +- demo/resque.php | 3 +- lib/Resque.php | 17 +- lib/Resque/Job.php | 13 +- lib/Resque/Worker.php | 99 ++++--- test/Resque/Tests/JobStatusTest.php | 6 + 14 files changed, 606 insertions(+), 177 deletions(-) mode change 100644 => 100755 bin/resque create mode 100644 bin/resque.php diff --git a/bin/resque b/bin/resque old mode 100644 new mode 100755 index 2cb5ffd..5475c98 --- a/bin/resque +++ b/bin/resque @@ -1,105 +1,3 @@ +#!/usr/bin/env php 1) { - $count = $COUNT; -} - -if($count > 1) { - for($i = 0; $i < $count; ++$i) { - $pid = Resque::fork(); - if($pid == -1) { - die("Could not fork worker ".$i."\n"); - } - // Child, start the worker - else if(!$pid) { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval); - break; - } - } -} -// Start a single worker -else { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; - - $PIDFILE = getenv('PIDFILE'); - if ($PIDFILE) { - file_put_contents($PIDFILE, getmypid()) or - die('Could not write PID information to ' . $PIDFILE); - } - - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval); -} -?> +include("resque.php"); \ No newline at end of file diff --git a/bin/resque.php b/bin/resque.php new file mode 100644 index 0000000..3fc788f --- /dev/null +++ b/bin/resque.php @@ -0,0 +1,105 @@ + 1) { + $count = $COUNT; +} + +if($count > 1) { + for($i = 0; $i < $count; ++$i) { + $pid = Resque::fork(); + if($pid == -1) { + die("Could not fork worker ".$i."\n"); + } + // Child, start the worker + else if(!$pid) { + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); + break; + } + } +} +// Start a single worker +else { + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; + + $PIDFILE = getenv('PIDFILE'); + if ($PIDFILE) { + file_put_contents($PIDFILE, getmypid()) or + die('Could not write PID information to ' . $PIDFILE); + } + + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); +} \ No newline at end of file diff --git a/composer.lock b/composer.lock index 877c243..760d4cf 100644 --- a/composer.lock +++ b/composer.lock @@ -18,7 +18,6 @@ "require": { "php": ">=5.3.0" }, - "time": "2013-01-12 10:15:31", "type": "library", "autoload": { "classmap": [ @@ -39,15 +38,424 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "source": "https://github.com/chrisboulton/credis/tree/master" - } + }, + "time": "2013-01-12 10:15:31" + } + ], + "packages-dev": [ + { + "name": "phpunit/php-code-coverage", + "version": "1.2.9", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-code-coverage.git", + "reference": "1.2.9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/1.2.9", + "reference": "1.2.9", + "shasum": "" + }, + "require": { + "php": ">=5.3.3", + "phpunit/php-file-iterator": ">=1.3.0@stable", + "phpunit/php-text-template": ">=1.1.1@stable", + "phpunit/php-token-stream": ">=1.1.3@stable" + }, + "suggest": { + "ext-dom": "*", + "ext-xdebug": ">=2.0.5" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", + "homepage": "https://github.com/sebastianbergmann/php-code-coverage", + "keywords": [ + "coverage", + "testing", + "xunit" + ], + "time": "2013-02-26 18:55:56" + }, + { + "name": "phpunit/php-file-iterator", + "version": "1.3.3", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-file-iterator.git", + "reference": "1.3.3" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-file-iterator/zipball/1.3.3", + "reference": "1.3.3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "File/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "FilterIterator implementation that filters files based on a list of suffixes.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "filesystem", + "iterator" + ], + "time": "2012-10-11 04:44:38" + }, + { + "name": "phpunit/php-text-template", + "version": "1.1.4", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-text-template.git", + "reference": "1.1.4" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-text-template/zipball/1.1.4", + "reference": "1.1.4", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "Text/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Simple template engine.", + "homepage": "https://github.com/sebastianbergmann/php-text-template/", + "keywords": [ + "template" + ], + "time": "2012-10-31 11:15:28" + }, + { + "name": "phpunit/php-timer", + "version": "1.0.4", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-timer.git", + "reference": "1.0.4" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-timer/zipball/1.0.4", + "reference": "1.0.4", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Utility class for timing", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "timer" + ], + "time": "2012-10-11 04:45:58" + }, + { + "name": "phpunit/php-token-stream", + "version": "1.1.5", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-token-stream.git", + "reference": "1.1.5" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-token-stream/zipball/1.1.5", + "reference": "1.1.5", + "shasum": "" + }, + "require": { + "ext-tokenizer": "*", + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Wrapper around PHP's tokenizer extension.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "tokenizer" + ], + "time": "2012-10-11 04:47:14" + }, + { + "name": "phpunit/phpunit", + "version": "3.7.18", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/phpunit.git", + "reference": "3.7.18" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3.7.18", + "reference": "3.7.18", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-pcre": "*", + "ext-reflection": "*", + "ext-spl": "*", + "php": ">=5.3.3", + "phpunit/php-code-coverage": ">=1.2.1,<1.3.0", + "phpunit/php-file-iterator": ">=1.3.1", + "phpunit/php-text-template": ">=1.1.1", + "phpunit/php-timer": ">=1.0.2,<1.1.0", + "phpunit/phpunit-mock-objects": ">=1.2.0,<1.3.0", + "symfony/yaml": ">=2.2.0" + }, + "require-dev": { + "pear-pear/pear": "1.9.4" + }, + "suggest": { + "ext-json": "*", + "ext-simplexml": "*", + "ext-tokenizer": "*", + "phpunit/php-invoker": ">=1.1.0,<1.2.0" + }, + "bin": [ + "composer/bin/phpunit" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.7.x-dev" + } + }, + "autoload": { + "classmap": [ + "PHPUnit/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "", + "../../symfony/yaml/" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "The PHP Unit Testing framework.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "phpunit", + "testing", + "xunit" + ], + "time": "2013-03-07 21:45:39" + }, + { + "name": "phpunit/phpunit-mock-objects", + "version": "1.2.3", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/phpunit-mock-objects.git", + "reference": "1.2.3" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/phpunit-mock-objects/archive/1.2.3.zip", + "reference": "1.2.3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3", + "phpunit/php-text-template": ">=1.1.1@stable" + }, + "suggest": { + "ext-soap": "*" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHPUnit/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Mock Object library for PHPUnit", + "homepage": "https://github.com/sebastianbergmann/phpunit-mock-objects/", + "keywords": [ + "mock", + "xunit" + ], + "time": "2013-01-13 10:24:48" + }, + { + "name": "symfony/yaml", + "version": "v2.2.0", + "target-dir": "Symfony/Component/Yaml", + "source": { + "type": "git", + "url": "https://github.com/symfony/Yaml.git", + "reference": "v2.2.0-RC3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/Yaml/zipball/v2.2.0-RC3", + "reference": "v2.2.0-RC3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.2-dev" + } + }, + "autoload": { + "psr-0": { + "Symfony\\Component\\Yaml\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "http://symfony.com/contributors" + } + ], + "description": "Symfony Yaml Component", + "homepage": "http://symfony.com", + "time": "2013-01-27 16:49:19" } ], - "packages-dev": null, "aliases": [ ], "minimum-stability": "stable", "stability-flags": { "colinmollenhour/credis": 20 - } + }, + "platform": { + "php": ">=5.3.0" + }, + "platform-dev": [ + + ] } diff --git a/demo/bad_job.php b/demo/bad_job.php index bc12620..cd719cc 100644 --- a/demo/bad_job.php +++ b/demo/bad_job.php @@ -5,5 +5,4 @@ class Bad_PHP_Job { throw new Exception('Unable to run this job!'); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/check_status.php b/demo/check_status.php index 061a83a..645bf6d 100644 --- a/demo/check_status.php +++ b/demo/check_status.php @@ -17,5 +17,4 @@ echo "Tracking status of ".$argv[1].". Press [break] to stop.\n\n"; while(true) { fwrite(STDOUT, "Status of ".$argv[1]." is: ".$status->get()."\n"); sleep(1); -} -?> +} \ No newline at end of file diff --git a/demo/job.php b/demo/job.php index ec3d143..ddda893 100644 --- a/demo/job.php +++ b/demo/job.php @@ -7,5 +7,4 @@ class PHP_Job sleep(1); fwrite(STDOUT, 'Job ended!' . PHP_EOL); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/long_job.php b/demo/long_job.php index 8c9f0f9..1cfe5cb 100644 --- a/demo/long_job.php +++ b/demo/long_job.php @@ -5,5 +5,4 @@ class Long_PHP_Job { sleep(600); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/php_error_job.php b/demo/php_error_job.php index 93bf2bc..9824405 100644 --- a/demo/php_error_job.php +++ b/demo/php_error_job.php @@ -5,5 +5,4 @@ class PHP_Error_Job { callToUndefinedFunction(); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/queue.php b/demo/queue.php index 393df87..52f2f0b 100644 --- a/demo/queue.php +++ b/demo/queue.php @@ -15,5 +15,4 @@ $args = array( ); $jobId = Resque::enqueue($argv[1], $argv[2], $args, true); -echo "Queued job ".$jobId."\n\n"; -?> \ No newline at end of file +echo "Queued job ".$jobId."\n\n"; \ No newline at end of file diff --git a/demo/resque.php b/demo/resque.php index cdaf756..3dfa5a9 100644 --- a/demo/resque.php +++ b/demo/resque.php @@ -4,5 +4,4 @@ require 'bad_job.php'; require 'job.php'; require 'php_error_job.php'; -require '../bin/resque'; -?> \ No newline at end of file +require '../bin/resque.php'; \ No newline at end of file diff --git a/lib/Resque.php b/lib/Resque.php index 2e459f0..d65680b 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -122,20 +122,21 @@ class Resque } /** - * Pop an item off the end of the specified queue, decode it and - * return it. + * Pop an item off the end of the specified queues, using blocking list pop, + * decode it and return it. * - * @param string $queue The name of the queue to fetch an item from. - * @return array Decoded item from the queue. + * @param array $queues + * @param int $timeout + * @return null|array Decoded item from the queue. */ - public static function blpop($queues, $interval = null) + public static function blpop(array $queues, $timeout) { $list = array(); foreach($queues AS $queue) { $list[] = 'queue:' . $queue; } - $item = self::redis()->blpop($list, $interval ? (int)$interval : Resque::DEFAULT_INTERVAL); + $item = self::redis()->blpop($list, (int)$timeout); if(!$item) { return; @@ -186,9 +187,9 @@ class Resque * @param string $queue Queue to fetch next available job from. * @return Resque_Job Instance of Resque_Job to be processed, false if none or error. */ - public static function reserve($queue, $interval = null) + public static function reserve($queue) { - return Resque_Job::reserve($queue, $interval); + return Resque_Job::reserve($queue); } /** diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 0b1f01e..ff1dd5c 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -89,21 +89,20 @@ class Resque_Job } /** - * Find the next available job from the specified queue and return an - * instance of Resque_Job for it. + * Find the next available job from the specified queues using blocking list pop + * and return an instance of Resque_Job for it. * - * @param string $queue The name of the queue to check for a job in. + * @param array $queues + * @param int $timeout * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. */ - public static function reserveBlocking($queues, $interval = null) + public static function reserveBlocking(array $queues, $timeout = null) { - $payload = Resque::blpop($queues, $interval); + $payload = Resque::blpop($queues, $timeout); if(!is_array($payload)) { return false; } - var_dump($payload); - return new Resque_Job($payload->queue, $payload); } diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 03487f5..3a730d8 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -147,13 +147,52 @@ class Resque_Worker * * @param int $interval How often to check for new jobs across the queues. */ - public function work($interval = Resque::DEFAULT_INTERVAL) + public function work($interval = Resque::DEFAULT_INTERVAL, $blocking = false) { $this->updateProcLine('Starting'); $this->startup(); - while($job = $this->reserveBlocking($interval)) { - $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + while(true) { + if($this->shutdown) { + break; + } + + // Attempt to find and reserve a job + $job = false; + if(!$this->paused) { + if($blocking === true) { + $this->log('Starting blocking with timeout of ' . $interval, self::LOG_VERBOSE); + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with blocking timeout ' . $interval); + } else { + $this->log('Starting with interval of ' . $interval, self::LOG_VERBOSE); + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + } + + $job = $this->reserve($blocking, $interval); + } + + if(!$job) { + // For an interval of 0, break now - helps with unit testing etc + if($interval == 0) { + break; + } + + if($blocking === false) + { + // If no job was found, we sleep for $interval before continuing and checking again + $this->log('Sleeping for ' . $interval, self::LOG_VERBOSE); + if($this->paused) { + $this->updateProcLine('Paused'); + } + else { + $this->updateProcLine('Waiting for ' . implode(',', $this->queues)); + } + + usleep($interval * 1000000); + } + + continue; + } $this->log('got ' . $job); Resque_Event::trigger('beforeFork', $job); @@ -190,14 +229,6 @@ class Resque_Worker $this->child = null; $this->doneWorking(); - - if($this->shutdown) { - break; - } - - if($this->paused) { - break; - } } $this->unregisterWorker(); @@ -225,44 +256,32 @@ class Resque_Worker } /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. + * @param bool $blocking + * @param int $timeout + * @return object|boolean Instance of Resque_Job if a job is found, false if not. */ - public function reserve() + public function reserve($blocking = false, $timeout = null) { $queues = $this->queues(); if(!is_array($queues)) { return; } - foreach($queues as $queue) { - $this->log('Checking ' . $queue, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue); + + if($blocking === true) { + $job = Resque_Job::reserveBlocking($queues, $timeout); if($job) { - $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); return $job; } - } - - return false; - } - - /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. - */ - public function reserveBlocking($interval = null) - { - $queues = $this->queues(); - if(!is_array($queues)) { - return; - } - - $job = Resque_Job::reserveBlocking($queues, $interval); - if($job) { - $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); - return $job; + } else { + foreach($queues as $queue) { + $this->log('Checking ' . $queue, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue); + if($job) { + $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + return $job; + } + } } return false; diff --git a/test/Resque/Tests/JobStatusTest.php b/test/Resque/Tests/JobStatusTest.php index 3a12b4b..68a25ff 100644 --- a/test/Resque/Tests/JobStatusTest.php +++ b/test/Resque/Tests/JobStatusTest.php @@ -8,6 +8,11 @@ */ class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase { + /** + * @var \Resque_Worker + */ + protected $worker; + public function setUp() { parent::setUp(); @@ -36,6 +41,7 @@ class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase $status = new Resque_Job_Status($token); $this->assertEquals(Resque_Job_Status::STATUS_WAITING, $status->get()); } + public function testRunningJobReturnsRunningStatus() { $token = Resque::enqueue('jobs', 'Failing_Job', null, true); From 66fec608a1efc499c151e5c7790626fc235ed44f Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Wed, 13 Mar 2013 13:03:19 +0100 Subject: [PATCH 06/15] Fixed unit test --- lib/Resque/Worker.php | 1 - test/Resque/Tests/WorkerTest.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 3a730d8..870d4c8 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -164,7 +164,6 @@ class Resque_Worker $this->log('Starting blocking with timeout of ' . $interval, self::LOG_VERBOSE); $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with blocking timeout ' . $interval); } else { - $this->log('Starting with interval of ' . $interval, self::LOG_VERBOSE); $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); } diff --git a/test/Resque/Tests/WorkerTest.php b/test/Resque/Tests/WorkerTest.php index b0001c9..1ab7dd2 100644 --- a/test/Resque/Tests/WorkerTest.php +++ b/test/Resque/Tests/WorkerTest.php @@ -257,7 +257,7 @@ class Resque_Tests_WorkerTest extends Resque_Tests_TestCase Resque::enqueue('jobs', 'Test_Job_2'); $i = 1; - while($job = $worker->reserve(60)) + while($job = $worker->reserve(true, 1)) { $this->assertEquals('Test_Job_' . $i, $job->payload['class']); From 4d5552867a30a314939209ba6b161b60db5c1d5e Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Fri, 24 Aug 2012 16:51:03 +0200 Subject: [PATCH 07/15] Basic support for blocking list pop --- lib/Resque.php | 16 ++++++++++++---- lib/Resque/Job.php | 6 +++--- lib/Resque/Redis.php | 1 + lib/Resque/Worker.php | 42 +++++++++++++++++++++--------------------- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/lib/Resque.php b/lib/Resque.php index a463bea..7478fa4 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -10,6 +10,8 @@ class Resque { const VERSION = '1.2'; + const DEFAULT_INTERVAL = 5; + /** * @var Resque_Redis Instance of Resque_Redis that talks to redis. */ @@ -112,14 +114,19 @@ class Resque * @param string $queue The name of the queue to fetch an item from. * @return array Decoded item from the queue. */ - public static function pop($queue) + public static function pop($queue, $interval = null) { - $item = self::redis()->lpop('queue:' . $queue); + if($interval == null) { + $item = self::redis()->lpop('queue:' . $queue); + } else { + $item = self::redis()->blpop('queue:' . $queue, $interval ?: Resque::DEFAULT_INTERVAL); + } + if(!$item) { return; } - return json_decode($item, true); + return json_decode($interval == 0 ? $item : $item[1], true); } /** @@ -164,8 +171,9 @@ class Resque * @param string $queue Queue to fetch next available job from. * @return Resque_Job Instance of Resque_Job to be processed, false if none or error. */ - public static function reserve($queue) + public static function reserve($queue, $interval = null) { + require_once dirname(__FILE__) . '/Resque/Job.php'; return Resque_Job::reserve($queue); } diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 8cedd4b..618b118 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -80,9 +80,9 @@ class Resque_Job * @param string $queue The name of the queue to check for a job in. * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. */ - public static function reserve($queue) + public static function reserve($queue, $interval = null) { - $payload = Resque::pop($queue); + $payload = Resque::pop($queue, $interval); if(!is_array($payload)) { return false; } @@ -153,7 +153,7 @@ class Resque_Job ); } - $this->instance = new $this->payload['class'](); + $this->instance = new $this->payload['class']; $this->instance->job = $this; $this->instance->args = $this->getArguments(); $this->instance->queue = $this->queue; diff --git a/lib/Resque/Redis.php b/lib/Resque/Redis.php index bbb0643..4447146 100644 --- a/lib/Resque/Redis.php +++ b/lib/Resque/Redis.php @@ -47,6 +47,7 @@ class Resque_Redis 'lset', 'lrem', 'lpop', + 'blpop', 'rpop', 'sadd', 'srem', diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 45852bb..204aae6 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -147,7 +147,7 @@ class Resque_Worker * * @param int $interval How often to check for new jobs across the queues. */ - public function work($interval = 5) + public function work($interval = Resque::DEFAULT_INTERVAL) { $this->updateProcLine('Starting'); $this->startup(); @@ -160,25 +160,25 @@ class Resque_Worker // Attempt to find and reserve a job $job = false; if(!$this->paused) { - $job = $this->reserve(); + $job = $this->reserve($interval); } - if(!$job) { - // For an interval of 0, break now - helps with unit testing etc - if($interval == 0) { - break; - } - // If no job was found, we sleep for $interval before continuing and checking again - $this->log('Sleeping for ' . $interval, self::LOG_VERBOSE); - if($this->paused) { - $this->updateProcLine('Paused'); - } - else { - $this->updateProcLine('Waiting for ' . implode(',', $this->queues)); - } - usleep($interval * 1000000); - continue; - } + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + + if(!$job) { + // For an interval of 0, break now - helps with unit testing etc + if($interval == 0) { + break; + } + + // If no job was found, we sleep for $interval before continuing and checking again + if($this->paused) { + $this->updateProcLine('Paused'); + usleep($interval * 1000000); //it's paused, so don't hog redis with requests. + } + + continue; + } $this->log('got ' . $job); Resque_Event::trigger('beforeFork', $job); @@ -246,15 +246,15 @@ class Resque_Worker * * @return object|boolean Instance of Resque_Job if a job is found, false if not. */ - public function reserve() + public function reserve($interval = null) { $queues = $this->queues(); if(!is_array($queues)) { return; } foreach($queues as $queue) { - $this->log('Checking ' . $queue, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue); + $this->log('Checking ' . $queue . ' with interval ' . $interval, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue, $interval); if($job) { $this->log('Found job on ' . $queue, self::LOG_VERBOSE); return $job; From 15324ca410ebc963e116a9b3815b94dbc026d388 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Mon, 10 Sep 2012 10:23:18 +0200 Subject: [PATCH 08/15] Fixed bug --- lib/Resque.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Resque.php b/lib/Resque.php index 7478fa4..928ef5e 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -119,7 +119,7 @@ class Resque if($interval == null) { $item = self::redis()->lpop('queue:' . $queue); } else { - $item = self::redis()->blpop('queue:' . $queue, $interval ?: Resque::DEFAULT_INTERVAL); + $item = self::redis()->blpop('queue:' . $queue, $interval ? $interval : Resque::DEFAULT_INTERVAL); } if(!$item) { From e541fa9b70c944b9a7f90fc636531db6c88c2e54 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Tue, 12 Mar 2013 09:55:03 +0100 Subject: [PATCH 09/15] Added test for blpop --- test/Resque/Tests/WorkerTest.php | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/Resque/Tests/WorkerTest.php b/test/Resque/Tests/WorkerTest.php index b2f0e00..b0001c9 100644 --- a/test/Resque/Tests/WorkerTest.php +++ b/test/Resque/Tests/WorkerTest.php @@ -247,4 +247,27 @@ class Resque_Tests_WorkerTest extends Resque_Tests_TestCase $this->assertEquals(1, Resque_Stat::get('failed')); } + + public function testBlockingListPop() + { + $worker = new Resque_Worker('jobs'); + $worker->registerWorker(); + + Resque::enqueue('jobs', 'Test_Job_1'); + Resque::enqueue('jobs', 'Test_Job_2'); + + $i = 1; + while($job = $worker->reserve(60)) + { + $this->assertEquals('Test_Job_' . $i, $job->payload['class']); + + if($i == 2) { + break; + } + + $i++; + } + + $this->assertEquals(2, $i); + } } \ No newline at end of file From b8f98eecd2f4cfad157c28a7a26f6122d679ef64 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Tue, 12 Mar 2013 11:18:37 +0100 Subject: [PATCH 10/15] WIP --- bin/resque | 1 - demo/job.php | 5 +- demo/queue.php | 2 +- lib/Resque.php | 33 +++++++-- lib/Resque/Job.php | 49 ++++++++++---- lib/Resque/Redis.php | 8 ++- lib/Resque/Worker.php | 154 ++++++++++++++++++++++-------------------- 7 files changed, 150 insertions(+), 102 deletions(-) diff --git a/bin/resque b/bin/resque index 2b605ce..fbae329 100644 --- a/bin/resque +++ b/bin/resque @@ -1,4 +1,3 @@ -#!/usr/bin/env php '); + sleep(1); + fwrite(STDOUT, 'Job ended!' . PHP_EOL); } } ?> \ No newline at end of file diff --git a/demo/queue.php b/demo/queue.php index 95ff5b0..393df87 100644 --- a/demo/queue.php +++ b/demo/queue.php @@ -14,6 +14,6 @@ $args = array( ), ); -$jobId = Resque::enqueue('default', $argv[1], $args, true); +$jobId = Resque::enqueue($argv[1], $argv[2], $args, true); echo "Queued job ".$jobId."\n\n"; ?> \ No newline at end of file diff --git a/lib/Resque.php b/lib/Resque.php index 928ef5e..c7ddf4c 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -114,21 +114,40 @@ class Resque * @param string $queue The name of the queue to fetch an item from. * @return array Decoded item from the queue. */ - public static function pop($queue, $interval = null) + public static function pop($queue) { - if($interval == null) { - $item = self::redis()->lpop('queue:' . $queue); - } else { - $item = self::redis()->blpop('queue:' . $queue, $interval ? $interval : Resque::DEFAULT_INTERVAL); - } + $item = self::redis()->lpop('queue:' . $queue); if(!$item) { return; } - return json_decode($interval == 0 ? $item : $item[1], true); + return json_decode($item, true); } + /** + * Pop an item off the end of the specified queue, decode it and + * return it. + * + * @param string $queue The name of the queue to fetch an item from. + * @return array Decoded item from the queue. + */ + public static function blpop($queues, $interval = null) + { + $list = array(); + foreach($queues AS $queue) { + $list[] = 'queue:' . $queue; + } + + $item = self::redis()->blpop($list, $interval ? (int)$interval : Resque::DEFAULT_INTERVAL); + + if(!$item) { + return; + } + + return json_decode($item[1], true); + } + /** * Return the size (number of pending jobs) of the specified queue. * diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 618b118..e2fb60e 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -73,22 +73,41 @@ class Resque_Job return $id; } - /** - * Find the next available job from the specified queue and return an - * instance of Resque_Job for it. - * - * @param string $queue The name of the queue to check for a job in. - * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. - */ - public static function reserve($queue, $interval = null) - { - $payload = Resque::pop($queue, $interval); - if(!is_array($payload)) { - return false; - } + /** + * Find the next available job from the specified queue and return an + * instance of Resque_Job for it. + * + * @param string $queue The name of the queue to check for a job in. + * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. + */ + public static function reserve($queue) + { + $payload = Resque::pop($queue); + if(!is_array($payload)) { + return false; + } - return new Resque_Job($queue, $payload); - } + return new Resque_Job($queue, $payload); + } + + /** + * Find the next available job from the specified queue and return an + * instance of Resque_Job for it. + * + * @param string $queue The name of the queue to check for a job in. + * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. + */ + public static function reserveBlocking($queues, $interval = null) + { + $payload = Resque::blpop($queues, $interval); + if(!is_array($payload)) { + return false; + } + + var_dump($payload); + + return new Resque_Job($payload->queue, $payload); + } /** * Update the status of the current job. diff --git a/lib/Resque/Redis.php b/lib/Resque/Redis.php index 4447146..6cef3d2 100644 --- a/lib/Resque/Redis.php +++ b/lib/Resque/Redis.php @@ -143,7 +143,13 @@ class Resque_Redis */ public function __call($name, $args) { if(in_array($name, $this->keyCommands)) { - $args[0] = self::$defaultNamespace . $args[0]; + if(is_array($args[0])) { + foreach($args[0] AS $i => $v) { + $args[0][$i] = self::$defaultNamespace . $v; + } + } else { + $args[0] = self::$defaultNamespace . $args[0]; + } } try { return $this->driver->__call($name, $args); diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 204aae6..03487f5 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -152,70 +152,53 @@ class Resque_Worker $this->updateProcLine('Starting'); $this->startup(); - while(true) { - if($this->shutdown) { - break; - } - - // Attempt to find and reserve a job - $job = false; - if(!$this->paused) { - $job = $this->reserve($interval); - } - + while($job = $this->reserveBlocking($interval)) { $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); - if(!$job) { - // For an interval of 0, break now - helps with unit testing etc - if($interval == 0) { - break; - } + $this->log('got ' . $job); + Resque_Event::trigger('beforeFork', $job); + $this->workingOn($job); - // If no job was found, we sleep for $interval before continuing and checking again - if($this->paused) { - $this->updateProcLine('Paused'); - usleep($interval * 1000000); //it's paused, so don't hog redis with requests. - } + $this->child = Resque::fork(); - continue; + // Forked and we're the child. Run the job. + if ($this->child === 0 || $this->child === false) { + $status = 'Processing ' . $job->queue . ' since ' . strftime('%F %T'); + $this->updateProcLine($status); + $this->log($status, self::LOG_VERBOSE); + $this->perform($job); + if ($this->child === 0) { + exit(0); + } } - $this->log('got ' . $job); - Resque_Event::trigger('beforeFork', $job); - $this->workingOn($job); + if($this->child > 0) { + // Parent process, sit and wait + $status = 'Forked ' . $this->child . ' at ' . strftime('%F %T'); + $this->updateProcLine($status); + $this->log($status, self::LOG_VERBOSE); - $this->child = Resque::fork(); + // Wait until the child process finishes before continuing + pcntl_wait($status); + $exitStatus = pcntl_wexitstatus($status); + if($exitStatus !== 0) { + $job->fail(new Resque_Job_DirtyExitException( + 'Job exited with exit code ' . $exitStatus + )); + } + } - // Forked and we're the child. Run the job. - if ($this->child === 0 || $this->child === false) { - $status = 'Processing ' . $job->queue . ' since ' . strftime('%F %T'); - $this->updateProcLine($status); - $this->log($status, self::LOG_VERBOSE); - $this->perform($job); - if ($this->child === 0) { - exit(0); - } - } + $this->child = null; + $this->doneWorking(); - if($this->child > 0) { - // Parent process, sit and wait - $status = 'Forked ' . $this->child . ' at ' . strftime('%F %T'); - $this->updateProcLine($status); - $this->log($status, self::LOG_VERBOSE); + if($this->shutdown) { + break; + } - // Wait until the child process finishes before continuing - pcntl_wait($status); - $exitStatus = pcntl_wexitstatus($status); - if($exitStatus !== 0) { - $job->fail(new Resque_Job_DirtyExitException( - 'Job exited with exit code ' . $exitStatus - )); - } - } - - $this->child = null; - $this->doneWorking(); - } + if($this->paused) { + break; + } + } $this->unregisterWorker(); } @@ -241,28 +224,49 @@ class Resque_Worker $this->log('done ' . $job); } - /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. - */ - public function reserve($interval = null) - { - $queues = $this->queues(); - if(!is_array($queues)) { - return; - } - foreach($queues as $queue) { - $this->log('Checking ' . $queue . ' with interval ' . $interval, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue, $interval); - if($job) { - $this->log('Found job on ' . $queue, self::LOG_VERBOSE); - return $job; - } - } + /** + * Attempt to find a job from the top of one of the queues for this worker. + * + * @return object|boolean Instance of Resque_Job if a job is found, false if not. + */ + public function reserve() + { + $queues = $this->queues(); + if(!is_array($queues)) { + return; + } + foreach($queues as $queue) { + $this->log('Checking ' . $queue, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue); + if($job) { + $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + return $job; + } + } - return false; - } + return false; + } + + /** + * Attempt to find a job from the top of one of the queues for this worker. + * + * @return object|boolean Instance of Resque_Job if a job is found, false if not. + */ + public function reserveBlocking($interval = null) + { + $queues = $this->queues(); + if(!is_array($queues)) { + return; + } + + $job = Resque_Job::reserveBlocking($queues, $interval); + if($job) { + $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); + return $job; + } + + return false; + } /** * Return an array containing all of the queues that this worker should use From 86ae77811d6515a120ea01b4a33761b61da61413 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Wed, 13 Mar 2013 12:41:32 +0100 Subject: [PATCH 11/15] Working blocking list pop :) --- bin/resque | 1 + bin/resque.php | 105 +++++++ composer.lock | 416 +++++++++++++++++++++++++++- demo/bad_job.php | 3 +- demo/check_status.php | 3 +- demo/job.php | 3 +- demo/long_job.php | 3 +- demo/php_error_job.php | 3 +- demo/queue.php | 3 +- demo/resque.php | 3 +- lib/Resque.php | 16 +- lib/Resque/Job.php | 13 +- lib/Resque/Worker.php | 99 ++++--- test/Resque/Tests/JobStatusTest.php | 6 + 14 files changed, 604 insertions(+), 73 deletions(-) mode change 100644 => 100755 bin/resque create mode 100644 bin/resque.php diff --git a/bin/resque b/bin/resque old mode 100644 new mode 100755 index fbae329..2b605ce --- a/bin/resque +++ b/bin/resque @@ -1,3 +1,4 @@ +#!/usr/bin/env php 1) { + $count = $COUNT; +} + +if($count > 1) { + for($i = 0; $i < $count; ++$i) { + $pid = Resque::fork(); + if($pid == -1) { + die("Could not fork worker ".$i."\n"); + } + // Child, start the worker + else if(!$pid) { + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); + break; + } + } +} +// Start a single worker +else { + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; + + $PIDFILE = getenv('PIDFILE'); + if ($PIDFILE) { + file_put_contents($PIDFILE, getmypid()) or + die('Could not write PID information to ' . $PIDFILE); + } + + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); +} \ No newline at end of file diff --git a/composer.lock b/composer.lock index 877c243..760d4cf 100644 --- a/composer.lock +++ b/composer.lock @@ -18,7 +18,6 @@ "require": { "php": ">=5.3.0" }, - "time": "2013-01-12 10:15:31", "type": "library", "autoload": { "classmap": [ @@ -39,15 +38,424 @@ "homepage": "https://github.com/colinmollenhour/credis", "support": { "source": "https://github.com/chrisboulton/credis/tree/master" - } + }, + "time": "2013-01-12 10:15:31" + } + ], + "packages-dev": [ + { + "name": "phpunit/php-code-coverage", + "version": "1.2.9", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-code-coverage.git", + "reference": "1.2.9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/1.2.9", + "reference": "1.2.9", + "shasum": "" + }, + "require": { + "php": ">=5.3.3", + "phpunit/php-file-iterator": ">=1.3.0@stable", + "phpunit/php-text-template": ">=1.1.1@stable", + "phpunit/php-token-stream": ">=1.1.3@stable" + }, + "suggest": { + "ext-dom": "*", + "ext-xdebug": ">=2.0.5" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", + "homepage": "https://github.com/sebastianbergmann/php-code-coverage", + "keywords": [ + "coverage", + "testing", + "xunit" + ], + "time": "2013-02-26 18:55:56" + }, + { + "name": "phpunit/php-file-iterator", + "version": "1.3.3", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-file-iterator.git", + "reference": "1.3.3" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-file-iterator/zipball/1.3.3", + "reference": "1.3.3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "File/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "FilterIterator implementation that filters files based on a list of suffixes.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "filesystem", + "iterator" + ], + "time": "2012-10-11 04:44:38" + }, + { + "name": "phpunit/php-text-template", + "version": "1.1.4", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-text-template.git", + "reference": "1.1.4" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-text-template/zipball/1.1.4", + "reference": "1.1.4", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "Text/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Simple template engine.", + "homepage": "https://github.com/sebastianbergmann/php-text-template/", + "keywords": [ + "template" + ], + "time": "2012-10-31 11:15:28" + }, + { + "name": "phpunit/php-timer", + "version": "1.0.4", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-timer.git", + "reference": "1.0.4" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-timer/zipball/1.0.4", + "reference": "1.0.4", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Utility class for timing", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "timer" + ], + "time": "2012-10-11 04:45:58" + }, + { + "name": "phpunit/php-token-stream", + "version": "1.1.5", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/php-token-stream.git", + "reference": "1.1.5" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/php-token-stream/zipball/1.1.5", + "reference": "1.1.5", + "shasum": "" + }, + "require": { + "ext-tokenizer": "*", + "php": ">=5.3.3" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHP/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Wrapper around PHP's tokenizer extension.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "tokenizer" + ], + "time": "2012-10-11 04:47:14" + }, + { + "name": "phpunit/phpunit", + "version": "3.7.18", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/phpunit.git", + "reference": "3.7.18" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3.7.18", + "reference": "3.7.18", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-pcre": "*", + "ext-reflection": "*", + "ext-spl": "*", + "php": ">=5.3.3", + "phpunit/php-code-coverage": ">=1.2.1,<1.3.0", + "phpunit/php-file-iterator": ">=1.3.1", + "phpunit/php-text-template": ">=1.1.1", + "phpunit/php-timer": ">=1.0.2,<1.1.0", + "phpunit/phpunit-mock-objects": ">=1.2.0,<1.3.0", + "symfony/yaml": ">=2.2.0" + }, + "require-dev": { + "pear-pear/pear": "1.9.4" + }, + "suggest": { + "ext-json": "*", + "ext-simplexml": "*", + "ext-tokenizer": "*", + "phpunit/php-invoker": ">=1.1.0,<1.2.0" + }, + "bin": [ + "composer/bin/phpunit" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.7.x-dev" + } + }, + "autoload": { + "classmap": [ + "PHPUnit/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "", + "../../symfony/yaml/" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "The PHP Unit Testing framework.", + "homepage": "http://www.phpunit.de/", + "keywords": [ + "phpunit", + "testing", + "xunit" + ], + "time": "2013-03-07 21:45:39" + }, + { + "name": "phpunit/phpunit-mock-objects", + "version": "1.2.3", + "source": { + "type": "git", + "url": "git://github.com/sebastianbergmann/phpunit-mock-objects.git", + "reference": "1.2.3" + }, + "dist": { + "type": "zip", + "url": "https://github.com/sebastianbergmann/phpunit-mock-objects/archive/1.2.3.zip", + "reference": "1.2.3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3", + "phpunit/php-text-template": ">=1.1.1@stable" + }, + "suggest": { + "ext-soap": "*" + }, + "type": "library", + "autoload": { + "classmap": [ + "PHPUnit/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "" + ], + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sb@sebastian-bergmann.de", + "role": "lead" + } + ], + "description": "Mock Object library for PHPUnit", + "homepage": "https://github.com/sebastianbergmann/phpunit-mock-objects/", + "keywords": [ + "mock", + "xunit" + ], + "time": "2013-01-13 10:24:48" + }, + { + "name": "symfony/yaml", + "version": "v2.2.0", + "target-dir": "Symfony/Component/Yaml", + "source": { + "type": "git", + "url": "https://github.com/symfony/Yaml.git", + "reference": "v2.2.0-RC3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/Yaml/zipball/v2.2.0-RC3", + "reference": "v2.2.0-RC3", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.2-dev" + } + }, + "autoload": { + "psr-0": { + "Symfony\\Component\\Yaml\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "http://symfony.com/contributors" + } + ], + "description": "Symfony Yaml Component", + "homepage": "http://symfony.com", + "time": "2013-01-27 16:49:19" } ], - "packages-dev": null, "aliases": [ ], "minimum-stability": "stable", "stability-flags": { "colinmollenhour/credis": 20 - } + }, + "platform": { + "php": ">=5.3.0" + }, + "platform-dev": [ + + ] } diff --git a/demo/bad_job.php b/demo/bad_job.php index bc12620..cd719cc 100644 --- a/demo/bad_job.php +++ b/demo/bad_job.php @@ -5,5 +5,4 @@ class Bad_PHP_Job { throw new Exception('Unable to run this job!'); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/check_status.php b/demo/check_status.php index 061a83a..645bf6d 100644 --- a/demo/check_status.php +++ b/demo/check_status.php @@ -17,5 +17,4 @@ echo "Tracking status of ".$argv[1].". Press [break] to stop.\n\n"; while(true) { fwrite(STDOUT, "Status of ".$argv[1]." is: ".$status->get()."\n"); sleep(1); -} -?> +} \ No newline at end of file diff --git a/demo/job.php b/demo/job.php index ec3d143..ddda893 100644 --- a/demo/job.php +++ b/demo/job.php @@ -7,5 +7,4 @@ class PHP_Job sleep(1); fwrite(STDOUT, 'Job ended!' . PHP_EOL); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/long_job.php b/demo/long_job.php index 8c9f0f9..1cfe5cb 100644 --- a/demo/long_job.php +++ b/demo/long_job.php @@ -5,5 +5,4 @@ class Long_PHP_Job { sleep(600); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/php_error_job.php b/demo/php_error_job.php index 93bf2bc..9824405 100644 --- a/demo/php_error_job.php +++ b/demo/php_error_job.php @@ -5,5 +5,4 @@ class PHP_Error_Job { callToUndefinedFunction(); } -} -?> \ No newline at end of file +} \ No newline at end of file diff --git a/demo/queue.php b/demo/queue.php index 393df87..52f2f0b 100644 --- a/demo/queue.php +++ b/demo/queue.php @@ -15,5 +15,4 @@ $args = array( ); $jobId = Resque::enqueue($argv[1], $argv[2], $args, true); -echo "Queued job ".$jobId."\n\n"; -?> \ No newline at end of file +echo "Queued job ".$jobId."\n\n"; \ No newline at end of file diff --git a/demo/resque.php b/demo/resque.php index cdaf756..3dfa5a9 100644 --- a/demo/resque.php +++ b/demo/resque.php @@ -4,5 +4,4 @@ require 'bad_job.php'; require 'job.php'; require 'php_error_job.php'; -require '../bin/resque'; -?> \ No newline at end of file +require '../bin/resque.php'; \ No newline at end of file diff --git a/lib/Resque.php b/lib/Resque.php index c7ddf4c..b715767 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -126,20 +126,21 @@ class Resque } /** - * Pop an item off the end of the specified queue, decode it and - * return it. + * Pop an item off the end of the specified queues, using blocking list pop, + * decode it and return it. * - * @param string $queue The name of the queue to fetch an item from. - * @return array Decoded item from the queue. + * @param array $queues + * @param int $timeout + * @return null|array Decoded item from the queue. */ - public static function blpop($queues, $interval = null) + public static function blpop(array $queues, $timeout) { $list = array(); foreach($queues AS $queue) { $list[] = 'queue:' . $queue; } - $item = self::redis()->blpop($list, $interval ? (int)$interval : Resque::DEFAULT_INTERVAL); + $item = self::redis()->blpop($list, (int)$timeout); if(!$item) { return; @@ -190,9 +191,8 @@ class Resque * @param string $queue Queue to fetch next available job from. * @return Resque_Job Instance of Resque_Job to be processed, false if none or error. */ - public static function reserve($queue, $interval = null) + public static function reserve($queue) { - require_once dirname(__FILE__) . '/Resque/Job.php'; return Resque_Job::reserve($queue); } diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index e2fb60e..7d90d0a 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -91,21 +91,20 @@ class Resque_Job } /** - * Find the next available job from the specified queue and return an - * instance of Resque_Job for it. + * Find the next available job from the specified queues using blocking list pop + * and return an instance of Resque_Job for it. * - * @param string $queue The name of the queue to check for a job in. + * @param array $queues + * @param int $timeout * @return null|object Null when there aren't any waiting jobs, instance of Resque_Job when a job was found. */ - public static function reserveBlocking($queues, $interval = null) + public static function reserveBlocking(array $queues, $timeout = null) { - $payload = Resque::blpop($queues, $interval); + $payload = Resque::blpop($queues, $timeout); if(!is_array($payload)) { return false; } - var_dump($payload); - return new Resque_Job($payload->queue, $payload); } diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 03487f5..3a730d8 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -147,13 +147,52 @@ class Resque_Worker * * @param int $interval How often to check for new jobs across the queues. */ - public function work($interval = Resque::DEFAULT_INTERVAL) + public function work($interval = Resque::DEFAULT_INTERVAL, $blocking = false) { $this->updateProcLine('Starting'); $this->startup(); - while($job = $this->reserveBlocking($interval)) { - $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + while(true) { + if($this->shutdown) { + break; + } + + // Attempt to find and reserve a job + $job = false; + if(!$this->paused) { + if($blocking === true) { + $this->log('Starting blocking with timeout of ' . $interval, self::LOG_VERBOSE); + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with blocking timeout ' . $interval); + } else { + $this->log('Starting with interval of ' . $interval, self::LOG_VERBOSE); + $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); + } + + $job = $this->reserve($blocking, $interval); + } + + if(!$job) { + // For an interval of 0, break now - helps with unit testing etc + if($interval == 0) { + break; + } + + if($blocking === false) + { + // If no job was found, we sleep for $interval before continuing and checking again + $this->log('Sleeping for ' . $interval, self::LOG_VERBOSE); + if($this->paused) { + $this->updateProcLine('Paused'); + } + else { + $this->updateProcLine('Waiting for ' . implode(',', $this->queues)); + } + + usleep($interval * 1000000); + } + + continue; + } $this->log('got ' . $job); Resque_Event::trigger('beforeFork', $job); @@ -190,14 +229,6 @@ class Resque_Worker $this->child = null; $this->doneWorking(); - - if($this->shutdown) { - break; - } - - if($this->paused) { - break; - } } $this->unregisterWorker(); @@ -225,44 +256,32 @@ class Resque_Worker } /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. + * @param bool $blocking + * @param int $timeout + * @return object|boolean Instance of Resque_Job if a job is found, false if not. */ - public function reserve() + public function reserve($blocking = false, $timeout = null) { $queues = $this->queues(); if(!is_array($queues)) { return; } - foreach($queues as $queue) { - $this->log('Checking ' . $queue, self::LOG_VERBOSE); - $job = Resque_Job::reserve($queue); + + if($blocking === true) { + $job = Resque_Job::reserveBlocking($queues, $timeout); if($job) { - $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); return $job; } - } - - return false; - } - - /** - * Attempt to find a job from the top of one of the queues for this worker. - * - * @return object|boolean Instance of Resque_Job if a job is found, false if not. - */ - public function reserveBlocking($interval = null) - { - $queues = $this->queues(); - if(!is_array($queues)) { - return; - } - - $job = Resque_Job::reserveBlocking($queues, $interval); - if($job) { - $this->log('Found job on ' . $job->queue, self::LOG_VERBOSE); - return $job; + } else { + foreach($queues as $queue) { + $this->log('Checking ' . $queue, self::LOG_VERBOSE); + $job = Resque_Job::reserve($queue); + if($job) { + $this->log('Found job on ' . $queue, self::LOG_VERBOSE); + return $job; + } + } } return false; diff --git a/test/Resque/Tests/JobStatusTest.php b/test/Resque/Tests/JobStatusTest.php index 3a12b4b..68a25ff 100644 --- a/test/Resque/Tests/JobStatusTest.php +++ b/test/Resque/Tests/JobStatusTest.php @@ -8,6 +8,11 @@ */ class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase { + /** + * @var \Resque_Worker + */ + protected $worker; + public function setUp() { parent::setUp(); @@ -36,6 +41,7 @@ class Resque_Tests_JobStatusTest extends Resque_Tests_TestCase $status = new Resque_Job_Status($token); $this->assertEquals(Resque_Job_Status::STATUS_WAITING, $status->get()); } + public function testRunningJobReturnsRunningStatus() { $token = Resque::enqueue('jobs', 'Failing_Job', null, true); From f379d3a36d8685d72b86b3b740a85fcb17ee880d Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Wed, 13 Mar 2013 13:03:19 +0100 Subject: [PATCH 12/15] Fixed unit test --- lib/Resque/Worker.php | 1 - test/Resque/Tests/WorkerTest.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/Resque/Worker.php b/lib/Resque/Worker.php index 3a730d8..870d4c8 100644 --- a/lib/Resque/Worker.php +++ b/lib/Resque/Worker.php @@ -164,7 +164,6 @@ class Resque_Worker $this->log('Starting blocking with timeout of ' . $interval, self::LOG_VERBOSE); $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with blocking timeout ' . $interval); } else { - $this->log('Starting with interval of ' . $interval, self::LOG_VERBOSE); $this->updateProcLine('Waiting for ' . implode(',', $this->queues) . ' with interval ' . $interval); } diff --git a/test/Resque/Tests/WorkerTest.php b/test/Resque/Tests/WorkerTest.php index b0001c9..1ab7dd2 100644 --- a/test/Resque/Tests/WorkerTest.php +++ b/test/Resque/Tests/WorkerTest.php @@ -257,7 +257,7 @@ class Resque_Tests_WorkerTest extends Resque_Tests_TestCase Resque::enqueue('jobs', 'Test_Job_2'); $i = 1; - while($job = $worker->reserve(60)) + while($job = $worker->reserve(true, 1)) { $this->assertEquals('Test_Job_' . $i, $job->payload['class']); From bfbff5e58e92e96778c108086cf96df44bc9c5d6 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Wed, 29 May 2013 09:29:24 +0200 Subject: [PATCH 13/15] Unit tests pass --- lib/Resque.php | 12 +++++++++++- lib/Resque/Job.php | 7 ++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/Resque.php b/lib/Resque.php index b715767..3ec80f7 100644 --- a/lib/Resque.php +++ b/lib/Resque.php @@ -146,7 +146,17 @@ class Resque return; } - return json_decode($item[1], true); + /** + * Normally the Resque_Redis class returns queue names without the prefix + * But the blpop is a bit different. It returns the name as prefix:queue:name + * So we need to strip off the prefix:queue: part + */ + $queue = substr($item[0], strlen(self::redis()->getPrefix() . 'queue:')); + + return array( + 'queue' => $queue, + 'payload' => json_decode($item[1], true) + ); } /** diff --git a/lib/Resque/Job.php b/lib/Resque/Job.php index 7d90d0a..63cb203 100755 --- a/lib/Resque/Job.php +++ b/lib/Resque/Job.php @@ -100,12 +100,13 @@ class Resque_Job */ public static function reserveBlocking(array $queues, $timeout = null) { - $payload = Resque::blpop($queues, $timeout); - if(!is_array($payload)) { + $item = Resque::blpop($queues, $timeout); + + if(!is_array($item)) { return false; } - return new Resque_Job($payload->queue, $payload); + return new Resque_Job($item['queue'], $item['payload']); } /** From ffff86c40ca68764fef0151a47ae711c7326b215 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Mon, 3 Jun 2013 18:09:54 +0200 Subject: [PATCH 14/15] Fixes bin/resque --- bin/resque | 106 +++++++++++++++++++++++++------------------------ bin/resque.php | 105 ------------------------------------------------ 2 files changed, 54 insertions(+), 157 deletions(-) delete mode 100644 bin/resque.php diff --git a/bin/resque b/bin/resque index 2b605ce..f0f1429 100755 --- a/bin/resque +++ b/bin/resque @@ -3,40 +3,40 @@ // Find and initialize Composer $files = array( - __DIR__ . '/../../vendor/autoload.php', - __DIR__ . '/../../../autoload.php', - __DIR__ . '/../../../../autoload.php', - __DIR__ . '/../vendor/autoload.php', + __DIR__ . '/../../vendor/autoload.php', + __DIR__ . '/../../../autoload.php', + __DIR__ . '/../../../../autoload.php', + __DIR__ . '/../vendor/autoload.php', ); $found = false; foreach ($files as $file) { - if (file_exists($file)) { - require_once $file; - break; - } + if (file_exists($file)) { + require_once $file; + break; + } } if (!class_exists('Composer\Autoload\ClassLoader', false)) { - die( - 'You need to set up the project dependencies using the following commands:' . PHP_EOL . - 'curl -s http://getcomposer.org/installer | php' . PHP_EOL . - 'php composer.phar install' . PHP_EOL - ); + die( + 'You need to set up the project dependencies using the following commands:' . PHP_EOL . + 'curl -s http://getcomposer.org/installer | php' . PHP_EOL . + 'php composer.phar install' . PHP_EOL + ); } $QUEUE = getenv('QUEUE'); if(empty($QUEUE)) { - die("Set QUEUE env var containing the list of queues to work.\n"); + die("Set QUEUE env var containing the list of queues to work.\n"); } $REDIS_BACKEND = getenv('REDIS_BACKEND'); $REDIS_BACKEND_DB = getenv('REDIS_BACKEND_DB'); if(!empty($REDIS_BACKEND)) { - if (empty($REDIS_BACKEND_DB)) - Resque::setBackend($REDIS_BACKEND); - else - Resque::setBackend($REDIS_BACKEND, $REDIS_BACKEND_DB); + if (empty($REDIS_BACKEND_DB)) + Resque::setBackend($REDIS_BACKEND); + else + Resque::setBackend($REDIS_BACKEND, $REDIS_BACKEND_DB); } $logLevel = 0; @@ -44,31 +44,33 @@ $LOGGING = getenv('LOGGING'); $VERBOSE = getenv('VERBOSE'); $VVERBOSE = getenv('VVERBOSE'); if(!empty($LOGGING) || !empty($VERBOSE)) { - $logLevel = Resque_Worker::LOG_NORMAL; + $logLevel = Resque_Worker::LOG_NORMAL; } else if(!empty($VVERBOSE)) { - $logLevel = Resque_Worker::LOG_VERBOSE; + $logLevel = Resque_Worker::LOG_VERBOSE; } $APP_INCLUDE = getenv('APP_INCLUDE'); if($APP_INCLUDE) { - if(!file_exists($APP_INCLUDE)) { - die('APP_INCLUDE ('.$APP_INCLUDE.") does not exist.\n"); - } + if(!file_exists($APP_INCLUDE)) { + die('APP_INCLUDE ('.$APP_INCLUDE.") does not exist.\n"); + } - require_once $APP_INCLUDE; + require_once $APP_INCLUDE; } +$BLOCKING = getenv('BLOCKING') !== FALSE; + $interval = 5; $INTERVAL = getenv('INTERVAL'); if(!empty($INTERVAL)) { - $interval = $INTERVAL; + $interval = $INTERVAL; } $count = 1; $COUNT = getenv('COUNT'); if(!empty($COUNT) && $COUNT > 1) { - $count = $COUNT; + $count = $COUNT; } $PREFIX = getenv('PREFIX'); @@ -78,35 +80,35 @@ if(!empty($PREFIX)) { } if($count > 1) { - for($i = 0; $i < $count; ++$i) { - $pid = Resque::fork(); - if($pid == -1) { - die("Could not fork worker ".$i."\n"); - } - // Child, start the worker - else if(!$pid) { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval); - break; - } - } + for($i = 0; $i < $count; ++$i) { + $pid = Resque::fork(); + if($pid == -1) { + die("Could not fork worker ".$i."\n"); + } + // Child, start the worker + else if(!$pid) { + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); + break; + } + } } // Start a single worker else { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; + $queues = explode(',', $QUEUE); + $worker = new Resque_Worker($queues); + $worker->logLevel = $logLevel; - $PIDFILE = getenv('PIDFILE'); - if ($PIDFILE) { - file_put_contents($PIDFILE, getmypid()) or - die('Could not write PID information to ' . $PIDFILE); - } + $PIDFILE = getenv('PIDFILE'); + if ($PIDFILE) { + file_put_contents($PIDFILE, getmypid()) or + die('Could not write PID information to ' . $PIDFILE); + } - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval); + fwrite(STDOUT, '*** Starting worker '.$worker."\n"); + $worker->work($interval, $BLOCKING); } -?> +?> \ No newline at end of file diff --git a/bin/resque.php b/bin/resque.php deleted file mode 100644 index 3fc788f..0000000 --- a/bin/resque.php +++ /dev/null @@ -1,105 +0,0 @@ - 1) { - $count = $COUNT; -} - -if($count > 1) { - for($i = 0; $i < $count; ++$i) { - $pid = Resque::fork(); - if($pid == -1) { - die("Could not fork worker ".$i."\n"); - } - // Child, start the worker - else if(!$pid) { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval, $BLOCKING); - break; - } - } -} -// Start a single worker -else { - $queues = explode(',', $QUEUE); - $worker = new Resque_Worker($queues); - $worker->logLevel = $logLevel; - - $PIDFILE = getenv('PIDFILE'); - if ($PIDFILE) { - file_put_contents($PIDFILE, getmypid()) or - die('Could not write PID information to ' . $PIDFILE); - } - - fwrite(STDOUT, '*** Starting worker '.$worker."\n"); - $worker->work($interval, $BLOCKING); -} \ No newline at end of file From a0eaac47094de2bd0f451fa3e3a3fa2caa3c9c55 Mon Sep 17 00:00:00 2001 From: Ruud Kamphuis Date: Mon, 3 Jun 2013 18:10:14 +0200 Subject: [PATCH 15/15] Updated credis dependency to 1.2.* (fixes a bug in blpop) --- composer.json | 2 +- composer.lock | 31 +++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index d94ef1a..6ea9b0d 100644 --- a/composer.json +++ b/composer.json @@ -20,7 +20,7 @@ "require": { "php": ">=5.3.0", "ext-pcntl": "*", - "colinmollenhour/credis": "dev-master" + "colinmollenhour/credis": "1.2.*" }, "suggest": { "ext-proctitle": "Allows php-resque to rename the title of UNIX processes to show the status of a worker.", diff --git a/composer.lock b/composer.lock index 760d4cf..7140535 100644 --- a/composer.lock +++ b/composer.lock @@ -1,18 +1,22 @@ { - "hash": "d37909ad0ffc11ed4d1e67dcaabe00b2", + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file" + ], + "hash": "1f551c3cdade1b7ff7d9e32a44eeb3dc", "packages": [ { "name": "colinmollenhour/credis", - "version": "dev-master", + "version": "1.2", "source": { "type": "git", - "url": "https://github.com/chrisboulton/credis", - "reference": "62c73dd16e08069e3fd8f224cb4a5ddd73db8095" + "url": "https://github.com/colinmollenhour/credis.git", + "reference": "1.2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/chrisboulton/credis/zipball/62c73dd16e08069e3fd8f224cb4a5ddd73db8095", - "reference": "62c73dd16e08069e3fd8f224cb4a5ddd73db8095", + "url": "https://api.github.com/repos/colinmollenhour/credis/zipball/1.2", + "reference": "1.2", "shasum": "" }, "require": { @@ -25,6 +29,7 @@ "Cluster.php" ] }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], @@ -36,10 +41,7 @@ ], "description": "Credis is a lightweight interface to the Redis key-value store which wraps the phpredis library when available for better performance.", "homepage": "https://github.com/colinmollenhour/credis", - "support": { - "source": "https://github.com/chrisboulton/credis/tree/master" - }, - "time": "2013-01-12 10:15:31" + "time": "2013-03-19 02:57:04" } ], "packages-dev": [ @@ -449,11 +451,12 @@ ], "minimum-stability": "stable", - "stability-flags": { - "colinmollenhour/credis": 20 - }, + "stability-flags": [ + + ], "platform": { - "php": ">=5.3.0" + "php": ">=5.3.0", + "ext-pcntl": "*" }, "platform-dev": [