This commit is contained in:
wangjinlei
2020-11-12 17:15:37 +08:00
parent 824380664c
commit 1abf99316f
893 changed files with 278997 additions and 0 deletions

View File

@@ -0,0 +1,49 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think;
use think\helper\Str;
use think\queue\Connector;
/**
* Class Queue
* @package think\queue
*
* @method static push($job, $data = '', $queue = null)
* @method static later($delay, $job, $data = '', $queue = null)
* @method static pop($queue = null)
* @method static marshal()
*/
class Queue
{
/** @var Connector */
protected static $connector;
private static function buildConnector()
{
$options = Config::get('queue');
$type = !empty($options['connector']) ? $options['connector'] : 'Sync';
if (!isset(self::$connector)) {
$class = false !== strpos($type, '\\') ? $type : '\\think\\queue\\connector\\' . Str::studly($type);
self::$connector = new $class($options);
}
return self::$connector;
}
public static function __callStatic($name, $arguments)
{
return call_user_func_array([self::buildConnector(), $name], $arguments);
}
}

View File

@@ -0,0 +1,36 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
\think\Console::addDefaultCommands([
"think\\queue\\command\\Work",
"think\\queue\\command\\Restart",
"think\\queue\\command\\Listen",
"think\\queue\\command\\Subscribe"
]);
if (!function_exists('queue')) {
/**
* 添加到队列
* @param $job
* @param string $data
* @param int $delay
* @param null $queue
*/
function queue($job, $data = '', $delay = 0, $queue = null)
{
if ($delay > 0) {
\think\Queue::later($delay, $job, $data, $queue);
} else {
\think\Queue::push($job, $data, $queue);
}
}
}

View File

@@ -0,0 +1,14 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
return [
'connector' => 'Sync'
];

View File

@@ -0,0 +1,36 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
class CallQueuedHandler
{
public function call(Job $job, array $data)
{
$command = unserialize($data['command']);
call_user_func([$command, 'handle']);
if (!$job->isDeletedOrReleased()) {
$job->delete();
}
}
public function failed(array $data)
{
$command = unserialize($data['command']);
if (method_exists($command, 'failed')) {
$command->failed();
}
}
}

View File

@@ -0,0 +1,69 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
use InvalidArgumentException;
abstract class Connector
{
protected $options = [];
abstract public function push($job, $data = '', $queue = null);
abstract public function later($delay, $job, $data = '', $queue = null);
abstract public function pop($queue = null);
public function marshal()
{
throw new \RuntimeException('pop queues not support for this type');
}
protected function createPayload($job, $data = '', $queue = null)
{
if (is_object($job)) {
$payload = json_encode([
'job' => 'think\queue\CallQueuedHandler@call',
'data' => [
'commandName' => get_class($job),
'command' => serialize(clone $job),
],
]);
} else {
$payload = json_encode($this->createPlainPayload($job, $data));
}
if (JSON_ERROR_NONE !== json_last_error()) {
throw new InvalidArgumentException('Unable to create payload: ' . json_last_error_msg());
}
return $payload;
}
protected function createPlainPayload($job, $data)
{
return ['job' => $job, 'data' => $data];
}
protected function setMeta($payload, $key, $value)
{
$payload = json_decode($payload, true);
$payload[$key] = $value;
$payload = json_encode($payload);
if (JSON_ERROR_NONE !== json_last_error()) {
throw new InvalidArgumentException('Unable to create payload: ' . json_last_error_msg());
}
return $payload;
}
}

View File

@@ -0,0 +1,213 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
use DateTime;
use think\Config;
abstract class Job
{
/**
* The job handler instance.
* @var mixed
*/
protected $instance;
/**
* The name of the queue the job belongs to.
* @var string
*/
protected $queue;
/**
* Indicates if the job has been deleted.
* @var bool
*/
protected $deleted = false;
/**
* Indicates if the job has been released.
* @var bool
*/
protected $released = false;
/**
* Fire the job.
* @return void
*/
abstract public function fire();
/**
* Delete the job from the queue.
* @return void
*/
public function delete()
{
$this->deleted = true;
}
/**
* Determine if the job has been deleted.
* @return bool
*/
public function isDeleted()
{
return $this->deleted;
}
/**
* Release the job back into the queue.
* @param int $delay
* @return void
*/
public function release($delay = 0)
{
$this->released = true;
}
/**
* Determine if the job was released back into the queue.
* @return bool
*/
public function isReleased()
{
return $this->released;
}
/**
* Determine if the job has been deleted or released.
* @return bool
*/
public function isDeletedOrReleased()
{
return $this->isDeleted() || $this->isReleased();
}
/**
* Get the number of times the job has been attempted.
* @return int
*/
abstract public function attempts();
/**
* Get the raw body string for the job.
* @return string
*/
abstract public function getRawBody();
/**
* Resolve and fire the job handler method.
* @param array $payload
* @return void
*/
protected function resolveAndFire(array $payload)
{
list($class, $method) = $this->parseJob($payload['job']);
$this->instance = $this->resolve($class);
if ($this->instance) {
$this->instance->{$method}($this, $payload['data']);
}
}
/**
* Parse the job declaration into class and method.
* @param string $job
* @return array
*/
protected function parseJob($job)
{
$segments = explode('@', $job);
return count($segments) > 1 ? $segments : [$segments[0], 'fire'];
}
/**
* Resolve the given job handler.
* @param string $name
* @return mixed
*/
protected function resolve($name)
{
if (strpos($name, '\\') === false) {
if (strpos($name, '/') === false) {
$module = '';
} else {
list($module, $name) = explode('/', $name, 2);
}
$name = Config::get('app_namespace') . ($module ? '\\' . strtolower($module) : '') . '\\job\\' . $name;
}
if (class_exists($name)) {
return new $name();
}
}
/**
* Call the failed method on the job instance.
* @return void
*/
public function failed()
{
$payload = json_decode($this->getRawBody(), true);
list($class, $method) = $this->parseJob($payload['job']);
$this->instance = $this->resolve($class);
if ($this->instance && method_exists($this->instance, 'failed')) {
$this->instance->failed($payload['data']);
}
}
/**
* Calculate the number of seconds with the given delay.
* @param \DateTime|int $delay
* @return int
*/
protected function getSeconds($delay)
{
if ($delay instanceof DateTime) {
return max(0, $delay->getTimestamp() - $this->getTime());
}
return (int) $delay;
}
/**
* Get the current system time.
* @return int
*/
protected function getTime()
{
return time();
}
/**
* Get the name of the queued job class.
* @return string
*/
public function getName()
{
return json_decode($this->getRawBody(), true)['job'];
}
/**
* Get the name of the queue the job belongs to.
* @return string
*/
public function getQueue()
{
return $this->queue;
}
}

View File

@@ -0,0 +1,164 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
use Closure;
use think\Process;
class Listener
{
/**
* @var string
*/
protected $commandPath;
/**
* @var int
*/
protected $sleep = 3;
/**
* @var int
*/
protected $maxTries = 0;
/**
* @var string
*/
protected $workerCommand;
/**
* @var \Closure|null
*/
protected $outputHandler;
/**
* @param string $commandPath
*/
public function __construct($commandPath)
{
$this->commandPath = $commandPath;
$this->workerCommand =
'"' . PHP_BINARY . '" think queue:work --queue="%s" --delay=%s --memory=%s --sleep=%s --tries=%s';
}
/**
* @param string $queue
* @param string $delay
* @param string $memory
* @param int $timeout
* @return void
*/
public function listen($queue, $delay, $memory, $timeout = 60)
{
$process = $this->makeProcess($queue, $delay, $memory, $timeout);
while (true) {
$this->runProcess($process, $memory);
}
}
/**
* @param \Think\Process $process
* @param int $memory
*/
public function runProcess(Process $process, $memory)
{
$process->run(function ($type, $line) {
$this->handleWorkerOutput($type, $line);
});
if ($this->memoryExceeded($memory)) {
$this->stop();
}
}
/**
* @param string $queue
* @param int $delay
* @param int $memory
* @param int $timeout
* @return \think\Process
*/
public function makeProcess($queue, $delay, $memory, $timeout)
{
$string = $this->workerCommand;
$command = sprintf($string, $queue, $delay, $memory, $this->sleep, $this->maxTries);
return new Process($command, $this->commandPath, null, null, $timeout);
}
/**
* @param int $type
* @param string $line
* @return void
*/
protected function handleWorkerOutput($type, $line)
{
if (isset($this->outputHandler)) {
call_user_func($this->outputHandler, $type, $line);
}
}
/**
* @param int $memoryLimit
* @return bool
*/
public function memoryExceeded($memoryLimit)
{
return (memory_get_usage() / 1024 / 1024) >= $memoryLimit;
}
/**
* @return void
*/
public function stop()
{
die;
}
/**
* @param \Closure $outputHandler
* @return void
*/
public function setOutputHandler(Closure $outputHandler)
{
$this->outputHandler = $outputHandler;
}
/**
* @return int
*/
public function getSleep()
{
return $this->sleep;
}
/**
* @param int $sleep
* @return void
*/
public function setSleep($sleep)
{
$this->sleep = $sleep;
}
/**
* @param int $tries
* @return void
*/
public function setMaxTries($tries)
{
$this->maxTries = $tries;
}
}

View File

@@ -0,0 +1,46 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
trait Queueable
{
/** @var string 队列名称 */
public $queue;
/** @var integer 延迟时间 */
public $delay;
/**
* 设置队列名
* @param $queue
* @return $this
*/
public function queue($queue)
{
$this->queue = $queue;
return $this;
}
/**
* 设置延迟时间
* @param $delay
* @return $this
*/
public function delay($delay)
{
$this->delay = $delay;
return $this;
}
}

View File

@@ -0,0 +1,17 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2016 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
interface ShouldQueue
{
}

View File

@@ -0,0 +1,119 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue;
use Exception;
use think\Hook;
use think\Queue;
class Worker
{
/**
* 执行下个任务
* @param string $queue
* @param int $delay
* @param int $sleep
* @param int $maxTries
* @return array
*/
public function pop($queue = null, $delay = 0, $sleep = 3, $maxTries = 0)
{
$job = $this->getNextJob($queue);
if (!is_null($job)) {
Hook::listen('worker_before_process', $queue);
return $this->process($job, $maxTries, $delay);
}
Hook::listen('worker_before_sleep', $queue);
$this->sleep($sleep);
return ['job' => null, 'failed' => false];
}
/**
* 获取下个任务
* @param string $queue
* @return Job
*/
protected function getNextJob($queue)
{
if (is_null($queue)) {
return Queue::pop();
}
foreach (explode(',', $queue) as $queue) {
if (!is_null($job = Queue::pop($queue))) {
return $job;
}
}
}
/**
* Process a given job from the queue.
* @param \think\queue\Job $job
* @param int $maxTries
* @param int $delay
* @return array
* @throws Exception
*/
public function process(Job $job, $maxTries = 0, $delay = 0)
{
if ($maxTries > 0 && $job->attempts() > $maxTries) {
return $this->logFailedJob($job);
}
try {
$job->fire();
return ['job' => $job, 'failed' => false];
} catch (Exception $e) {
if (!$job->isDeleted()) {
$job->release($delay);
}
throw $e;
}
}
/**
* Log a failed job into storage.
* @param \Think\Queue\Job $job
* @return array
*/
protected function logFailedJob(Job $job)
{
if (!$job->isDeleted()) {
try {
$job->delete();
$job->failed();
} finally {
Hook::listen('queue_failed', $job);
}
}
return ['job' => $job, 'failed' => true];
}
/**
* Sleep the script for a given number of seconds.
* @param int $seconds
* @return void
*/
public function sleep($seconds)
{
sleep($seconds);
}
}

View File

@@ -0,0 +1,65 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\command;
use think\console\Command;
use think\console\Input;
use think\console\input\Option;
use think\console\Output;
use think\queue\Listener;
class Listen extends Command
{
/** @var Listener */
protected $listener;
public function configure()
{
$this->setName('queue:listen')
->addOption('queue', null, Option::VALUE_OPTIONAL, 'The queue to listen on', null)
->addOption('delay', null, Option::VALUE_OPTIONAL, 'Amount of time to delay failed jobs', 0)
->addOption('memory', null, Option::VALUE_OPTIONAL, 'The memory limit in megabytes', 128)
->addOption('timeout', null, Option::VALUE_OPTIONAL, 'Seconds a job may run before timing out', 60)
->addOption('sleep', null, Option::VALUE_OPTIONAL, 'Seconds to wait before checking queue for jobs', 3)
->addOption('tries', null, Option::VALUE_OPTIONAL, 'Number of times to attempt a job before logging it failed', 0)
->setDescription('Listen to a given queue');
}
public function initialize(Input $input, Output $output)
{
$this->listener = new Listener($this->findCommandPath());
$this->listener->setSleep($input->getOption('sleep'));
$this->listener->setMaxTries($input->getOption('tries'));
$this->listener->setOutputHandler(function ($type, $line) use ($output) {
$output->write($line);
});
}
public function execute(Input $input, Output $output)
{
$delay = $input->getOption('delay');
$memory = $input->getOption('memory');
$timeout = $input->getOption('timeout');
$queue = $input->getOption('queue') ?: 'default';
$this->listener->listen($queue, $delay, $memory, $timeout);
}
protected function findCommandPath()
{
return defined('ROOT_PATH') ? ROOT_PATH : dirname($_SERVER['argv'][0]);
}
}

View File

@@ -0,0 +1,31 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\command;
use think\Cache;
use think\console\Command;
use think\console\Input;
use think\console\Output;
class Restart extends Command
{
public function configure()
{
$this->setName('queue:restart')->setDescription('Restart queue worker daemons after their current job');
}
public function execute(Input $input, Output $output)
{
Cache::set('think:queue:restart', time());
$output->writeln("<info>Broadcasting queue restart signal.</info>");
}
}

View File

@@ -0,0 +1,46 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\command;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\input\Option;
use think\console\Output;
use think\Queue;
use think\Url;
class Subscribe extends Command
{
public function configure()
{
$this->setName('queue:subscribe')
->setDescription('Subscribe a URL to an push queue')
->addArgument('name', Argument::REQUIRED, 'name')
->addArgument('url', Argument::REQUIRED, 'The URL to be subscribed.')
->addArgument('queue', Argument::OPTIONAL, 'The URL to be subscribed.')
->addOption('option', null, Option::VALUE_IS_ARRAY | Option::VALUE_OPTIONAL, 'the options');
}
public function execute(Input $input, Output $output)
{
$url = $input->getArgument('url');
if (!preg_match('/^https?:\/\//', $url)) {
$url = Url::build($url);
}
Queue::subscribe($input->getArgument('name'), $url, $input->getArgument('queue'), $input->getOption('option'));
$output->write('<info>Queue subscriber added:</info> <comment>' . $input->getArgument('url') . '</comment>');
}
}

View File

@@ -0,0 +1,210 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\command;
use think\Config;
use think\console\Command;
use think\console\Input;
use think\console\input\Option;
use think\console\Output;
use think\Hook;
use think\queue\Job;
use think\queue\Worker;
use Exception;
use Throwable;
use think\Cache;
use think\exception\Handle;
use think\exception\ThrowableError;
class Work extends Command
{
/**
* The queue worker instance.
* @var \think\queue\Worker
*/
protected $worker;
protected function initialize(Input $input, Output $output)
{
$this->worker = new Worker();
}
protected function configure()
{
$this->setName('queue:work')
->addOption('queue', null, Option::VALUE_OPTIONAL, 'The queue to listen on')
->addOption('daemon', null, Option::VALUE_NONE, 'Run the worker in daemon mode')
->addOption('delay', null, Option::VALUE_OPTIONAL, 'Amount of time to delay failed jobs', 0)
->addOption('force', null, Option::VALUE_NONE, 'Force the worker to run even in maintenance mode')
->addOption('memory', null, Option::VALUE_OPTIONAL, 'The memory limit in megabytes', 128)
->addOption('sleep', null, Option::VALUE_OPTIONAL, 'Number of seconds to sleep when no job is available', 3)
->addOption('tries', null, Option::VALUE_OPTIONAL, 'Number of times to attempt a job before logging it failed', 0)
->setDescription('Process the next job on a queue');
}
/**
* Execute the console command.
* @param Input $input
* @param Output $output
* @return int|null|void
*/
public function execute(Input $input, Output $output)
{
$queue = $input->getOption('queue');
$delay = $input->getOption('delay');
$memory = $input->getOption('memory');
if ($input->getOption('daemon')) {
Hook::listen('worker_daemon_start', $queue);
$this->daemon(
$queue, $delay, $memory,
$input->getOption('sleep'), $input->getOption('tries')
);
} else {
$response = $this->worker->pop($queue, $delay, $input->getOption('sleep'), $input->getOption('tries'));
$this->output($response);
}
}
protected function output($response)
{
if (!is_null($response['job'])) {
/** @var Job $job */
$job = $response['job'];
if ($response['failed']) {
$this->output->writeln('<error>Failed:</error> ' . $job->getName());
} else {
$this->output->writeln('<info>Processed:</info> ' . $job->getName());
}
}
}
/**
* 启动一个守护进程执行任务.
*
* @param string $queue
* @param int $delay
* @param int $memory
* @param int $sleep
* @param int $maxTries
* @return array
*/
protected function daemon($queue = null, $delay = 0, $memory = 128, $sleep = 3, $maxTries = 0)
{
$lastRestart = $this->getTimestampOfLastQueueRestart();
while (true) {
$this->runNextJobForDaemon(
$queue, $delay, $sleep, $maxTries
);
if ( $this->memoryExceeded($memory) ) {
Hook::listen('worker_memory_exceeded', $queue);
$this->stop();
}
if ( $this->queueShouldRestart($lastRestart) ) {
Hook::listen('worker_queue_restart', $queue);
$this->stop();
}
}
}
/**
* 以守护进程的方式执行下个任务.
*
* @param string $queue
* @param int $delay
* @param int $sleep
* @param int $maxTries
* @return void
*/
protected function runNextJobForDaemon($queue, $delay, $sleep, $maxTries)
{
try {
$response = $this->worker->pop($queue, $delay, $sleep, $maxTries);
$this->output($response);
} catch (Exception $e) {
$this->getExceptionHandler()->report($e);
} catch (Throwable $e) {
$this->getExceptionHandler()->report(new ThrowableError($e));
}
}
/**
* 获取上次重启守护进程的时间
*
* @return int|null
*/
protected function getTimestampOfLastQueueRestart()
{
return Cache::get('think:queue:restart');
}
/**
* 检查是否要重启守护进程
*
* @param int|null $lastRestart
* @return bool
*/
protected function queueShouldRestart($lastRestart)
{
return $this->getTimestampOfLastQueueRestart() != $lastRestart;
}
/**
* 检查内存是否超出
* @param int $memoryLimit
* @return bool
*/
protected function memoryExceeded($memoryLimit)
{
return (memory_get_usage() / 1024 / 1024) >= $memoryLimit;
}
/**
* 获取异常处理实例
*
* @return \think\exception\Handle
*/
protected function getExceptionHandler()
{
static $handle;
if (!$handle) {
if ($class = Config::get('exception_handle')) {
if (class_exists($class) && is_subclass_of($class, "\\think\\exception\\Handle")) {
$handle = new $class;
}
}
if (!$handle) {
$handle = new Handle();
}
}
return $handle;
}
/**
* 停止执行任务的守护进程.
* @return void
*/
public function stop()
{
die;
}
}

View File

@@ -0,0 +1,171 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\connector;
use think\Db;
use think\queue\Connector;
use think\queue\job\Database as DatabaseJob;
class Database extends Connector
{
protected $db;
protected $options = [
'expire' => 60,
'default' => 'default',
'table' => 'jobs',
'dsn' => []
];
public function __construct($options)
{
if (!empty($options)) {
$this->options = array_merge($this->options, $options);
}
$this->db = Db::connect($this->options['dsn']);
}
public function push($job, $data = '', $queue = null)
{
return $this->pushToDatabase(0, $queue, $this->createPayload($job, $data));
}
public function later($delay, $job, $data = '', $queue = null)
{
return $this->pushToDatabase($delay, $queue, $this->createPayload($job, $data));
}
public function pop($queue = null)
{
$queue = $this->getQueue($queue);
if (!is_null($this->options['expire'])) {
$this->releaseJobsThatHaveBeenReservedTooLong($queue);
}
if ($job = $this->getNextAvailableJob($queue)) {
$this->markJobAsReserved($job->id);
$this->db->commit();
return new DatabaseJob($this, $job, $queue);
}
$this->db->commit();
}
/**
* 重新发布任务
* @param string $queue
* @param \StdClass $job
* @param int $delay
* @return mixed
*/
public function release($queue, $job, $delay)
{
return $this->pushToDatabase($delay, $queue, $job->payload, $job->attempts);
}
/**
* Push a raw payload to the database with a given delay.
*
* @param \DateTime|int $delay
* @param string|null $queue
* @param string $payload
* @param int $attempts
* @return mixed
*/
protected function pushToDatabase($delay, $queue, $payload, $attempts = 0)
{
return $this->db->name($this->options['table'])->insert([
'queue' => $this->getQueue($queue),
'payload' => $payload,
'attempts' => $attempts,
'reserved' => 0,
'reserved_at' => null,
'available_at' => time() + $delay,
'created_at' => time()
]);
}
/**
* 获取下个有效任务
*
* @param string|null $queue
* @return \StdClass|null
*/
protected function getNextAvailableJob($queue)
{
$this->db->startTrans();
$job = $this->db->name($this->options['table'])
->lock(true)
->where('queue', $this->getQueue($queue))
->where('reserved', 0)
->where('available_at', '<=', time())
->order('id', 'asc')
->find();
return $job ? (object) $job : null;
}
/**
* 标记任务正在执行.
*
* @param string $id
* @return void
*/
protected function markJobAsReserved($id)
{
$this->db->name($this->options['table'])->where('id', $id)->update([
'reserved' => 1,
'reserved_at' => time()
]);
}
/**
* 重新发布超时的任务
*
* @param string $queue
* @return void
*/
protected function releaseJobsThatHaveBeenReservedTooLong($queue)
{
$expired = time() - $this->options['expire'];
$this->db->name($this->options['table'])
->where('queue', $this->getQueue($queue))
->where('reserved', 1)
->where('reserved_at', '<=', $expired)
->update([
'reserved' => 0,
'reserved_at' => null,
'attempts' => ['inc', 1]
]);
}
/**
* 删除任务
* @param string $id
* @return void
*/
public function deleteReserved($id)
{
$this->db->name($this->options['table'])->delete($id);
}
protected function getQueue($queue)
{
return $queue ?: $this->options['default'];
}
}

View File

@@ -0,0 +1,236 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\connector;
use Exception;
use think\helper\Str;
use think\queue\Connector;
use think\queue\job\Redis as RedisJob;
class Redis extends Connector
{
/** @var \Redis */
protected $redis;
protected $options = [
'expire' => 60,
'default' => 'default',
'host' => '127.0.0.1',
'port' => 6379,
'password' => '',
'select' => 0,
'timeout' => 0,
'persistent' => false
];
public function __construct($options)
{
if (!extension_loaded('redis')) {
throw new Exception('redis扩展未安装');
}
if (!empty($options)) {
$this->options = array_merge($this->options, $options);
}
$func = $this->options['persistent'] ? 'pconnect' : 'connect';
$this->redis = new \Redis;
$this->redis->$func($this->options['host'], $this->options['port'], $this->options['timeout']);
if ('' != $this->options['password']) {
$this->redis->auth($this->options['password']);
}
if (0 != $this->options['select']) {
$this->redis->select($this->options['select']);
}
}
public function push($job, $data = '', $queue = null)
{
return $this->pushRaw($this->createPayload($job, $data), $queue);
}
public function later($delay, $job, $data = '', $queue = null)
{
$payload = $this->createPayload($job, $data);
$this->redis->zAdd($this->getQueue($queue) . ':delayed', time() + $delay, $payload);
}
public function pop($queue = null)
{
$original = $queue ?: $this->options['default'];
$queue = $this->getQueue($queue);
$this->migrateExpiredJobs($queue . ':delayed', $queue, false);
if (!is_null($this->options['expire'])) {
$this->migrateExpiredJobs($queue . ':reserved', $queue);
}
$job = $this->redis->lPop($queue);
if ($job !== false) {
$this->redis->zAdd($queue . ':reserved', time() + $this->options['expire'], $job);
return new RedisJob($this, $job, $original);
}
}
/**
* 重新发布任务
*
* @param string $queue
* @param string $payload
* @param int $delay
* @param int $attempts
* @return void
*/
public function release($queue, $payload, $delay, $attempts)
{
$payload = $this->setMeta($payload, 'attempts', $attempts);
$this->redis->zAdd($this->getQueue($queue) . ':delayed', time() + $delay, $payload);
}
public function pushRaw($payload, $queue = null)
{
$this->redis->rPush($this->getQueue($queue), $payload);
return json_decode($payload, true)['id'];
}
protected function createPayload($job, $data = '', $queue = null)
{
$payload = $this->setMeta(
parent::createPayload($job, $data), 'id', $this->getRandomId()
);
return $this->setMeta($payload, 'attempts', 1);
}
/**
* 删除任务
*
* @param string $queue
* @param string $job
* @return void
*/
public function deleteReserved($queue, $job)
{
$this->redis->zRem($this->getQueue($queue) . ':reserved', $job);
}
/**
* 移动延迟任务
*
* @param string $from
* @param string $to
* @param bool $attempt
*/
public function migrateExpiredJobs($from, $to, $attempt = true)
{
$this->redis->watch($from);
$jobs = $this->getExpiredJobs(
$from, $time = time()
);
if (count($jobs) > 0) {
$this->transaction(function () use ($from, $to, $time, $jobs, $attempt) {
$this->removeExpiredJobs($from, $time);
$this->pushExpiredJobsOntoNewQueue($to, $jobs, $attempt);
});
}
$this->redis->unwatch();
}
/**
* redis事务
* @param \Closure $closure
*/
protected function transaction(\Closure $closure)
{
$this->redis->multi();
try {
call_user_func($closure);
if (!$this->redis->exec()) {
$this->redis->discard();
}
} catch (Exception $e) {
$this->redis->discard();
}
}
/**
* 获取所有到期任务
*
* @param string $from
* @param int $time
* @return array
*/
protected function getExpiredJobs($from, $time)
{
return $this->redis->zRangeByScore($from, '-inf', $time);
}
/**
* 删除过期任务
*
* @param string $from
* @param int $time
* @return void
*/
protected function removeExpiredJobs($from, $time)
{
$this->redis->zRemRangeByScore($from, '-inf', $time);
}
/**
* 重新发布到期任务
*
* @param string $to
* @param array $jobs
* @param boolean $attempt
*/
protected function pushExpiredJobsOntoNewQueue($to, $jobs, $attempt = true)
{
if ($attempt) {
foreach ($jobs as &$job) {
$attempts = json_decode($job, true)['attempts'];
$job = $this->setMeta($job, 'attempts', $attempts + 1);
}
}
call_user_func_array([$this->redis, 'rPush'], array_merge([$to], $jobs));
}
/**
* 随机id
*
* @return string
*/
protected function getRandomId()
{
return Str::random(32);
}
/**
* 获取队列名
*
* @param string|null $queue
* @return string
*/
protected function getQueue($queue)
{
return 'queues:' . ($queue ?: $this->options['default']);
}
}

View File

@@ -0,0 +1,57 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\connector;
use Exception;
use think\queue\Connector;
use think\queue\job\Sync as SyncJob;
use Throwable;
class Sync extends Connector
{
public function push($job, $data = '', $queue = null)
{
$queueJob = $this->resolveJob($this->createPayload($job, $data, $queue));
try {
set_time_limit(0);
$queueJob->fire();
} catch (Exception $e) {
$queueJob->failed();
throw $e;
} catch (Throwable $e) {
$queueJob->failed();
throw $e;
}
return 0;
}
public function later($delay, $job, $data = '', $queue = null)
{
return $this->push($job, $data, $queue);
}
public function pop($queue = null)
{
}
protected function resolveJob($payload)
{
return new SyncJob($payload);
}
}

View File

@@ -0,0 +1,225 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\connector;
use think\exception\HttpException;
use think\queue\Connector;
use think\Request;
use think\queue\job\Topthink as TopthinkJob;
use think\Response;
class Topthink extends Connector
{
protected $options = [
'token' => '',
'project_id' => '',
'protocol' => 'https',
'host' => 'qns.topthink.com',
'port' => 443,
'api_version' => 1,
'max_retries' => 3,
'default' => 'default'
];
/** @var Request */
protected $request;
protected $url;
protected $curl = null;
protected $last_status;
protected $headers = [];
public function __construct($options)
{
if (!empty($options)) {
$this->options = array_merge($this->options, $options);
}
$this->url = "{$this->options['protocol']}://{$this->options['host']}:{$this->options['port']}/v{$this->options['api_version']}/";
$this->headers['Authorization'] = "Bearer {$this->options['token']}";
$this->request = Request::instance();
}
public function push($job, $data = '', $queue = null)
{
return $this->pushRaw(0, $queue, $this->createPayload($job, $data));
}
public function later($delay, $job, $data = '', $queue = null)
{
return $this->pushRaw($delay, $queue, $this->createPayload($job, $data));
}
public function release($queue, $job, $delay)
{
return $this->pushRaw($delay, $queue, $job->payload, $job->attempts);
}
public function marshal()
{
$job = new TopthinkJob($this, $this->marshalPushedJob(), $this->request->header('topthink-message-queue'));
if ($this->request->header('topthink-message-status') == 'success') {
$job->fire();
} else {
$job->failed();
}
return new Response('OK');
}
public function pushRaw($delay, $queue, $payload, $attempts = 0)
{
$queue_name = $this->getQueue($queue);
$queue = rawurlencode($queue_name);
$url = "project/{$this->options['project_id']}/queue/{$queue}/message";
$message = [
'payload' => $payload,
'attempts' => $attempts,
'delay' => $delay
];
return $this->apiCall('POST', $url, $message)->id;
}
public function deleteMessage($queue, $id)
{
$queue = rawurlencode($queue);
$url = "project/{$this->options['project_id']}/queue/{$queue}/message/{$id}";
return $this->apiCall('DELETE', $url);
}
protected function apiCall($type, $url, $params = [])
{
$url = "{$this->url}$url";
if ($this->curl == null) {
$this->curl = curl_init();
}
switch ($type = strtoupper($type)) {
case 'DELETE':
curl_setopt($this->curl, CURLOPT_URL, $url);
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, $type);
curl_setopt($this->curl, CURLOPT_POSTFIELDS, json_encode($params));
break;
case 'PUT':
curl_setopt($this->curl, CURLOPT_URL, $url);
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, $type);
curl_setopt($this->curl, CURLOPT_POSTFIELDS, json_encode($params));
break;
case 'POST':
curl_setopt($this->curl, CURLOPT_URL, $url);
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, $type);
curl_setopt($this->curl, CURLOPT_POST, true);
curl_setopt($this->curl, CURLOPT_POSTFIELDS, $params);
break;
case 'GET':
curl_setopt($this->curl, CURLOPT_POSTFIELDS, null);
curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, $type);
curl_setopt($this->curl, CURLOPT_HTTPGET, true);
$url .= '?' . http_build_query($params);
curl_setopt($this->curl, CURLOPT_URL, $url);
break;
}
curl_setopt($this->curl, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($this->curl, CURLOPT_RETURNTRANSFER, true);
$headers = [];
foreach ($this->headers as $k => $v) {
if ($k == 'Connection') {
$v = 'Close';
}
$headers[] = "$k: $v";
}
curl_setopt($this->curl, CURLOPT_HTTPHEADER, $headers);
curl_setopt($this->curl, CURLOPT_CONNECTTIMEOUT, 10);
return $this->callWithRetries();
}
protected function callWithRetries()
{
for ($retry = 0; $retry < $this->options['max_retries']; $retry++) {
$out = curl_exec($this->curl);
if ($out === false) {
$this->reportHttpError(0, curl_error($this->curl));
}
$this->last_status = curl_getinfo($this->curl, CURLINFO_HTTP_CODE);
if ($this->last_status >= 200 && $this->last_status < 300) {
return self::jsonDecode($out);
} elseif ($this->last_status >= 500) {
self::waitRandomInterval($retry);
} else {
$this->reportHttpError($this->last_status, $out);
}
}
$this->reportHttpError($this->last_status, "Service unavailable");
return;
}
protected static function jsonDecode($response)
{
$data = json_decode($response);
$json_error = json_last_error();
if ($json_error != JSON_ERROR_NONE) {
throw new \RuntimeException($json_error);
}
return $data;
}
protected static function waitRandomInterval($retry)
{
$max_delay = pow(4, $retry) * 100 * 1000;
usleep(rand(0, $max_delay));
}
protected function reportHttpError($status, $text)
{
throw new HttpException($status, "http error: {$status} | {$text}");
}
/**
* Marshal out the pushed job and payload.
*
* @return object
*/
protected function marshalPushedJob()
{
return (object) [
'id' => $this->request->header('topthink-message-id'),
'payload' => $this->request->getContent(),
'attempts' => $this->request->header('topthink-message-attempts')
];
}
public function __destruct()
{
if ($this->curl != null) {
curl_close($this->curl);
$this->curl = null;
}
}
public function pop($queue = null)
{
throw new \RuntimeException('pop queues not support for this type');
}
}

View File

@@ -0,0 +1,88 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\job;
use think\queue\Job;
use think\queue\connector\Database as DatabaseQueue;
class Database extends Job
{
/**
* The database queue instance.
* @var DatabaseQueue
*/
protected $database;
/**
* The database job payload.
* @var Object
*/
protected $job;
public function __construct(DatabaseQueue $database, $job, $queue)
{
$this->job = $job;
$this->queue = $queue;
$this->database = $database;
$this->job->attempts = $this->job->attempts + 1;
}
/**
* 执行任务
* @return void
*/
public function fire()
{
$this->resolveAndFire(json_decode($this->job->payload, true));
}
/**
* 删除任务
* @return void
*/
public function delete()
{
parent::delete();
$this->database->deleteReserved($this->job->id);
}
/**
* 重新发布任务
* @param int $delay
* @return void
*/
public function release($delay = 0)
{
parent::release($delay);
$this->delete();
$this->database->release($this->queue, $this->job, $delay);
}
/**
* 获取当前任务尝试次数
* @return int
*/
public function attempts()
{
return (int) $this->job->attempts;
}
/**
* Get the raw body string for the job.
* @return string
*/
public function getRawBody()
{
return $this->job->payload;
}
}

View File

@@ -0,0 +1,92 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\job;
use think\queue\Job;
use think\queue\connector\Redis as RedisQueue;
class Redis extends Job
{
/**
* The redis queue instance.
* @var RedisQueue
*/
protected $redis;
/**
* The database job payload.
* @var Object
*/
protected $job;
public function __construct(RedisQueue $redis, $job, $queue)
{
$this->job = $job;
$this->queue = $queue;
$this->redis = $redis;
}
/**
* Fire the job.
* @return void
*/
public function fire()
{
$this->resolveAndFire(json_decode($this->getRawBody(), true));
}
/**
* Get the number of times the job has been attempted.
* @return int
*/
public function attempts()
{
return json_decode($this->job, true)['attempts'];
}
/**
* Get the raw body string for the job.
* @return string
*/
public function getRawBody()
{
return $this->job;
}
/**
* 删除任务
*
* @return void
*/
public function delete()
{
parent::delete();
$this->redis->deleteReserved($this->queue, $this->job);
}
/**
* 重新发布任务
*
* @param int $delay
* @return void
*/
public function release($delay = 0)
{
parent::release($delay);
$this->delete();
$this->redis->release($this->queue, $this->job, $delay, $this->attempts() + 1);
}
}

View File

@@ -0,0 +1,56 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\job;
use think\queue\Job;
class Sync extends Job
{
/**
* The queue message data.
*
* @var string
*/
protected $payload;
public function __construct($payload)
{
$this->payload = $payload;
}
/**
* Fire the job.
* @return void
*/
public function fire()
{
$this->resolveAndFire(json_decode($this->payload, true));
}
/**
* Get the number of times the job has been attempted.
* @return int
*/
public function attempts()
{
return 1;
}
/**
* Get the raw body string for the job.
* @return string
*/
public function getRawBody()
{
return $this->payload;
}
}

View File

@@ -0,0 +1,85 @@
<?php
// +----------------------------------------------------------------------
// | ThinkPHP [ WE CAN DO IT JUST THINK IT ]
// +----------------------------------------------------------------------
// | Copyright (c) 2006-2015 http://thinkphp.cn All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: yunwuxin <448901948@qq.com>
// +----------------------------------------------------------------------
namespace think\queue\job;
use think\queue\Job;
use think\queue\connector\Topthink as TopthinkQueue;
class Topthink extends Job
{
/**
* The Iron queue instance.
*
* @var TopthinkQueue
*/
protected $topthink;
/**
* The IronMQ message instance.
*
* @var object
*/
protected $job;
public function __construct(TopthinkQueue $topthink, $job, $queue)
{
$this->topthink = $topthink;
$this->job = $job;
$this->queue = $queue;
$this->job->attempts = $this->job->attempts + 1;
}
/**
* Fire the job.
* @return void
*/
public function fire()
{
$this->resolveAndFire(json_decode($this->job->payload, true));
}
/**
* Get the number of times the job has been attempted.
* @return int
*/
public function attempts()
{
return (int) $this->job->attempts;
}
public function delete()
{
parent::delete();
$this->topthink->deleteMessage($this->queue, $this->job->id);
}
public function release($delay = 0)
{
parent::release($delay);
$this->delete();
$this->topthink->release($this->queue, $this->job, $delay);
}
/**
* Get the raw body string for the job.
* @return string
*/
public function getRawBody()
{
return $this->job->payload;
}
}