Skip to content

Commit

Permalink
Merge pull request #4 from bwaidelich/master
Browse files Browse the repository at this point in the history 
FEATURE: Adjust to major overhaul of Flowpack.JobQueue.Common package
  • Loading branch information
Bastian Waidelich authored Jul 20, 2016
2 parents 66e35b0 + 9c2f4db commit b911c79
Show file tree
Hide file tree
Showing 5 changed files with 179 additions and 240 deletions.
209 changes: 92 additions & 117 deletions Classes/Queue/RedisQueue.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

use Flowpack\JobQueue\Common\Queue\Message;
use Flowpack\JobQueue\Common\Queue\QueueInterface;
use TYPO3\Flow\Utility\Algorithms;
use Flowpack\JobQueue\Common\Exception as JobQueueException;

/**
* A queue implementation using Redis as the queue backend
Expand Down Expand Up @@ -58,51 +60,49 @@ class RedisQueue implements QueueInterface
protected $maxReconnectDelay = 30.0;

/**
* Constructor
*
* @param string $name
* @param array $options
* @throws JobQueueException
*/
public function __construct($name, array $options = array())
public function __construct($name, array $options = [])
{
$this->name = $name;
if (isset($options['defaultTimeout'])) {
$this->defaultTimeout = (integer)$options['defaultTimeout'];
}
$this->clientOptions = isset($options['client']) ? $options['client'] : array();

$this->clientOptions = isset($options['client']) ? $options['client'] : [];
$this->client = new \Redis();
if (!$this->connectClient()) {
throw new \Flowpack\JobQueue\Common\Exception('Could not connect to Redis', 1467382685);
throw new JobQueueException('Could not connect to Redis', 1467382685);
}
}

/**
* Submit a message to the queue
*
* @param Message $message
* @return void
* @inheritdoc
*/
public function getName()
{
return $this->name;
}

/**
* @inheritdoc
*/
public function submit(Message $message)
public function submit($payload, array $options = [])
{
$this->checkClientConnection();
if ($message->getIdentifier() !== null) {
$added = $this->client->sAdd("queue:{$this->name}:ids", $message->getIdentifier());
if (!$added) {
return;
}
$messageId = Algorithms::generateUUID();
$idStored = $this->client->hSet("queue:{$this->name}:ids", $messageId, json_encode($payload));
if ($idStored === 0) {
return null;
}
$encodedMessage = $this->encodeMessage($message);
$this->client->lPush("queue:{$this->name}:messages", $encodedMessage);
$message->setState(Message::STATE_SUBMITTED);

$this->client->lPush("queue:{$this->name}:messages", $messageId);
return $messageId;
}

/**
* Wait for a message in the queue and return the message for processing
* (without safety queue)
*
* @param int $timeout
* @return Message The received message or NULL if a timeout occurred
* @inheritdoc
*/
public function waitAndTake($timeout = null)
{
Expand All @@ -111,149 +111,128 @@ public function waitAndTake($timeout = null)
}
$this->checkClientConnection();
$keyAndValue = $this->client->brPop("queue:{$this->name}:messages", $timeout);
$value = isset($keyAndValue[1]) ? $keyAndValue[1] : null;
if (is_string($value)) {
$message = $this->decodeMessage($value);

if ($message->getIdentifier() !== null) {
$this->client->sRem("queue:{$this->name}:ids", $message->getIdentifier());
}

// The message is marked as done
$message->setState(Message::STATE_DONE);

return $message;
} else {
$messageId = isset($keyAndValue[1]) ? $keyAndValue[1] : null;
if ($messageId === null) {
return null;
}
$message = $this->getMessageById($messageId);
if ($message !== null) {
$this->client->hDel("queue:{$this->name}:ids", $messageId);
}
return $message;
}

/**
* Wait for a message in the queue and save the message to a safety queue
*
* TODO: Idea for implementing a TTR (time to run) with monitoring of safety queue. E.g.
* use different queue names with encoded times? With "brpoplpush" we cannot modify the
* queued item on transfer to the safety queue and we cannot update a timestamp to mark
* the run start time in the message, so separate keys should be used for this.
*
* @param int $timeout
* @return Message
* @inheritdoc
*/
public function waitAndReserve($timeout = null)
{
if ($timeout === null) {
$timeout = $this->defaultTimeout;
}
$this->checkClientConnection();
$value = $this->client->brpoplpush("queue:{$this->name}:messages", "queue:{$this->name}:processing", $timeout);
if (is_string($value)) {
$message = $this->decodeMessage($value);
if ($message->getIdentifier() !== null) {
$this->client->sRem("queue:{$this->name}:ids", $message->getIdentifier());
}
return $message;
} else {
return null;
}
$messageId = $this->client->brpoplpush("queue:{$this->name}:messages", "queue:{$this->name}:processing", $timeout);
return $this->getMessageById($messageId);
}

/**
* Mark a message as finished
*
* @param Message $message
* @return boolean TRUE if the message could be removed
* @inheritdoc
*/
public function release($messageId, array $options = [])
{
$this->checkClientConnection();
$this->client->lRem("queue:{$this->name}:processing", $messageId, 0);
$numberOfReleases = (integer)$this->client->hGet("queue:{$this->name}:releases", $messageId);
$this->client->hSet("queue:{$this->name}:releases", $messageId, $numberOfReleases + 1);
$this->client->lPush("queue:{$this->name}:messages", $messageId);
}

/**
* @inheritdoc
*/
public function finish(Message $message)
public function abort($messageId)
{
$this->checkClientConnection();
$originalValue = $message->getOriginalValue();
$success = $this->client->lRem("queue:{$this->name}:processing", $originalValue, 0) > 0;
if ($success) {
$message->setState(Message::STATE_DONE);
$numberOfRemoved = $this->client->lRem("queue:{$this->name}:processing", $messageId, 0);
if ($numberOfRemoved === 1) {
$this->client->lPush("queue:{$this->name}:failed", $messageId);
}
return $success;
}

/**
* Peek for messages
*
* @param integer $limit
* @return Message[] Messages or empty array if no messages were present
* @inheritdoc
*/
public function finish($messageId)
{
$this->checkClientConnection();
$this->client->hDel("queue:{$this->name}:ids", $messageId);
$this->client->hDel("queue:{$this->name}:releases", $messageId);
return $this->client->lRem("queue:{$this->name}:processing", $messageId, 0) > 0;
}

/**
* @inheritdoc
*/
public function peek($limit = 1)
{
$this->checkClientConnection();
$result = $this->client->lRange("queue:{$this->name}:messages", -($limit), -1);
if (is_array($result) && count($result) > 0) {
$messages = array();
foreach ($result as $value) {
$message = $this->decodeMessage($value);
// The message is still submitted and should not be processed!
$message->setState(Message::STATE_SUBMITTED);
$messages[] = $message;
}
return $messages;
if (!is_array($result) || count($result) === 0) {
return [];
}
$messages = [];
foreach ($result as $messageId) {
$encodedPayload = $this->client->hGet("queue:{$this->name}:ids", $messageId);
$messages[] = new Message($messageId, json_decode($encodedPayload, true));
}
return array();
return $messages;
}

/**
* Count messages in the queue
*
* @return integer
* @inheritdoc
*/
public function count()
{
$this->checkClientConnection();
$count = $this->client->lLen("queue:{$this->name}:messages");
return $count;
return $this->client->lLen("queue:{$this->name}:messages");
}

/**
* Encode a message
*
* Updates the original value property of the message to resemble the
* encoded representation.
*
* @param Message $message
* @return string
* @return void
*/
protected function encodeMessage(Message $message)
public function setUp()
{
$value = json_encode($message->toArray());
$message->setOriginalValue($value);
return $value;
$this->checkClientConnection();
}

/**
* Decode a message from a string representation
*
* @param string $value
* @return Message
* @inheritdoc
*/
protected function decodeMessage($value)
public function flush()
{
$decodedMessage = json_decode($value, true);
$message = new Message($decodedMessage['payload']);
if (isset($decodedMessage['identifier'])) {
$message->setIdentifier($decodedMessage['identifier']);
}
$message->setOriginalValue($value);
return $message;
$this->checkClientConnection();
$this->client->flushDB();
}

/**
*
* @param string $identifier
* @param string $messageId
* @return Message
*/
public function getMessage($identifier)
protected function getMessageById($messageId)
{
return null;
if (!is_string($messageId)) {
return null;
}
$encodedPayload = $this->client->hGet("queue:{$this->name}:ids", $messageId);
$numberOfReleases = (integer)$this->client->hGet("queue:{$this->name}:releases", $messageId);
return new Message($messageId, json_decode($encodedPayload, true), $numberOfReleases);
}

/**
* Check if the Redis client connection is still up and reconnect if Redis was disconnected
*
* @return void
* @throws JobQueueException
*/
protected function checkClientConnection()
{
Expand All @@ -268,7 +247,7 @@ protected function checkClientConnection()
}
if ($reconnect) {
if (!$this->connectClient()) {
throw new \Flowpack\JobQueue\Common\Exception('Could not connect to Redis', 1467382685);
throw new JobQueueException('Could not connect to Redis', 1467382685);
}
}
}
Expand All @@ -286,20 +265,16 @@ protected function connectClient()
$host = isset($this->clientOptions['host']) ? $this->clientOptions['host'] : '127.0.0.1';
$port = isset($this->clientOptions['port']) ? $this->clientOptions['port'] : 6379;
$database = isset($this->clientOptions['database']) ? $this->clientOptions['database'] : 0;

// The connection read timeout should be higher than the timeout for blocking operations!
$timeout = isset($this->clientOptions['timeout']) ? $this->clientOptions['timeout'] : round($this->defaultTimeout * 1.5);
$connected = $this->client->connect($host, $port, $timeout) && $this->client->select($database);

// Break the cycle that could cause a high CPU load
if (!$connected) {
usleep($this->reconnectDelay * 1e6);
$this->reconnectDelay = min($this->reconnectDelay * $this->reconnectDecay, $this->maxReconnectDelay);
} else {
$this->reconnectDelay = 1.0;
}

return $connected;
}

}
23 changes: 23 additions & 0 deletions CodeOfConduct.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
Contributor Code of Conduct
---------------------------

As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.

We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.

Examples of unacceptable behavior by participants include:

* The use of sexualized language or imagery
* Personal attacks
* Trolling or insulting/derogatory comments
* Public or private harassment
* Publishing other's private information, such as physical or electronic addresses, without explicit permission
* Other unethical or unprofessional conduct.

Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team.

This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community.

Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.

This Code of Conduct is adapted from the `Contributor Covenant <http://contributor-covenant.org>`_, version 1.2.0, available at (http://contributor-covenant.org/version/1/2/0/
2 changes: 1 addition & 1 deletion LICENSE
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
The MIT License (MIT)

Copyright (c) 2015 Neos project contributors
Copyright (c) 2016 Neos project contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Expand Down
Loading

0 comments on commit b911c79

Please sign in to comment.