lastHeartbeatAt = time(); // Everything below sits inside the try so that the finally releases the lock // even if setup itself fails. Resolving the connection outside it would leave // the lock held until its TTL whenever connection resolution threw. try { $emptyPollThreshold = $this->emptyPollThreshold(); $popFailureThreshold = $this->popFailureThreshold(); $connection = $manager->connection(self::CONNECTION_NAME); $queueName = $this->queueName(); // WorkerOptions on 5.5 is ($delay, $memory, $timeout, $sleep, $maxTries, $force). // maxTries = 1 so a failing job is recorded in failed_jobs and deleted rather // than released back onto the queue, which would keep this loop alive forever. // delay must stay 0: Worker::handleJobException() releases an un-failed job with // release($options->delay), and a non-zero delay would hide that message from // the empty-poll counting below. $options = new WorkerOptions(0, 128, 60, 3, 1, false); Log::info('SQS drain started', [ 'connection' => self::CONNECTION_NAME, 'queue' => $queueName, 'empty_poll_threshold' => $emptyPollThreshold, ]); while (true) { try { $job = $connection->pop($queueName); $popFailures = 0; } catch (\Exception $e) { $popFailures++; Log::error('SQS drain failed to poll the queue', [ 'message' => $e->getMessage(), 'consecutive_failures' => $popFailures, ]); if ($popFailures >= $popFailureThreshold) { $exitCode = 1; break; } $this->pause(); $this->heartbeat(); continue; } if (is_null($job)) { $emptyPolls++; if ($emptyPolls >= $emptyPollThreshold) { break; } $this->pause(); $this->heartbeat(); continue; } $emptyPolls = 0; $processed++; try { // Must go through Worker::process() rather than $job->fire(): // process() raises JobProcessing/JobProcessed, which the // Queue::before hook in AppServiceProvider listens for. That hook // keys on $event->connectionName === 'sqs', hence the literal // connection name here. $worker->process(self::CONNECTION_NAME, $job, $options); Log::debug('SQS drain processed a job', [ 'job' => $this->jobName($job), 'processed' => $processed, ]); } catch (\Exception $e) { // Worker::handleJobException() always rethrows, so this catch is // what keeps one bad message from aborting the whole drain. The job // has already been failed and deleted by FailingJob at maxTries=1. Log::error('SQS drain job failed', [ 'job' => $this->jobName($job), 'processed' => $processed, 'message' => $e->getMessage(), ]); } $this->heartbeat(); } } finally { Cache::forget(self::LOCK_KEY); Log::info('SQS drain finished', [ 'processed' => $processed, 'elapsed_seconds' => round(microtime(true) - $startedAt, 2), 'exit_code' => $exitCode, ]); } return $exitCode; } /** * Wait between polls so an empty queue does not spin the CPU. * * Extracted as a seam so tests can drain without sleeping in real time. * * @return void */ protected function pause() { sleep(self::EMPTY_POLL_SLEEP_SECONDS); } /** * Refresh the lock so a long drain keeps holding it. * * Rate limited to one cache write per HEARTBEAT_SECONDS so that a drain * processing many fast jobs does not write on every iteration. * * @return void */ protected function heartbeat() { if ((time() - $this->lastHeartbeatAt) < self::HEARTBEAT_SECONDS) { return; } Cache::put(self::LOCK_KEY, true, self::LOCK_MINUTES); $this->lastHeartbeatAt = time(); } /** * @return int */ protected function emptyPollThreshold() { return (int) config('queue.sqs_drain.empty_poll_threshold', self::EMPTY_POLL_THRESHOLD) ?: self::EMPTY_POLL_THRESHOLD; } /** * @return int */ protected function popFailureThreshold() { return (int) config('queue.sqs_drain.pop_failure_threshold', self::POP_FAILURE_THRESHOLD) ?: self::POP_FAILURE_THRESHOLD; } /** * @return string|null */ protected function queueName() { return config('queue.connections.'.self::CONNECTION_NAME.'.queue'); } /** * Resolve a loggable name for a job. * * Deliberately the job class name only. The raw payload is never logged: * job bodies can carry sensitive application data. * * @param mixed $job * @return string */ protected function jobName($job) { try { return method_exists($job, 'resolveName') ? (string) $job->resolveName() : get_class($job); } catch (\Exception $e) { return 'unknown'; } } }