Add statsd collection for installs and workers

composer.json

@@ -56,7 +56,8 @@
         "seld/signal-handler": "^1.1",
         "php-http/httplug-bundle": "^1.11",
         "php-http/guzzle6-adapter": "^1.1",
-        "zendframework/zenddiagnostics": "^1.4"
+        "zendframework/zenddiagnostics": "^1.4",
+        "graze/dog-statsd": "^0.4.2"
     },
     "require-dev": {
         "symfony/phpunit-bridge": "^4.2",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "35167d88e0cf8bedad8b6820e3c5946e",
+    "content-hash": "bb9e56d2f9c92be63842385864ff2290",
     "packages": [
         {
             "name": "algolia/algoliasearch-client-php",
@@ -1539,6 +1539,63 @@
             ],
             "time": "2018-03-08T08:59:27+00:00"
         },
+        {
+            "name": "graze/dog-statsd",
+            "version": "0.4.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/graze/dog-statsd.git",
+                "reference": "94fe0c0d649384f13527eff556136726f6c9803e"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/graze/dog-statsd/zipball/94fe0c0d649384f13527eff556136726f6c9803e",
+                "reference": "94fe0c0d649384f13527eff556136726f6c9803e",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^5.5|^7.0"
+            },
+            "require-dev": {
+                "graze/standards": "^2",
+                "johnkary/phpunit-speedtrap": "^1|^2|^3",
+                "phpunit/phpunit": "^5.7.21|^6|^7",
+                "squizlabs/php_codesniffer": "^3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Graze\\DogStatsD\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Harry Bragg",
+                    "role": "Developer",
+                    "email": "harry.bragg@graze.com"
+                },
+                {
+                    "name": "Graze Developers",
+                    "role": "Development Team",
+                    "email": "developers@graze.com",
+                    "homepage": "http://www.graze.com"
+                }
+            ],
+            "description": "DataDog StatsD Client",
+            "homepage": "https://github.com/graze/dog-statsd",
+            "keywords": [
+                "DataDog",
+                "dog-statsd",
+                "dogstatsd",
+                "graze",
+                "statsd"
+            ],
+            "time": "2019-07-29T16:45:12+00:00"
+        },
         {
             "name": "guzzlehttp/guzzle",
             "version": "6.3.3",

src/Packagist/WebBundle/Controller/ApiController.php

@@ -14,6 +14,7 @@ namespace Packagist\WebBundle\Controller;
 
 use Packagist\WebBundle\Entity\Package;
 use Packagist\WebBundle\Entity\User;
+use Packagist\WebBundle\Util\UserAgentParser;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
@@ -226,6 +227,16 @@ class ApiController extends Controller
 
         if ($jobs) {
             $this->get('packagist.download_manager')->addDownloads($jobs);
+
+            $uaParser = new UserAgentParser($request->headers->get('User-Agent'));
+            $this->get('Graze\DogStatsD\Client')->increment('installs', 1, 1, [
+                'os' => $uaParser->getOs() ?: 'unknown',
+                'composer' => $uaParser->getComposerVersion() ?: 'unknown',
+                'php_minor' => preg_replace('{^(\d+\.\d+).*}', '$1', $uaParser->getPhpVersion()) ?: 'unknown',
+                'php_patch' => $uaParser->getPhpVersion() ?: 'unknown',
+                'http' => $uaParser->getHttpVersion() ?: 'unknown',
+                'ci' => $uaParser->getCI() ? 'true' : 'false',
+            ]);
         }
 
         if ($failed) {

src/Packagist/WebBundle/Resources/config/services.yml

@@ -32,6 +32,9 @@ services:
         tags:
             - { name: twig.extension }
 
+    Graze\DogStatsD\Client:
+        public: true
+
     twig.extension.text:
         public: true
         class: Twig_Extensions_Extension_Text

src/Packagist/WebBundle/Service/QueueWorker.php

@@ -7,6 +7,7 @@ use Psr\Log\LoggerInterface;
 use Symfony\Bridge\Doctrine\RegistryInterface;
 use Packagist\WebBundle\Entity\Job;
 use Seld\Signal\SignalHandler;
+use Graze\DogStatsD\Client as StatsDClient;
 
 class QueueWorker
 {
@@ -16,13 +17,16 @@ class QueueWorker
     private $doctrine;
     private $jobWorkers;
     private $processedJobs = 0;
+    /** @var StatsDClient */
+    private $statsd;
 
-    public function __construct(Redis $redis, RegistryInterface $doctrine, LoggerInterface $logger, array $jobWorkers)
+    public function __construct(Redis $redis, RegistryInterface $doctrine, LoggerInterface $logger, array $jobWorkers, StatsDClient $statsd)
     {
         $this->redis = $redis;
         $this->logger = $logger;
         $this->doctrine = $doctrine;
         $this->jobWorkers = $jobWorkers;
+        $this->statsd = $statsd;
     }
 
     /**
@@ -104,6 +108,12 @@ class QueueWorker
             return $record;
         });
 
+        $expectedStart = $job->getExecuteAfter() ?: $job->getCreatedAt();
+        $start = microtime(true);
+        $this->statsd->timing('worker.queue.waittime', round(($start - $expectedStart->getTimestamp()) * 1000, 4), [
+            'jobType' => $job->getType(),
+        ]);
+
         $processor = $this->jobWorkers[$job->getType()];
 
         $this->logger->reset();
@@ -119,6 +129,15 @@ class QueueWorker
             ];
         }
 
+        $this->statsd->increment('worker.queue.processed', 1, 1, [
+            'jobType' => $job->getType(),
+            'status' => $result['status'],
+        ] + (!empty($result['vendor']) ? ['vendor' => $result['vendor']] : []));
+
+        $this->statsd->timing('worker.queue.processtime', round((microtime(true) - $start) * 1000, 4), [
+            'jobType' => $job->getType(),
+        ] + (!empty($result['vendor']) ? ['vendor' => $result['vendor']] : []));
+
         // If an exception is thrown during a transaction the EntityManager is closed
         // and we won't be able to update the job or handle future jobs
         if (!$this->doctrine->getEntityManager()->isOpen()) {

src/Packagist/WebBundle/Service/UpdaterWorker.php

@@ -71,10 +71,11 @@ class UpdaterWorker
         }
 
         $packageName = $package->getName();
+        $packageVendor = $package->getVendor();
 
         $lockAcquired = $this->locker->lockPackageUpdate($id);
         if (!$lockAcquired) {
-            return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds')];
+            return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds'), 'vendor' => $packageVendor];
         }
 
         $this->logger->info('Updating '.$packageName);
@@ -156,6 +157,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, package appears to be gone',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -168,6 +170,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, invalid composer.json metadata',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -210,6 +213,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, package appears to be 404/gone and has been marked as crawled for 1year',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -218,7 +222,8 @@ class UpdaterWorker
                 return [
                     'status' => Job::STATUS_FAILED,
                     'message' => 'Package data of '.$packageName.' could not be downloaded. Could not reach remote VCS server. Please try again later.',
-                    'exception' => $e
+                    'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -227,7 +232,8 @@ class UpdaterWorker
                 return [
                     'status' => Job::STATUS_FAILED,
                     'message' => 'Package data of '.$packageName.' could not be downloaded.',
-                    'exception' => $e
+                    'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -242,7 +248,8 @@ class UpdaterWorker
         return [
             'status' => Job::STATUS_COMPLETED,
             'message' => 'Update of '.$packageName.' complete',
-            'details' => '<pre>'.$this->cleanupOutput($io->getOutput()).'</pre>'
+            'details' => '<pre>'.$this->cleanupOutput($io->getOutput()).'</pre>',
+            'vendor' => $packageVendor,
         ];
     }
 
@@ -275,6 +282,7 @@ class UpdaterWorker
                             'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been deleted',
                             'details' => '<pre>'.$output.'</pre>',
                             'exception' => $e,
+                            'vendor' => $package->getVendor()
                         ];
                     }
                 } catch (\Throwable $e) {

src/Packagist/WebBundle/Util/UserAgentParser.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Util;
+
+class UserAgentParser
+{
+    /** @var ?string */
+    private $composerVersion;
+    /** @var ?string */
+    private $phpVersion;
+    /** @var ?string */
+    private $os;
+    /** @var ?string */
+    private $httpVersion;
+    /** @var ?bool */
+    private $ci;
+
+    public function __construct(?string $userAgent)
+    {
+        if ($userAgent && preg_match('#^Composer/(?P<composer>[a-z0-9.+-]+) \((?P<os>\S+).*?; PHP (?P<php>[0-9.]+)[^;]*(?:; (?P<http>streams|curl [0-9.]+)[^;]*)?(?P<ci>; CI)?#i', $userAgent, $matches)) {
+            if ($matches['composer'] === 'source' || preg_match('{^[a-f0-9]{40}$}', $matches['composer'])) {
+                $matches['composer'] = 'pre-1.8.5';
+            }
+            $this->composerVersion = preg_replace('{\+[a-f0-9]{40}}', '', $matches['composer']);
+            $this->phpVersion = $matches['php'];
+            $this->os = $matches['os'];
+            $this->httpVersion = $matches['http'] ?? null;
+            $this->ci = (bool) ($matches['ci'] ?? null);
+        }
+    }
+
+    public function getComposerVersion(): ?string
+    {
+        return $this->composerVersion;
+    }
+
+    public function getPhpVersion(): ?string
+    {
+        return $this->phpVersion;
+    }
+
+    public function getOs(): ?string
+    {
+        return $this->os;
+    }
+
+    public function getHttpVersion(): ?string
+    {
+        return $this->httpVersion;
+    }
+
+    public function getCI(): ?bool
+    {
+        return $this->ci;
+    }
+}