Merge branch 'master' into feature/dev-setup

README.md

@@ -41,5 +41,6 @@ These steps are provided for development purposes only.
    ```bash
    symfony serve
    ```
+6. Run a CRON job `app/console packagist:run-workers` to make sure packages update.
 
 You should now be able to access the site, create a user, etc.

app/Resources/FOSUserBundle/views/Profile/edit_content.html.twig

@@ -19,6 +19,7 @@
         </div>
     </div>
 
+    {% if not app.user.githubId %}
     <div class="form-group clearfix">
         {{ form_label(form.current_password) }}
         <div class="input-group">
@@ -27,6 +28,7 @@
             <span class="input-group-addon"><span class="icon-lock"></span></span>
         </div>
     </div>
+    {% endif %}
 
     <div class="notifications form-group">
         {{ form_errors(form.failureNotifications) }}

composer.json

@@ -57,6 +57,7 @@
         "php-http/httplug-bundle": "^1.11",
         "php-http/guzzle6-adapter": "^1.1",
         "zendframework/zenddiagnostics": "^1.4",
+        "graze/dog-statsd": "^0.4.2",
         "incenteev/composer-parameter-handler": "^2.1"
     },
     "require-dev": {

src/Packagist/WebBundle/Command/CompileStatsCommand.php

@@ -43,6 +43,16 @@ class CompileStatsCommand extends ContainerAwareCommand
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $locker = $this->getContainer()->get('locker');
+
+        $lockAcquired = $locker->lockCommand($this->getName());
+        if (!$lockAcquired) {
+            if ($input->getOption('verbose')) {
+                $output->writeln('Aborting, another task is running already');
+            }
+            return;
+        }
+
         $verbose = $input->getOption('verbose');
 
         $doctrine = $this->getContainer()->get('doctrine');
@@ -76,6 +86,8 @@ class CompileStatsCommand extends ContainerAwareCommand
 
         $redis->rename('downloads:trending:new', 'downloads:trending');
         $redis->rename('downloads:absolute:new', 'downloads:absolute');
+
+        $locker->unlockCommand($this->getName());
     }
 
     protected function sumLastNDays($days, $id, \DateTime $yesterday, $conn)

src/Packagist/WebBundle/Command/DumpPackagesCommand.php

@@ -16,7 +16,6 @@ use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Filesystem\LockHandler;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -54,12 +53,12 @@ class DumpPackagesCommand extends ContainerAwareCommand
         }
 
         // another dumper is still active
-        $lock = new LockHandler('packagist_package_dumper');
-        if (!$lock->lock()) {
+        $locker = $this->getContainer()->get('locker');
+        if (!$locker->lockCommand($this->getName())) {
             if ($verbose) {
-                $output->writeln('Aborting, another dumper is still active');
+                $output->writeln('Aborting, another task is running already');
             }
-            return;
+            return 0;
         }
 
         $doctrine = $this->getContainer()->get('doctrine');
@@ -85,9 +84,9 @@ class DumpPackagesCommand extends ContainerAwareCommand
         gc_enable();
 
         try {
-             $result = $this->getContainer()->get('packagist.package_dumper')->dump($ids, $force, $verbose);
+            $result = $this->getContainer()->get('packagist.package_dumper')->dump($ids, $force, $verbose);
         } finally {
-             $lock->release();
+            $locker->unlockCommand($this->getName());
         }
 
         return $result ? 0 : 1;

src/Packagist/WebBundle/Command/IndexPackagesCommand.php

@@ -20,7 +20,6 @@ use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Filesystem\LockHandler;
 use Doctrine\DBAL\Connection;
 
 class IndexPackagesCommand extends ContainerAwareCommand
@@ -60,6 +59,16 @@ class IndexPackagesCommand extends ContainerAwareCommand
             return;
         }
 
+        $locker = $this->getContainer()->get('locker');
+
+        $lockAcquired = $locker->lockCommand($this->getName());
+        if (!$lockAcquired) {
+            if ($input->getOption('verbose')) {
+                $output->writeln('Aborting, another task is running already');
+            }
+            return;
+        }
+
         $doctrine = $this->getContainer()->get('doctrine');
         $algolia = $this->getContainer()->get('packagist.algolia.client');
         $index = $algolia->initIndex($indexName);
@@ -68,16 +77,6 @@ class IndexPackagesCommand extends ContainerAwareCommand
         $downloadManager = $this->getContainer()->get('packagist.download_manager');
         $favoriteManager = $this->getContainer()->get('packagist.favorite_manager');
 
-        $lock = new LockHandler('packagist_algolia_indexer');
-
-        // another dumper is still active
-        if (!$lock->lock()) {
-            if ($verbose) {
-                $output->writeln('Aborting, another indexer is still active');
-            }
-            return;
-        }
-
         if ($package) {
             $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId()));
         } elseif ($force || $indexAll) {
@@ -164,7 +163,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
             $this->updateIndexedAt($idsToUpdate, $doctrine, $indexTime->format('Y-m-d H:i:s'));
         }
 
-        $lock->release();
+        $locker->unlockCommand($this->getName());
     }
 
     private function packageToSearchableArray(

src/Packagist/WebBundle/Command/MigrateDownloadCountsCommand.php

@@ -5,7 +5,6 @@ namespace Packagist\WebBundle\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\Filesystem\LockHandler;
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Seld\Signal\SignalHandler;
 use Packagist\WebBundle\Entity\Package;

src/Packagist/WebBundle/Controller/ApiController.php

@@ -14,6 +14,7 @@ namespace Packagist\WebBundle\Controller;
 
 use Packagist\WebBundle\Entity\Package;
 use Packagist\WebBundle\Entity\User;
+use Packagist\WebBundle\Util\UserAgentParser;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
@@ -142,7 +143,8 @@ class ApiController extends Controller
         }
 
         $package->setRepository($payload['repository']);
-        $errors = $this->get('validator')->validate($package, array("Update"));
+
+        $errors = $this->get('validator')->validate($package, null, array("Update"));
         if (count($errors) > 0) {
             $errorArray = array();
             foreach ($errors as $error) {
@@ -226,6 +228,15 @@ class ApiController extends Controller
 
         if ($jobs) {
             $this->get('packagist.download_manager')->addDownloads($jobs);
+
+            $uaParser = new UserAgentParser($request->headers->get('User-Agent'));
+            $this->get('Graze\DogStatsD\Client')->increment('installs', 1, 1, [
+                'composer' => $uaParser->getComposerVersion() ?: 'unknown',
+                'php_minor' => preg_replace('{^(\d+\.\d+).*}', '$1', $uaParser->getPhpVersion()) ?: 'unknown',
+                'php_patch' => $uaParser->getPhpVersion() ?: 'unknown',
+                'http' => $uaParser->getHttpVersion() ?: 'unknown',
+                'ci' => $uaParser->getCI() ? 'true' : 'false',
+            ]);
         }
 
         if ($failed) {

src/Packagist/WebBundle/Controller/FeedController.php

@@ -264,11 +264,10 @@ class FeedController extends Controller
         $entry->setDateModified($version->getReleasedAt());
         $entry->setDateCreated($version->getReleasedAt());
 
-        foreach ($version->getAuthors() as $author) {
-            /** @var $author \Packagist\WebBundle\Entity\Author */
-            if ($author->getName()) {
+        foreach ($version->getAuthorData() as $author) {
+            if (!empty($author['name'])) {
                 $entry->addAuthor(array(
-                    'name' => $author->getName()
+                    'name' => $author['name']
                 ));
             }
         }

src/Packagist/WebBundle/Controller/PackageController.php

@@ -9,6 +9,7 @@ use Packagist\WebBundle\Entity\Download;
 use Packagist\WebBundle\Entity\Package;
 use Packagist\WebBundle\Entity\PackageRepository;
 use Packagist\WebBundle\Entity\Version;
+use Packagist\WebBundle\Entity\Vendor;
 use Packagist\WebBundle\Entity\VersionRepository;
 use Packagist\WebBundle\Form\Model\MaintainerRequest;
 use Packagist\WebBundle\Form\Type\AbandonedType;
@@ -313,7 +314,7 @@ class PackageController extends Controller
             throw new NotFoundHttpException();
         }
 
-        $page = $req->query->get('page', 1);
+        $page = max(1, (int) $req->query->get('page', 1));
 
         /** @var PackageRepository $repo */
         $repo = $this->getDoctrine()->getRepository(Package::class);
@@ -327,10 +328,40 @@ class PackageController extends Controller
         $data['packages'] = $paginator;
         $data['count'] = $count;
         $data['meta'] = $this->getPackagesMetadata($data['packages']);
+        $data['markSafeCsrfToken'] = $this->get('security.csrf.token_manager')->getToken('mark_safe');
 
         return $this->render('PackagistWebBundle:package:spam.html.twig', $data);
     }
 
+    /**
+     * @Route(
+     *     "/spam/nospam",
+     *     name="mark_nospam",
+     *     defaults={"_format"="html"},
+     *     methods={"POST"}
+     * )
+     */
+    public function markSafeAction(Request $req)
+    {
+        if (!$this->getUser() || !$this->isGranted('ROLE_ANTISPAM')) {
+            throw new NotFoundHttpException();
+        }
+
+        $expectedToken = $this->get('security.csrf.token_manager')->getToken('mark_safe')->getValue();
+
+        $vendors = array_filter((array) $req->request->get('vendor'));
+        if (!hash_equals($expectedToken, $req->request->get('token'))) {
+            throw new BadRequestHttpException('Invalid CSRF token');
+        }
+
+        $repo = $this->getDoctrine()->getRepository(Vendor::class);
+        foreach ($vendors as $vendor) {
+            $repo->verify($vendor);
+        }
+
+        return $this->redirectToRoute('view_spam');
+    }
+
     /**
      * @Template()
      * @Route(
@@ -374,8 +405,8 @@ class PackageController extends Controller
         }
 
         if ('json' === $req->getRequestFormat()) {
-            $data = $package->toArray($this->getDoctrine()->getRepository(Version::class));
-            $data['dependents'] = $repo->getDependentCount($package->getName());
+            $data = $package->toArray($this->getDoctrine()->getRepository(Version::class), true);
+            $data['dependents'] = $repo->getDependantCount($package->getName());
             $data['suggesters'] = $repo->getSuggestCount($package->getName());
 
             try {
@@ -433,9 +464,16 @@ class PackageController extends Controller
         try {
             $data['downloads'] = $this->get('packagist.download_manager')->getDownloads($package, null, true);
 
-            if (!$package->isSuspect() && ($data['downloads']['total'] ?? 0) <= 10 && ($data['downloads']['views'] ?? 0) >= 100) {
-                $package->setSuspect('Too many views');
-                $repo->markPackageSuspect($package);
+            if (
+                !$package->isSuspect()
+                && ($data['downloads']['total'] ?? 0) <= 10 && ($data['downloads']['views'] ?? 0) >= 100
+                && $package->getCreatedAt()->getTimestamp() >= strtotime('2019-05-01')
+            ) {
+                $vendorRepo = $this->getDoctrine()->getRepository(Vendor::class);
+                if (!$vendorRepo->isVerified($package->getVendor())) {
+                    $package->setSuspect('Too many views');
+                    $repo->markPackageSuspect($package);
+                }
             }
 
             if ($this->getUser()) {
@@ -444,7 +482,7 @@ class PackageController extends Controller
         } catch (ConnectionException $e) {
         }
 
-        $data['dependents'] = $repo->getDependentCount($package->getName());
+        $data['dependents'] = $repo->getDependantCount($package->getName());
         $data['suggesters'] = $repo->getSuggestCount($package->getName());
 
         if ($maintainerForm = $this->createAddMaintainerForm($package)) {
@@ -462,6 +500,9 @@ class PackageController extends Controller
             )) {
             $data['deleteVersionCsrfToken'] = $this->get('security.csrf.token_manager')->getToken('delete_version');
         }
+        if ($this->isGranted('ROLE_ANTISPAM')) {
+            $data['markSafeCsrfToken'] = $this->get('security.csrf.token_manager')->getToken('mark_safe');
+        }
 
         return $data;
     }
@@ -844,6 +885,7 @@ class PackageController extends Controller
             $package->setIndexedAt(null);
             $package->setCrawledAt(new \DateTime());
             $package->setUpdatedAt(new \DateTime());
+            $package->setDumpedAt(null);
 
             $em = $this->getDoctrine()->getManager();
             $em->flush();
@@ -875,6 +917,7 @@ class PackageController extends Controller
         $package->setIndexedAt(null);
         $package->setCrawledAt(new \DateTime());
         $package->setUpdatedAt(new \DateTime());
+        $package->setDumpedAt(null);
 
         $em = $this->getDoctrine()->getManager();
         $em->flush();
@@ -936,11 +979,11 @@ class PackageController extends Controller
      */
     public function dependentsAction(Request $req, $name)
     {
-        $page = $req->query->get('page', 1);
+        $page = max(1, (int) $req->query->get('page', 1));
 
         /** @var PackageRepository $repo */
         $repo = $this->getDoctrine()->getRepository(Package::class);
-        $depCount = $repo->getDependentCount($name);
+        $depCount = $repo->getDependantCount($name);
         $packages = $repo->getDependents($name, ($page - 1) * 15, 15);
 
         $paginator = new Pagerfanta(new FixedAdapter($depCount, $packages));
@@ -965,7 +1008,7 @@ class PackageController extends Controller
      */
     public function suggestersAction(Request $req, $name)
     {
-        $page = $req->query->get('page', 1);
+        $page = max(1, (int) $req->query->get('page', 1));
 
         /** @var PackageRepository $repo */
         $repo = $this->getDoctrine()->getRepository(Package::class);
@@ -1021,7 +1064,7 @@ class PackageController extends Controller
             foreach ($values as $valueKey) {
                 $value += $dlData[$valueKey] ?? 0;
             }
-            $datePoints[$label] = $value;
+            $datePoints[$label] = ceil($value / count($values));
         }
 
         $datePoints = array(
@@ -1031,17 +1074,6 @@ class PackageController extends Controller
 
         $datePoints['average'] = $average;
 
-        if ($average !== 'daily') {
-            $dividers = [
-                'monthly' => 30.41,
-                'weekly' => 7,
-            ];
-            $divider = $dividers[$average];
-            $datePoints['values'] = array_map(function ($val) use ($divider) {
-                return ceil($val / $divider);
-            }, $datePoints['values']);
-        }
-
         if (empty($datePoints['labels']) && empty($datePoints['values'])) {
             $datePoints['labels'][] = date('Y-m-d');
             $datePoints['values'][] = 0;
@@ -1138,13 +1170,15 @@ class PackageController extends Controller
         $dateKey = 'Ymd';
         $dateFormat = $average === 'monthly' ? 'Y-m' : 'Y-m-d';
         $dateJump = '+1day';
-        if ($average === 'monthly') {
-            $from = new DateTimeImmutable('first day of ' . $from->format('Y-m'));
-            $to = new DateTimeImmutable('last day of ' . $to->format('Y-m'));
-        }
 
         $nextDataPointLabel = $from->format($dateFormat);
-        $nextDataPoint = $from->modify($interval);
+
+        if ($average === 'monthly') {
+            $nextDataPoint = new DateTimeImmutable('first day of ' . $from->format('Y-m'));
+            $nextDataPoint = $nextDataPoint->modify($interval);
+        } else {
+            $nextDataPoint = $from->modify($interval);
+        }
 
         $datePoints = [];
         while ($from <= $to) {

src/Packagist/WebBundle/Controller/UserController.php

@@ -225,7 +225,7 @@ class UserController extends Controller
         );
 
         $paginator->setMaxPerPage(15);
-        $paginator->setCurrentPage($req->query->get('page', 1), false, true);
+        $paginator->setCurrentPage(max(1, (int) $req->query->get('page', 1)), false, true);
 
         return array('packages' => $paginator, 'user' => $user);
     }
@@ -283,7 +283,7 @@ class UserController extends Controller
 
         $paginator = new Pagerfanta(new DoctrineORMAdapter($packages, true));
         $paginator->setMaxPerPage(15);
-        $paginator->setCurrentPage($req->query->get('page', 1), false, true);
+        $paginator->setCurrentPage(max(1, (int) $req->query->get('page', 1)), false, true);
 
         return $paginator;
     }

src/Packagist/WebBundle/Controller/WebController.php

@@ -62,29 +62,29 @@ class WebController extends Controller
      */
     public function searchAction(Request $req)
     {
-        $form = $this->createForm(SearchQueryType::class, new SearchQuery());
-
-        $filteredOrderBys = $this->getFilteredOrderedBys($req);
-
-        $this->computeSearchQuery($req, $filteredOrderBys);
-
-        $typeFilter = str_replace('%type%', '', $req->query->get('type'));
-        $tagsFilter = $req->query->get('tags');
-
         if ($req->getRequestFormat() !== 'json') {
             return $this->render('PackagistWebBundle:web:search.html.twig', [
                 'packages' => [],
             ]);
         }
 
+        $typeFilter = str_replace('%type%', '', $req->query->get('type'));
+        $tagsFilter = $req->query->get('tags');
+
+        $filteredOrderBys = $this->getFilteredOrderedBys($req);
+
+        $this->computeSearchQuery($req, $filteredOrderBys);
+
         if (!$req->query->has('search_query') && !$typeFilter && !$tagsFilter) {
             return JsonResponse::create(array(
                 'error' => 'Missing search query, example: ?q=example'
             ), 400)->setCallback($req->query->get('callback'));
         }
 
-        $indexName = $this->container->getParameter('algolia.index_name');
+        $form = $this->createForm(SearchQueryType::class, new SearchQuery());
+
         $algolia = $this->get('packagist.algolia.client');
+        $indexName = $this->container->getParameter('algolia.index_name');
         $index = $algolia->initIndex($indexName);
         $query = '';
         $queryParams = [];
@@ -116,7 +116,7 @@ class WebController extends Controller
             $query = $form->getData()->getQuery();
         }
 
-        $perPage = $req->query->getInt('per_page', 15);
+        $perPage = max(1, (int) $req->query->getInt('per_page', 15));
         if ($perPage <= 0 || $perPage > 100) {
            if ($req->getRequestFormat() === 'json') {
                 return JsonResponse::create(array(
@@ -132,7 +132,7 @@ class WebController extends Controller
             $queryParams['filters'] = implode(' AND ', $queryParams['filters']);
         }
         $queryParams['hitsPerPage'] = $perPage;
-        $queryParams['page'] = $req->query->get('page', 1) - 1;
+        $queryParams['page'] = max(1, (int) $req->query->get('page', 1)) - 1;
 
         try {
             $results = $index->search($query, $queryParams);
@@ -288,6 +288,29 @@ class WebController extends Controller
         );
     }
 
+    /**
+     * @Route("/statistics.json", name="stats_json", defaults={"_format"="json"}, methods={"GET"})
+     */
+    public function statsTotalsAction()
+    {
+        $downloads = (int) ($this->get('snc_redis.default_client')->get('downloads') ?: 0);
+        $packages = (int) $this->getDoctrine()
+            ->getConnection()
+            ->fetchColumn('SELECT COUNT(*) count FROM `package`');
+
+        $versions = (int) $this->getDoctrine()
+            ->getConnection()
+            ->fetchColumn('SELECT COUNT(*) count FROM `package_version`');
+
+        $totals = [
+            'downloads' => $downloads,
+            'packages' => $packages,
+            'versions' => $versions,
+        ];
+
+        return new JsonResponse(['totals' => $totals], 200);
+    }
+
     /**
      * @param Request $req
      *

src/Packagist/WebBundle/Entity/Author.php

@@ -57,7 +57,7 @@ class Author
     private $versions;
 
     /**
-     * @ORM\ManyToOne(targetEntity="Packagist\WebBundle\Entity\User", inversedBy="authors")
+     * @ORM\ManyToOne(targetEntity="Packagist\WebBundle\Entity\User")
      */
     private $owner;
 

src/Packagist/WebBundle/Entity/Package.php

@@ -193,7 +193,7 @@ class Package
         $this->createdAt = new \DateTime;
     }
 
-    public function toArray(VersionRepository $versionRepo)
+    public function toArray(VersionRepository $versionRepo, bool $serializeForApi = false)
     {
         $versions = array();
         $partialVersions = $this->getVersions()->toArray();
@@ -202,7 +202,7 @@ class Package
             $slice = array_splice($partialVersions, 0, 100);
             $fullVersions = $versionRepo->refreshVersions($slice);
             $versionData = $versionRepo->getVersionData(array_map(function ($v) { return $v->getId(); }, $fullVersions));
-            $versions = array_merge($versions, $versionRepo->detachToArray($fullVersions, $versionData));
+            $versions = array_merge($versions, $versionRepo->detachToArray($fullVersions, $versionData, $serializeForApi));
         }
 
         $maintainers = array();
@@ -292,7 +292,10 @@ class Package
                 return;
             }
 
-            if (preg_match('{(free.*watch|watch.*free|(stream|online).*anschauver.*pelicula|ver.*completa|pelicula.*complet|season.*episode.*online|film.*(complet|entier)|(voir|regarder|guarda|assistir).*(film|complet)|full.*movie|online.*(free|tv|full.*hd)|(free|full|gratuit).*stream|movie.*free|free.*(movie|hack)|watch.*movie|watch.*full|generate.*resource|generate.*unlimited|hack.*coin|coin.*(hack|generat)|vbucks|hack.*cheat|hack.*generat|generat.*hack|hack.*unlimited|cheat.*(unlimited|generat)|(mod|cheat|apk).*(hack|cheat|mod)|hack.*(apk|mod|free|gold|gems|diamonds|coin)|putlocker|generat.*free|coins.*generat|(download|telecharg).*album|album.*(download|telecharg)|album.*(free|gratuit)|generat.*coins|unlimited.*coins|(fortnite|pubg|apex.*legend|t[1i]k.*t[o0]k).*(free|gratuit|generat|unlimited|coins|mobile|hack|follow))}i', str_replace(array('.', '-'), '', $information['name']))) {
+            if (
+                preg_match('{(free.*watch|watch.*free|(stream|online).*anschauver.*pelicula|ver.*completa|pelicula.*complet|season.*episode.*online|film.*(complet|entier)|(voir|regarder|guarda|assistir).*(film|complet)|full.*movie|online.*(free|tv|full.*hd)|(free|full|gratuit).*stream|movie.*free|free.*(movie|hack)|watch.*movie|watch.*full|generate.*resource|generate.*unlimited|hack.*coin|coin.*(hack|generat)|vbucks|hack.*cheat|hack.*generat|generat.*hack|hack.*unlimited|cheat.*(unlimited|generat)|(mod|cheat|apk).*(hack|cheat|mod)|hack.*(apk|mod|free|gold|gems|diamonds|coin)|putlocker|generat.*free|coins.*generat|(download|telecharg).*album|album.*(download|telecharg)|album.*(free|gratuit)|generat.*coins|unlimited.*coins|(fortnite|pubg|apex.*legend|t[1i]k.*t[o0]k).*(free|gratuit|generat|unlimited|coins|mobile|hack|follow))}i', str_replace(array('.', '-'), '', $information['name']))
+                && !preg_match('{^(hexmode|calgamo)/}', $information['name'])
+            ) {
                 $context->buildViolation('The package name '.htmlentities($information['name'], ENT_COMPAT, 'utf-8').' is blocked, if you think this is a mistake please get in touch with us.')
                     ->atPath($property)
                     ->addViolation()

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -295,6 +295,8 @@ class PackageRepository extends EntityRepository
             $qb->leftJoin('v.tags', 't');
         }
 
+        $qb->andWhere('(p.replacementPackage IS NULL OR p.replacementPackage != \'spam/spam\')');
+
         $qb->orderBy('p.abandoned');
         if (true === $orderByName) {
             $qb->addOrderBy('p.name');
@@ -360,7 +362,7 @@ class PackageRepository extends EntityRepository
         return $result;
     }
 
-    public function getDependentCount($name)
+    public function getDependantCount($name)
     {
         $sql = 'SELECT COUNT(*) count FROM (
                 SELECT pv.package_id FROM link_require r INNER JOIN package_version pv ON (pv.id = r.version_id AND pv.development = 1) WHERE r.packageName = :name

src/Packagist/WebBundle/Entity/User.php

@@ -85,11 +85,6 @@ class User extends BaseUser
      */
     private $packages;
 
-    /**
-     * @ORM\OneToMany(targetEntity="Packagist\WebBundle\Entity\Author", mappedBy="owner")
-     */
-    private $authors;
-
     /**
      * @ORM\Column(type="datetime")
      */
@@ -128,7 +123,6 @@ class User extends BaseUser
     public function __construct()
     {
         $this->packages = new ArrayCollection();
-        $this->authors = new ArrayCollection();
         $this->createdAt = new \DateTime();
         parent::__construct();
     }
@@ -161,26 +155,6 @@ class User extends BaseUser
         return $this->packages;
     }
 
-    /**
-     * Add authors
-     *
-     * @param Author $authors
-     */
-    public function addAuthors(Author $authors)
-    {
-        $this->authors[] = $authors;
-    }
-
-    /**
-     * Get authors
-     *
-     * @return Author[]
-     */
-    public function getAuthors()
-    {
-        return $this->authors;
-    }
-
     /**
      * Set createdAt
      *

src/Packagist/WebBundle/Entity/Vendor.php

@@ -0,0 +1,68 @@
+<?php
+
+/*
+ * This file is part of Packagist.
+ *
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
+ *     Nils Adermann <naderman@naderman.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Packagist\WebBundle\Entity;
+
+use Composer\Factory;
+use Composer\IO\NullIO;
+use Composer\Repository\VcsRepository;
+use Doctrine\Common\Collections\ArrayCollection;
+use Doctrine\ORM\Mapping as ORM;
+use Symfony\Component\Validator\Constraints as Assert;
+use Symfony\Component\Validator\Context\ExecutionContextInterface;
+use Composer\Repository\Vcs\GitHubDriver;
+
+/**
+ * @ORM\Entity(repositoryClass="Packagist\WebBundle\Entity\VendorRepository")
+ * @ORM\Table(
+ *     name="vendor",
+ *     indexes={
+ *         @ORM\Index(name="verified_idx",columns={"verified"})
+ *     }
+ * )
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ */
+class Vendor
+{
+    /**
+     * Unique vendor name
+     *
+     * @ORM\Id
+     * @ORM\Column(length=191)
+     */
+    private $name;
+
+    /**
+     * @ORM\Column(type="boolean")
+     */
+    private $verified = false;
+
+    public function setName(string $name)
+    {
+        $this->name = $name;
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function setVerified(bool $verified)
+    {
+        $this->verified = $verified;
+    }
+
+    public function getVerified(): bool
+    {
+        return $this->verified;
+    }
+}

src/Packagist/WebBundle/Entity/VendorRepository.php

@@ -0,0 +1,40 @@
+<?php
+
+/*
+ * This file is part of Packagist.
+ *
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
+ *     Nils Adermann <naderman@naderman.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Packagist\WebBundle\Entity;
+
+use Doctrine\ORM\EntityRepository;
+
+/**
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ */
+class VendorRepository extends EntityRepository
+{
+    public function isVerified(string $vendor): bool
+    {
+        $result = $this->getEntityManager()->getConnection()->fetchColumn('SELECT verified FROM vendor WHERE name = :vendor', ['vendor' => $vendor]);
+
+        return $result === '1';
+    }
+
+    public function verify(string $vendor)
+    {
+        $this->getEntityManager()->getConnection()->executeUpdate(
+            'INSERT INTO vendor (name, verified) VALUES (:vendor, 1) ON DUPLICATE KEY UPDATE verified=1',
+            ['vendor' => $vendor]
+        );
+        $this->getEntityManager()->getConnection()->executeUpdate(
+            'UPDATE package SET suspect = NULL WHERE name LIKE :vendor',
+            ['vendor' => $vendor.'/%']
+        );
+    }
+}

src/Packagist/WebBundle/Entity/Version.php

@@ -109,6 +109,8 @@ class Version
     private $license;
 
     /**
+     * Deprecated relation table, use the authorJson property instead
+     *
      * @ORM\ManyToMany(targetEntity="Packagist\WebBundle\Entity\Author", inversedBy="versions")
      * @ORM\JoinTable(name="version_author",
      *     joinColumns={@ORM\JoinColumn(name="version_id", referencedColumnName="id")},
@@ -177,6 +179,11 @@ class Version
      */
     private $support;
 
+    /**
+     * @ORM\Column(name="authors", type="json", nullable=true)
+     */
+    private $authorJson;
+
     /**
      * @ORM\Column(type="datetime")
      */
@@ -211,7 +218,7 @@ class Version
         $this->updatedAt = new \DateTime;
     }
 
-    public function toArray(array $versionData)
+    public function toArray(array $versionData, bool $serializeForApi = false)
     {
         if (isset($versionData[$this->id]['tags'])) {
             $tags = $versionData[$this->id]['tags'];
@@ -223,15 +230,23 @@ class Version
             }
         }
 
-        if (isset($versionData[$this->id]['authors'])) {
-            $authors = $versionData[$this->id]['authors'];
+        if (!is_null($this->getAuthorJson())) {
+            $authors = $this->getAuthorJson();
         } else {
-            $authors = array();
-            foreach ($this->getAuthors() as $author) {
-                /** @var $author Author */
-                $authors[] = $author->toArray();
+            if (isset($versionData[$this->id]['authors'])) {
+                $authors = $versionData[$this->id]['authors'];
+            } else {
+                $authors = array();
+                foreach ($this->getAuthors() as $author) {
+                    /** @var $author Author */
+                    $authors[] = $author->toArray();
+                }
             }
         }
+        foreach ($authors as &$author) {
+            uksort($author, [$this, 'sortAuthorKeys']);
+        }
+        unset($author);
 
         $data = array(
             'name' => $this->getName(),
@@ -247,6 +262,9 @@ class Version
             'type' => $this->getType(),
         );
 
+        if ($serializeForApi && $this->getSupport()) {
+            $data['support'] = $this->getSupport();
+        }
         if ($this->getReleasedAt()) {
             $data['time'] = $this->getReleasedAt()->format('Y-m-d\TH:i:sP');
         }
@@ -299,7 +317,10 @@ class Version
     {
         $array = $this->toArray($versionData);
 
-        unset($array['keywords'], $array['authors']);
+        if ($this->getSupport()) {
+            $array['support'] = $this->getSupport();
+            ksort($array['support']);
+        }
 
         return $array;
     }
@@ -715,6 +736,40 @@ class Version
         return $this->authors;
     }
 
+    public function getAuthorJson(): ?array
+    {
+        return $this->authorJson;
+    }
+
+    public function setAuthorJson(?array $authors): void
+    {
+        $this->authorJson = $authors ?: [];
+    }
+
+    /**
+     * Get authors
+     *
+     * @return array[]
+     */
+    public function getAuthorData(): array
+    {
+        if (!is_null($this->getAuthorJson())) {
+            return $this->getAuthorJson();
+        }
+
+        $authors = [];
+        foreach ($this->getAuthors() as $author) {
+            $authors[] = array_filter([
+                'name' => $author->getName(),
+                'homepage' => $author->getHomepage(),
+                'email' => $author->getEmail(),
+                'role' => $author->getRole(),
+            ]);
+        }
+
+        return $authors;
+    }
+
     /**
      * Set type
      *
@@ -979,4 +1034,16 @@ class Version
     {
         return $this->name.' '.$this->version.' ('.$this->normalizedVersion.')';
     }
+
+    private function sortAuthorKeys($a, $b)
+    {
+        static $order = ['name' => 1, 'email' => 2, 'homepage' => 3, 'role' => 4];
+        $aIndex = $order[$a] ?? 5;
+        $bIndex = $order[$b] ?? 5;
+        if ($aIndex === $bIndex) {
+            return $a <=> $b;
+        }
+
+        return $aIndex <=> $bIndex;
+    }
 }

src/Packagist/WebBundle/Entity/VersionRepository.php

@@ -82,12 +82,12 @@ class VersionRepository extends EntityRepository
     /**
      * @param Version[] $versions
      */
-    public function detachToArray(array $versions, array $versionData): array
+    public function detachToArray(array $versions, array $versionData, bool $serializeForApi = false): array
     {
         $res = [];
         $em = $this->getEntityManager();
         foreach ($versions as $version) {
-            $res[$version->getVersion()] = $version->toArray($versionData);
+            $res[$version->getVersion()] = $version->toArray($versionData, $serializeForApi);
             $em->detach($version);
         }
 
@@ -157,7 +157,7 @@ class VersionRepository extends EntityRepository
     public function getVersionMetadataForUpdate(Package $package)
     {
         $rows = $this->getEntityManager()->getConnection()->fetchAll(
-            'SELECT id, version, normalizedVersion, source, softDeletedAt FROM package_version v WHERE v.package_id = :id',
+            'SELECT id, version, normalizedVersion, source, softDeletedAt, `authors` IS NULL as needs_author_migration FROM package_version v WHERE v.package_id = :id',
             ['id' => $package->getId()]
         );
 
@@ -166,6 +166,7 @@ class VersionRepository extends EntityRepository
             if ($row['source']) {
                 $row['source'] = json_decode($row['source'], true);
             }
+            $row['needs_author_migration'] = (int) $row['needs_author_migration'];
             $versions[strtolower($row['normalizedVersion'])] = $row;
         }
 

src/Packagist/WebBundle/Form/Type/ProfileFormType.php

@@ -13,7 +13,13 @@
 namespace Packagist\WebBundle\Form\Type;
 
 use FOS\UserBundle\Form\Type\ProfileFormType as BaseType;
+use FOS\UserBundle\Util\LegacyFormHelper;
 use Symfony\Component\Form\FormBuilderInterface;
+use Symfony\Component\Form\FormEvents;
+use Symfony\Component\Form\FormEvent;
+use Symfony\Component\Security\Core\Validator\Constraints\UserPassword;
+use Symfony\Component\Validator\Constraints\NotBlank;
+use Symfony\Component\Form\Extension\Core\Type\PasswordType;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -22,11 +28,34 @@ class ProfileFormType extends BaseType
 {
     public function buildForm(FormBuilderInterface $builder, array $options)
     {
-        parent::buildForm($builder, $options);
+        $this->buildUserForm($builder, $options);
 
-        $builder->add('failureNotifications', null, array(
-            'required' => false,
-        ));
+        $builder->addEventListener(FormEvents::PRE_SET_DATA, function(FormEvent $event) {
+            if (!($user = $event->getData())) {
+                return;
+            }
+
+            if (!$user->getGithubId()) {
+                $constraintsOptions = array(
+                    'message' => 'fos_user.current_password.invalid',
+                );
+
+                $event->getForm()->add('current_password', PasswordType::class, array(
+                    'label' => 'form.current_password',
+                    'translation_domain' => 'FOSUserBundle',
+                    'mapped' => false,
+                    'constraints' => array(
+                        new NotBlank(),
+                        new UserPassword($constraintsOptions),
+                    ),
+                    'attr' => array(
+                        'autocomplete' => 'current-password',
+                    ),
+                ));
+            }
+        });
+
+        $builder->add('failureNotifications', null, array('required' => false, 'label' => 'Notify me of package update failures'));
     }
 
     /**

src/Packagist/WebBundle/HealthCheck/MetadataDirCheck.php

@@ -20,6 +20,10 @@ class MetadataDirCheck extends AbstractCheck
 
     public static function isMetadataStoreMounted(array $awsMeta): bool
     {
+        if (empty($awsMeta)) {
+            return true;
+        }
+
         if ($awsMeta['primary'] && $awsMeta['has_instance_store']) {
             // TODO in symfony4, use fromShellCommandline
             $proc = new Process('lsblk -io NAME,TYPE,SIZE,MOUNTPOINT,FSTYPE,MODEL | grep Instance | grep sdeph');
@@ -33,6 +37,10 @@ class MetadataDirCheck extends AbstractCheck
 
     public function check()
     {
+        if (empty($this->awsMeta)) {
+            return new Success('No AWS metadata given');
+        }
+
         if ($this->awsMeta['primary']) {
             if ($this->awsMeta['has_instance_store']) {
                 if (!self::isMetadataStoreMounted($this->awsMeta)) {

src/Packagist/WebBundle/HealthCheck/RedisHealthCheck.php

@@ -27,7 +27,7 @@ class RedisHealthCheck extends AbstractCheck
             }
 
             // only warn for fragmented memory when amount of memory is above 256MB
-            if ($info['Memory']['used_memory'] > 256*1024*1024 && $info['Memory']['mem_fragmentation_ratio'] > 1.6) {
+            if ($info['Memory']['used_memory'] > 256*1024*1024 && $info['Memory']['mem_fragmentation_ratio'] > 3) {
                 return new Warning('Redis memory fragmentation ratio is pretty high, maybe redis instances should be restarted', $info['Memory']);
             }
 

src/Packagist/WebBundle/Package/SymlinkDumper.php

@@ -22,6 +22,7 @@ use Packagist\WebBundle\Entity\Package;
 use Doctrine\DBAL\Connection;
 use Packagist\WebBundle\HealthCheck\MetadataDirCheck;
 use Predis\Client;
+use Graze\DogStatsD\Client as StatsDClient;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -117,6 +118,11 @@ class SymlinkDumper
      */
     private $awsMeta;
 
+    /**
+     * @var StatsDClient
+     */
+    private $statsd;
+
     /**
      * Constructor
      *
@@ -127,7 +133,7 @@ class SymlinkDumper
      * @param string                $targetDir
      * @param int                   $compress
      */
-    public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, Client $redis, $webDir, $targetDir, $compress, $awsMetadata)
+    public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, Client $redis, $webDir, $targetDir, $compress, $awsMetadata, StatsDClient $statsd)
     {
         $this->doctrine = $doctrine;
         $this->fs = $filesystem;
@@ -138,6 +144,7 @@ class SymlinkDumper
         $this->compress = $compress;
         $this->redis = $redis;
         $this->awsMeta = $awsMetadata;
+        $this->statsd = $statsd;
     }
 
     /**
@@ -507,6 +514,8 @@ class SymlinkDumper
             }
         }
 
+        $this->statsd->increment('packagist.metadata_dump');
+
         // TODO when a package is deleted, it should be removed from provider files, or marked for removal at least
         return true;
     }

src/Packagist/WebBundle/Package/Updater.php

@@ -105,7 +105,7 @@ class Updater
 
         if ($repository instanceof VcsRepository) {
             $cfg = $repository->getRepoConfig();
-            if (isset($cfg['url']) && preg_match('{\bgithub\.com\b}', $cfg['url'])) {
+            if (isset($cfg['url']) && preg_match('{\bgithub\.com\b}i', $cfg['url'])) {
                 foreach ($package->getMaintainers() as $maintainer) {
                     if (!($newGithubToken = $maintainer->getGithubToken())) {
                         continue;
@@ -271,6 +271,13 @@ class Updater
             // update if the right flag is set, or the source reference has changed (re-tag or new commit on branch)
             if ($source['reference'] !== $data->getSourceReference() || ($flags & self::UPDATE_EQUAL_REFS)) {
                 $version = $versionRepo->findOneById($existingVersion['id']);
+            } elseif ($existingVersion['needs_author_migration']) {
+                $version = $versionRepo->findOneById($existingVersion['id']);
+
+                $version->setAuthorJson($version->getAuthorData());
+                $version->getAuthors()->clear();
+
+                return ['updated' => true, 'id' => $version->getId(), 'version' => strtolower($normVersion), 'object' => $version];
             } else {
                 return ['updated' => false, 'id' => $existingVersion['id'], 'version' => strtolower($normVersion), 'object' => null];
             }
@@ -363,21 +370,19 @@ class Updater
             $version->getTags()->clear();
         }
 
-        $authorRepository = $this->doctrine->getRepository('PackagistWebBundle:Author');
-
         $version->getAuthors()->clear();
+        $version->setAuthorJson([]);
         if ($data->getAuthors()) {
+            $authors = [];
             foreach ($data->getAuthors() as $authorData) {
-                $author = null;
+                $author = [];
 
                 foreach (array('email', 'name', 'homepage', 'role') as $field) {
                     if (isset($authorData[$field])) {
-                        $authorData[$field] = trim($authorData[$field]);
-                        if ('' === $authorData[$field]) {
-                            $authorData[$field] = null;
+                        $author[$field] = trim($authorData[$field]);
+                        if ('' === $author[$field]) {
+                            unset($author[$field]);
                         }
-                    } else {
-                        $authorData[$field] = null;
                     }
                 }
 
@@ -386,32 +391,9 @@ class Updater
                     continue;
                 }
 
-                $author = $authorRepository->findOneBy(array(
-                    'email' => $authorData['email'],
-                    'name' => $authorData['name'],
-                    'homepage' => $authorData['homepage'],
-                    'role' => $authorData['role'],
-                ));
-
-                if (!$author) {
-                    $author = new Author();
-                    $em->persist($author);
-                }
-
-                foreach (array('email', 'name', 'homepage', 'role') as $field) {
-                    if (isset($authorData[$field])) {
-                        $author->{'set'.$field}($authorData[$field]);
-                    }
-                }
-
-                // only update the author timestamp once a month at most as the value is kinda unused
-                if ($author->getUpdatedAt() === null || $author->getUpdatedAt()->getTimestamp() < time() - 86400 * 30) {
-                    $author->setUpdatedAt(new \DateTime);
-                }
-                if (!$version->getAuthors()->contains($author)) {
-                    $version->addAuthor($author);
-                }
+                $authors[] = $author;
             }
+            $version->setAuthorJson($authors);
         }
 
         // handle links

src/Packagist/WebBundle/Resources/config/services.yml

@@ -32,6 +32,9 @@ services:
         tags:
             - { name: twig.extension }
 
+    Graze\DogStatsD\Client:
+        public: true
+
     twig.extension.text:
         public: true
         class: Twig_Extensions_Extension_Text
@@ -68,7 +71,7 @@ services:
     packagist.package_dumper:
         public: true
         class: Packagist\WebBundle\Package\SymlinkDumper
-        arguments: [ '@doctrine', '@filesystem', '@router', '@snc_redis.default_client', '%kernel.root_dir%/../web/', '%packagist_metadata_dir%', '%packagist_dumper_compress%', '%aws_metadata%' ]
+        arguments: [ '@doctrine', '@filesystem', '@router', '@snc_redis.default_client', '%kernel.root_dir%/../web/', '%packagist_metadata_dir%', '%packagist_dumper_compress%', '%aws_metadata%', '@Graze\DogStatsD\Client' ]
 
     packagist.user_provider:
         class: Packagist\WebBundle\Security\Provider\UserProvider

src/Packagist/WebBundle/Resources/public/css/main.css

@@ -537,6 +537,10 @@ button {
     font-family: 'Open Sans', sans-serif;
 }
 
+.form-group {
+    clear: left;
+}
+
 .form-control {
     min-height: 20px;
     padding: 7px 11px 6px 9px;
@@ -1505,30 +1509,44 @@ input:focus:invalid:focus, textarea:focus:invalid:focus, select:focus:invalid:fo
 }
 
 .alert {
-    background: none;
-    border: none;
     border-left: 4px solid #000;
     box-shadow: none;
     border-radius: 0;
     padding: 5px 0 5px 10px;
 }
+.flash-container .alert {
+    margin-bottom: 5px;
+}
+.flash-container .alert:last-child {
+    margin-bottom: 0;
+}
 .alert p {
   margin: 0;
 }
-.flash-container .alert {
-  margin-bottom: 0;
-}
 .alert-warning {
     border-left-color: #F28D1A;
     color: #2d2d32;
+    background-image: linear-gradient(to bottom,#fcf8e3 0,#f8efc0 100%);
+}
+.alert-warning::before {
+    content: "Warning: ";
+    float: left;
+    padding-right: 5px;
 }
 .alert-danger, .alert-error {
     border-left-color: #cd3729;
     color: #2d2d32;
+    background-image: linear-gradient(to bottom,#f2dede 0,#e7c3c3 100%);
+}
+.alert-error::before {
+    content: "Error: ";
+    float: left;
+    padding-right: 5px;
 }
 .alert-success {
     border-left-color: #69AD21;
     color: #2d2d32;
+    background-image: linear-gradient(to bottom,#dff0d8 0,#c8e5bc 100%);
 }
 
 

src/Packagist/WebBundle/Resources/translations/messages.en.yml

@@ -57,7 +57,7 @@ packages:
     maintained_by: "Packages maintained by %user%"
     my_favorites: My favorite packages
     users_favorites: "%user%'s favorite packages"
-    dependent_title: Dependent Packages
+    dependant_title: Dependant Packages
     dependents: dependents
     providers_title: "The following packages provide %name%"
     suggesters: suggesters
@@ -107,8 +107,8 @@ stats:
     installs: Installs
     overall: Overall
     lastmonth: Last 30 days
-    today: Today
-    since_midnight: "since midnight, UTC"
+    today: Last 24h
+    rolling_avg: "rolling average, approx."
     daily: Daily installs
     daily_per_version: Daily installs per version
     averaged: "averaged %avg%"
@@ -138,6 +138,7 @@ api_doc:
     by_name: By name
     by_type: By type
     by_tag: By tag
+    get_statistics: Get statistics
     list_by_organization: List packages by organization
     list_by_type: List packages by type
     searching: Searching for packages

src/Packagist/WebBundle/Resources/views/api_doc/index.html.twig

@@ -19,6 +19,7 @@
     </ul>
   </li>
   <li><a href="#get-package-data">{{ 'api_doc.get_package_data'|trans }}</a></li>
+  <li><a href="#get-statistics">{{ 'api_doc.get_statistics'|trans }}</a></li>
 </ul>
 
 
@@ -205,5 +206,25 @@ GET https://{{ packagist_host }}/packages/[vendor]/[package].json
 
 </section>
 
+<section class="col-d-12">
+<h3 id="get-statistics">{{ 'api_doc.get_statistics'|trans }}</h3>
+
+<h4 id="get-package-by-name">{{ 'api_doc.get_statistics'|trans }}</h4>
+
+<p>This endpoint provides basic some statistics.</p>
+
+<pre>
+GET https://{{ packagist_host }}/statistics.json
+<code>
+{
+  "totals": {
+    "downloads": [numbers of download]
+  }
+}
+</code></pre>
+<p>Working example: <code><a href="https://{{ packagist_host }}/statistics.json">https://{{ packagist_host }}/statistics.json</a></code></p>
+
+</section>
+
 
 {% endblock %}

src/Packagist/WebBundle/Resources/views/mirrors/index.html.twig

@@ -13,8 +13,10 @@
 <p>On top of this, we are aware of the following list of third-party-run mirrors, please refer to their website to see how to use them:</p>
 <ul>
   <li>Africa, South Africa <a href="https://packagist.co.za/">packagist.co.za</a></li>
+  <li>Asia, China <a href="https://mirrors.aliyun.com/composer/">mirrors.aliyun.com/composer</a></li>
   <li>Asia, China <a href="https://pkg.phpcomposer.com/">pkg.phpcomposer.com</a></li>
   <li>Asia, Indonesia <a href="https://packagist.phpindonesia.id/">packagist.phpindonesia.id</a></li>
+  <li>Asia, India <a href="https://packagist.in/">packagist.in</a></li>
   <li>Asia, Japan <a href="https://packagist.jp/">packagist.jp</a></li>
   <li>South America, Brazil <a href="https://packagist.com.br/">packagist.com.br</a></li>
 </ul>

src/Packagist/WebBundle/Resources/views/package/dependents.html.twig

@@ -4,7 +4,7 @@
 
 {% block head_additions %}<meta name="robots" content="noindex, nofollow">{% endblock %}
 
-{% block title %}{{ 'packages.dependent_title'|trans }} - {{ name }} - {{ parent() }}{% endblock %}
+{% block title %}{{ 'packages.dependant_title'|trans }} - {{ name }} - {{ parent() }}{% endblock %}
 
 {% block content %}
     <div class="row">

src/Packagist/WebBundle/Resources/views/package/spam.html.twig

@@ -8,7 +8,7 @@
 
 {% block content %}
     <div class="row">
-        <div class="col-xs-12 package">
+        <div class="col-xs-9 package">
             <div class="package-header">
                 <h2 class="title">
                     Suspect Packages
@@ -16,6 +16,15 @@
                 </h2>
             </div>
         </div>
+        <div class="col-xs-3">
+            <form class="action" action="{{ path("mark_nospam") }}" method="POST">
+                {% for p in packages %}
+                    <input type="hidden" name="vendor[]" value="{{ p.name|vendor }}" />
+                {% endfor %}
+                <input type="hidden" name="token" value="{{ markSafeCsrfToken }}" />
+                <input class="btn btn-danger" type="submit" value="Mark Whole Page As Not Spam" />
+            </form>
+        </div>
     </div>
 
     <section class="row">

src/Packagist/WebBundle/Resources/views/package/stats.html.twig

@@ -36,7 +36,7 @@
                 </div>
                 <div class="col-md-4 col-xs-12">
                     <dl class="dl-horizontal">
-                        <dt class="font-normal">{{ 'stats.today'|trans }}<br><small>({{ 'stats.since_midnight'|trans }})</small></dt>
+                        <dt class="font-normal">{{ 'stats.today'|trans }}<br><small>({{ 'stats.rolling_avg'|trans }})</small></dt>
                         <dd class="font-normal">{{ downloads.daily|number_format(0, '.', '&#8201;')|raw }}</dd>
                     </dl>
                 </div>

src/Packagist/WebBundle/Resources/views/package/version_details.html.twig

@@ -36,17 +36,17 @@
 <div class="metadata">
     <p class="license"><i class="glyphicon glyphicon-copyright-mark" title="License"></i> {% if not version.license %}<span class="unknown">{% endif %}{{ version.license ? version.license|join(', ') : 'Unknown License' }}{% if not version.license %}</span>{% endif %} <span class="source-reference"><i class="glyphicon glyphicon-bookmark" title="Source Reference"></i> {{ version.source.reference }}</span></p>
 
-    {% if version.authors|length %}
+    {% if version.authorData|length %}
         <i class="glyphicon glyphicon-user" title="Authors"></i>
         <ul class="authors">
-            {% for author in version.authors %}
+            {% for author in version.authorData %}
                 <li>
-                    {%- if author.homepage -%}
+                    {%- if author.homepage|default(false) -%}
                         <a rel="nofollow noopener external noindex" href="{{ author.homepage }}">{{ author.name }}</a>
                     {%- else -%}
                         {{ author.name }}
                     {%- endif -%}
-                    {% if author.email %}
+                    {% if author.email|default(false) %}
                         <span class="visible-sm-inline visible-md-inline visible-lg-inline">&lt;{{ author.email|split('@', 2)[0] }}<span style="display:none">.woop</span>@{{ author.email|split('@', 2)[1] }}&gt;</span>
                     {%- endif -%}
                 </li>

src/Packagist/WebBundle/Resources/views/package/view_package.html.twig

@@ -29,7 +29,7 @@
     {{ package.description|truncate(300) }}
 {%- endblock %}
 
-{% set hasActions = is_granted('ROLE_EDIT_PACKAGES') or is_granted('ROLE_UPDATE_PACKAGES') or package.maintainers.contains(app.user) %}
+{% set hasActions = is_granted('ROLE_EDIT_PACKAGES') or is_granted('ROLE_ANTISPAM') or is_granted('ROLE_UPDATE_PACKAGES') or package.maintainers.contains(app.user) %}
 
 {% block content %}
     <div class="row">
@@ -117,6 +117,13 @@
                                     <input class="btn btn-primary" type="submit" value="Edit" />
                                 </form>
                             {% endif %}
+                            {% if is_granted('ROLE_ANTISPAM') and package.isSuspect() %}
+                                <form class="action" action="{{ path("mark_nospam") }}" method="POST">
+                                    <input type="hidden" name="vendor" value="{{ package.vendor }}" />
+                                    <input type="hidden" name="token" value="{{ markSafeCsrfToken }}" />
+                                    <input class="btn btn-danger" type="submit" value="Mark Not Spam" />
+                                </form>
+                            {% endif %}
                         </div>
                     {% endif %}
                 </div>

src/Packagist/WebBundle/Resources/views/user/favorites.html.twig

@@ -2,19 +2,19 @@
 
 {% import "PackagistWebBundle::macros.html.twig" as macros %}
 
+{% block content %}
 {% set isActualUser = app.user and app.user.username is same as(user.username) %}
 
-{% block content %}
 <h2 class="title">
     {{ user.username }}
-    <small>
-        {%- if not isActualUser %}
+    {%- if not isActualUser %}
+        <small>
             member since: {{ user.createdAt|date('M d, Y') }}
             {%- if is_granted('ROLE_ADMIN') %}
                 <a href="mailto:{{ user.email }}">{{ user.email }}</a>
             {%- endif %}
-        {%- endif %}
-    </small>
+        </small>
+    {%- endif %}
 </h2>
 
 <section class="row">

src/Packagist/WebBundle/Resources/views/user/packages.html.twig

@@ -2,19 +2,19 @@
 
 {% import "PackagistWebBundle::macros.html.twig" as macros %}
 
-{% set isActualUser = app.user and app.user.username is same as(user.username) %}
 
 {% block content %}
+{% set isActualUser = app.user and app.user.username is same as(user.username) %}
 <h2 class="title">
     {{ user.username }}
-    <small>
-        {%- if not isActualUser %}
+    {%- if not isActualUser %}
+        <small>
             member since: {{ user.createdAt|date('M d, Y') }}
             {%- if is_granted('ROLE_ADMIN') %}
                 <a href="mailto:{{ user.email }}">{{ user.email }}</a>
             {%- endif %}
-        {%- endif %}
-    </small>
+        </small>
+    {%- endif %}
 </h2>
 
 <section class="row">

src/Packagist/WebBundle/Resources/views/user/profile.html.twig

@@ -2,9 +2,14 @@
 
 {% import "PackagistWebBundle::macros.html.twig" as macros %}
 
-{% set isActualUser = app.user and app.user.username is same as(user.username) %}
+{% block head_additions %}
+    {% if user.hasRole('ROLE_SPAMMER') %}
+        <meta name="robots" content="noindex, nofollow">
+    {% endif %}
+{% endblock %}
 
 {% block content %}
+{% set isActualUser = app.user and app.user.username is same as(user.username) %}
 <h2 class="title">
     {{ user.username }}
     <small>

src/Packagist/WebBundle/Security/Provider/UserProvider.php

@@ -55,6 +55,9 @@ class UserProvider implements OAuthAwareUserProviderInterface, UserProviderInter
     public function connect($user, UserResponseInterface $response)
     {
         $username = $response->getUsername();
+        if (!$username || $username <= 0) {
+            throw new \LogicException('Failed to load info from GitHub');
+        }
 
         /** @var User $previousUser */
         $previousUser = $this->userManager->findUserBy(array('githubId' => $username));
@@ -87,6 +90,10 @@ class UserProvider implements OAuthAwareUserProviderInterface, UserProviderInter
     public function loadUserByOAuthUserResponse(UserResponseInterface $response)
     {
         $username = $response->getUsername();
+        if (!$username || $username <= 0) {
+            throw new \LogicException('Failed to load info from GitHub');
+        }
+
         /** @var User $user */
         $user = $this->userManager->findUserBy(array('githubId' => $username));
 
@@ -94,6 +101,10 @@ class UserProvider implements OAuthAwareUserProviderInterface, UserProviderInter
             throw new AccountNotLinkedException(sprintf('No user with github username "%s" was found.', $username));
         }
 
+        if ($user->getGithubId() !== (string) $response->getUsername()) {
+            throw new \LogicException('This really should not happen but checking just in case');
+        }
+
         if ($user->getGithubToken() !== $response->getAccessToken()) {
             $user->setGithubToken($response->getAccessToken());
             $oldScope = $user->getGithubScope();

src/Packagist/WebBundle/Service/QueueWorker.php

@@ -7,6 +7,7 @@ use Psr\Log\LoggerInterface;
 use Symfony\Bridge\Doctrine\RegistryInterface;
 use Packagist\WebBundle\Entity\Job;
 use Seld\Signal\SignalHandler;
+use Graze\DogStatsD\Client as StatsDClient;
 
 class QueueWorker
 {
@@ -16,13 +17,16 @@ class QueueWorker
     private $doctrine;
     private $jobWorkers;
     private $processedJobs = 0;
+    /** @var StatsDClient */
+    private $statsd;
 
-    public function __construct(Redis $redis, RegistryInterface $doctrine, LoggerInterface $logger, array $jobWorkers)
+    public function __construct(Redis $redis, RegistryInterface $doctrine, LoggerInterface $logger, array $jobWorkers, StatsDClient $statsd)
     {
         $this->redis = $redis;
         $this->logger = $logger;
         $this->doctrine = $doctrine;
         $this->jobWorkers = $jobWorkers;
+        $this->statsd = $statsd;
     }
 
     /**
@@ -104,6 +108,12 @@ class QueueWorker
             return $record;
         });
 
+        $expectedStart = $job->getExecuteAfter() ?: $job->getCreatedAt();
+        $start = microtime(true);
+        $this->statsd->timing('worker.queue.waittime', round(($start - $expectedStart->getTimestamp()) * 1000, 4), [
+            'jobType' => $job->getType(),
+        ]);
+
         $processor = $this->jobWorkers[$job->getType()];
 
         $this->logger->reset();
@@ -119,6 +129,15 @@ class QueueWorker
             ];
         }
 
+        $this->statsd->increment('worker.queue.processed', 1, 1, [
+            'jobType' => $job->getType(),
+            'status' => $result['status'],
+        ]);
+
+        $this->statsd->timing('worker.queue.processtime', round((microtime(true) - $start) * 1000, 4), [
+            'jobType' => $job->getType(),
+        ]);
+
         // If an exception is thrown during a transaction the EntityManager is closed
         // and we won't be able to update the job or handle future jobs
         if (!$this->doctrine->getEntityManager()->isOpen()) {

src/Packagist/WebBundle/Service/UpdaterWorker.php

@@ -71,10 +71,11 @@ class UpdaterWorker
         }
 
         $packageName = $package->getName();
+        $packageVendor = $package->getVendor();
 
         $lockAcquired = $this->locker->lockPackageUpdate($id);
         if (!$lockAcquired) {
-            return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds')];
+            return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds'), 'vendor' => $packageVendor];
         }
 
         $this->logger->info('Updating '.$packageName);
@@ -156,6 +157,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, package appears to be gone',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -168,6 +170,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, invalid composer.json metadata',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -210,6 +213,7 @@ class UpdaterWorker
                     'message' => 'Update of '.$packageName.' failed, package appears to be 404/gone and has been marked as crawled for 1year',
                     'details' => '<pre>'.$output.'</pre>',
                     'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -218,7 +222,8 @@ class UpdaterWorker
                 return [
                     'status' => Job::STATUS_FAILED,
                     'message' => 'Package data of '.$packageName.' could not be downloaded. Could not reach remote VCS server. Please try again later.',
-                    'exception' => $e
+                    'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -227,7 +232,8 @@ class UpdaterWorker
                 return [
                     'status' => Job::STATUS_FAILED,
                     'message' => 'Package data of '.$packageName.' could not be downloaded.',
-                    'exception' => $e
+                    'exception' => $e,
+                    'vendor' => $packageVendor,
                 ];
             }
 
@@ -242,7 +248,8 @@ class UpdaterWorker
         return [
             'status' => Job::STATUS_COMPLETED,
             'message' => 'Update of '.$packageName.' complete',
-            'details' => '<pre>'.$this->cleanupOutput($io->getOutput()).'</pre>'
+            'details' => '<pre>'.$this->cleanupOutput($io->getOutput()).'</pre>',
+            'vendor' => $packageVendor,
         ];
     }
 
@@ -275,6 +282,7 @@ class UpdaterWorker
                             'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been deleted',
                             'details' => '<pre>'.$output.'</pre>',
                             'exception' => $e,
+                            'vendor' => $package->getVendor()
                         ];
                     }
                 } catch (\Throwable $e) {

src/Packagist/WebBundle/Tests/Package/UpdaterTest.php

@@ -58,7 +58,6 @@ class UpdaterTest extends TestCase
 
         $registryMock->expects($this->any())->method('getManager')->willReturn($emMock);
         $registryMock->expects($this->at(1))->method('getRepository')->with('PackagistWebBundle:Version')->willReturn($versionRepoMock);
-        $registryMock->expects($this->at(3))->method('getRepository')->with('PackagistWebBundle:Author')->willReturn($authorRepoMock);
         $this->repositoryMock->expects($this->any())->method('getPackages')->willReturn([
             $packageMock
         ]);

src/Packagist/WebBundle/Twig/PackagistExtension.php

@@ -34,6 +34,7 @@ class PackagistExtension extends \Twig\Extension\AbstractExtension
         return array(
             new \Twig_SimpleFilter('prettify_source_reference', [$this, 'prettifySourceReference']),
             new \Twig_SimpleFilter('gravatar_hash', [$this, 'generateGravatarHash']),
+            new \Twig_SimpleFilter('vendor', [$this, 'getVendor']),
         );
     }
 
@@ -49,6 +50,11 @@ class PackagistExtension extends \Twig\Extension\AbstractExtension
         return 'packagist';
     }
 
+    public function getVendor(string $packageName): string
+    {
+        return preg_replace('{/.*$}', '', $packageName);
+    }
+
     public function numericTest($val)
     {
         return ctype_digit((string) $val);

src/Packagist/WebBundle/Util/UserAgentParser.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Util;
+
+class UserAgentParser
+{
+    /** @var ?string */
+    private $composerVersion;
+    /** @var ?string */
+    private $phpVersion;
+    /** @var ?string */
+    private $os;
+    /** @var ?string */
+    private $httpVersion;
+    /** @var ?bool */
+    private $ci;
+
+    public function __construct(?string $userAgent)
+    {
+        if ($userAgent && preg_match('#^Composer/(?P<composer>[a-z0-9.+-]+) \((?P<os>\S+).*?; PHP (?P<php>[0-9.]+)[^;]*(?:; (?P<http>streams|curl [0-9.]+)[^;]*)?(?P<ci>; CI)?#i', $userAgent, $matches)) {
+            if ($matches['composer'] === 'source' || preg_match('{^[a-f0-9]{40}$}', $matches['composer'])) {
+                $matches['composer'] = 'pre-1.8.5';
+            }
+            $this->composerVersion = preg_replace('{\+[a-f0-9]{40}}', '', $matches['composer']);
+            $this->phpVersion = $matches['php'];
+            $this->os = preg_replace('{^cygwin_nt-.*}', 'cygwin', strtolower($matches['os']));
+            $this->httpVersion = $matches['http'] ?? null;
+            $this->ci = (bool) ($matches['ci'] ?? null);
+        }
+    }
+
+    public function getComposerVersion(): ?string
+    {
+        return $this->composerVersion;
+    }
+
+    public function getPhpVersion(): ?string
+    {
+        return $this->phpVersion;
+    }
+
+    public function getOs(): ?string
+    {
+        return $this->os;
+    }
+
+    public function getHttpVersion(): ?string
+    {
+        return $this->httpVersion;
+    }
+
+    public function getCI(): ?bool
+    {
+        return $this->ci;
+    }
+}