Improvements to suspect package handling

src/Packagist/WebBundle/Controller/PackageController.php

@@ -299,6 +299,38 @@ class PackageController extends Controller
         ));
     }
 
+    /**
+     * @Route(
+     *     "/spam",
+     *     name="view_spam",
+     *     defaults={"_format"="html"},
+     *     methods={"GET"}
+     * )
+     */
+    public function viewSpamAction(Request $req)
+    {
+        if (!$this->getUser() || !$this->isGranted('ROLE_ANTISPAM')) {
+            throw new NotFoundHttpException();
+        }
+
+        $page = $req->query->get('page', 1);
+
+        /** @var PackageRepository $repo */
+        $repo = $this->getDoctrine()->getRepository(Package::class);
+        $count = $repo->getSuspectPackageCount();
+        $packages = $repo->getSuspectPackages(($page - 1) * 15, 15);
+
+        $paginator = new Pagerfanta(new FixedAdapter($count, $packages));
+        $paginator->setMaxPerPage(15);
+        $paginator->setCurrentPage($page, false, true);
+
+        $data['packages'] = $paginator;
+        $data['count'] = $count;
+        $data['meta'] = $this->getPackagesMetadata($data['packages']);
+
+        return $this->render('PackagistWebBundle:package:spam.html.twig', $data);
+    }
+
     /**
      * @Template()
      * @Route(
@@ -399,7 +431,12 @@ class PackageController extends Controller
         );
 
         try {
-            $data['downloads'] = $this->get('packagist.download_manager')->getDownloads($package);
+            $data['downloads'] = $this->get('packagist.download_manager')->getDownloads($package, null, true);
+
+            if (!$package->isSuspect() && ($data['downloads']['total'] ?? 0) <= 10 && ($data['downloads']['views'] ?? 0) >= 100) {
+                $package->setSuspect('Too many views');
+                $repo->markPackageSuspect($package);
+            }
 
             if ($this->getUser()) {
                 $data['is_favorite'] = $this->get('packagist.favorite_manager')->isMarked($this->getUser(), $package);

src/Packagist/WebBundle/Controller/UserController.php

@@ -103,7 +103,7 @@ class UserController extends Controller
 
             $doctrine->getConnection()->executeUpdate(
                 'UPDATE package p JOIN maintainers_packages mp ON mp.package_id = p.id
-                 SET abandoned = 1, replacementPackage = "spam/spam", description = "", readme = "", indexedAt = NULL, dumpedAt = "2100-01-01 00:00:00"
+                 SET abandoned = 1, replacementPackage = "spam/spam", suspect = "spam", indexedAt = NULL, dumpedAt = "2100-01-01 00:00:00"
                  WHERE mp.user_id = :userId',
                 ['userId' => $user->getId()]
             );

src/Packagist/WebBundle/Entity/Package.php

@@ -168,6 +168,11 @@ class Package
      */
     private $updateFailureNotified = false;
 
+    /**
+     * @ORM\Column(type="string", length=255, nullable=true)
+     */
+    private $suspect;
+
     private $entityRepository;
     private $router;
 
@@ -863,6 +868,21 @@ class Package
         return $this->updateFailureNotified;
     }
 
+    public function setSuspect(?string $reason)
+    {
+        $this->suspect = $reason;
+    }
+
+    public function isSuspect(): bool
+    {
+        return !is_null($this->suspect);
+    }
+
+    public function getSuspect(): ?string
+    {
+        return $this->suspect;
+    }
+
     /**
      * @return boolean
      */

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -331,6 +331,35 @@ class PackageRepository extends EntityRepository
         return true;
     }
 
+    public function markPackageSuspect(Package $package)
+    {
+        $sql = 'UPDATE package SET suspect = :suspect WHERE id = :id';
+        $this->getEntityManager()->getConnection()->executeUpdate($sql, ['suspect' => $package->getSuspect(), 'id' => $package->getId()]);
+    }
+
+    public function getSuspectPackageCount()
+    {
+        $sql = 'SELECT COUNT(*) count FROM package p WHERE p.suspect IS NOT NULL AND (p.replacementPackage IS NULL OR p.replacementPackage != "spam/spam")';
+
+        $stmt = $this->getEntityManager()->getConnection()->executeQuery($sql);
+        $result = $stmt->fetchAll();
+        $stmt->closeCursor();
+
+        return (int) $result[0]['count'];
+    }
+
+    public function getSuspectPackages($offset = 0, $limit = 15)
+    {
+        $sql = 'SELECT p.id, p.name, p.description, p.language, p.abandoned, p.replacementPackage
+            FROM package p WHERE p.suspect IS NOT NULL AND (p.replacementPackage IS NULL OR p.replacementPackage != "spam/spam") ORDER BY p.createdAt DESC LIMIT '.((int)$limit).' OFFSET '.((int)$offset);
+
+        $stmt = $this->getEntityManager()->getConnection()->executeQuery($sql);
+        $result = $stmt->fetchAll();
+        $stmt->closeCursor();
+
+        return $result;
+    }
+
     public function getDependentCount($name)
     {
         $sql = 'SELECT COUNT(*) count FROM (

src/Packagist/WebBundle/Model/DownloadManager.php

@@ -42,7 +42,7 @@ class DownloadManager
      * @param \Packagist\WebBundle\Entity\Version|int|null $version
      * @return array
      */
-    public function getDownloads($package, $version = null)
+    public function getDownloads($package, $version = null, bool $incrViews = false)
     {
         if ($package instanceof Package) {
             $package = $package->getId();
@@ -91,12 +91,17 @@ class DownloadManager
         // how much of yesterday to add to today to make it a whole day (sort of..)
         $dayRatio = (2400 - (int) date('Hi')) / 2400;
 
-        return [
+        $result = [
             'total' => $total,
             'monthly' => $monthly,
             'daily' => round(($redisData[0] ?? $dlData[$todayDate] ?? 0) + (($redisData[1] ?? $dlData[$yesterdayDate] ?? 0) * $dayRatio)),
-            'views' => $this->redis->incr('views:'.$package),
         ];
+
+        if ($incrViews) {
+            $result['views'] = $this->redis->incr('views:'.$package);
+        }
+
+        return $result;
     }
 
     /**

src/Packagist/WebBundle/Model/PackageManager.php

@@ -16,6 +16,7 @@ use Symfony\Bridge\Doctrine\RegistryInterface;
 use Packagist\WebBundle\Entity\Package;
 use Psr\Log\LoggerInterface;
 use Algolia\AlgoliaSearch\SearchClient;
+use Predis\Client;
 use Packagist\WebBundle\Service\GitHubUserMigrationWorker;
 
 /**
@@ -35,7 +36,7 @@ class PackageManager
     protected $githubWorker;
     protected $metadataDir;
 
-    public function __construct(RegistryInterface $doctrine, \Swift_Mailer $mailer, \Swift_Mailer $instantMailer, \Twig_Environment $twig, LoggerInterface $logger, array $options, ProviderManager $providerManager, SearchClient $algoliaClient, string $algoliaIndexName, GitHubUserMigrationWorker $githubWorker, string $metadataDir)
+    public function __construct(RegistryInterface $doctrine, \Swift_Mailer $mailer, \Swift_Mailer $instantMailer, \Twig_Environment $twig, LoggerInterface $logger, array $options, ProviderManager $providerManager, SearchClient $algoliaClient, string $algoliaIndexName, GitHubUserMigrationWorker $githubWorker, string $metadataDir, Client $redis)
     {
         $this->doctrine = $doctrine;
         $this->mailer = $mailer;
@@ -48,6 +49,7 @@ class PackageManager
         $this->algoliaIndexName  = $algoliaIndexName;
         $this->githubWorker  = $githubWorker;
         $this->metadataDir  = $metadataDir;
+        $this->redis = $redis;
     }
 
     public function deletePackage(Package $package)
@@ -84,6 +86,7 @@ class PackageManager
         }
 
         $this->providerManager->deletePackage($package);
+        $packageId = $package->getId();
         $packageName = $package->getName();
 
         $em->remove($package);
@@ -104,6 +107,12 @@ class PackageManager
             @unlink($metadataV2Dev.'.gz');
         }
 
+        // delete redis stats
+        try {
+            $this->redis->del('views:'.$packageId);
+        } catch (\Predis\Connection\ConnectionException $e) {
+        }
+
         // attempt search index cleanup
         try {
             $indexName = $this->algoliaIndexName;

src/Packagist/WebBundle/Resources/config/services.yml

@@ -161,6 +161,7 @@ services:
             - '%algolia.index_name%'
             - '@github_user_migration_worker'
             - '%packagist_metadata_dir%'
+            - '@snc_redis.default_client'
 
     packagist.profile.form.type:
         public: true

src/Packagist/WebBundle/Resources/views/package/view_package.html.twig

@@ -2,11 +2,6 @@
 
 {% set showSearchDesc = 'hide' %}
 
-{% set allowIndex = true %}
-{% if downloads.total is defined and downloads.views >= 100 and downloads.total <= 10 %}
-    {% set allowIndex = false %}
-{% endif %}
-
 {% block title %}{{ package.name }} - {{ parent() }}{% endblock %}
 
 {% block head_feeds %}
@@ -21,7 +16,7 @@
 
 {% block head_additions %}
     <link rel="stylesheet" href="{{ asset('css/github/markdown.css') }}">
-    {% if not allowIndex %}
+    {% if package.isSuspect() %}
         <meta name="robots" content="noindex, nofollow">
     {% endif %}
 {% endblock %}
@@ -292,7 +287,7 @@
             {% endif %}
         </div>
 
-        {% if package.readme != null and (allowIndex or hasActions) %}
+        {% if package.readme != null and (not package.isSuspect() or hasActions) %}
             <hr class="clearfix">
             <div class="readme markdown-body">
                 <h1>README</h1>