Add version cache for VCS repo to reduce reliance on filesystem cache

src/Packagist/WebBundle/Command/RunWorkersCommand.php

@@ -6,11 +6,24 @@ use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Filesystem\LockHandler;
-use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Command\Command;
 use Seld\Signal\SignalHandler;
+use Packagist\WebBundle\Service\QueueWorker;
+use Psr\Log\LoggerInterface;
 
-class RunWorkersCommand extends ContainerAwareCommand
+class RunWorkersCommand extends Command
 {
+    private $logger;
+    private $worker;
+
+    public function __construct(LoggerInterface $logger, QueueWorker $worker)
+    {
+        $this->logger = $logger;
+        $this->worker = $worker;
+
+        parent::__construct();
+    }
+
     protected function configure()
     {
         $this
@@ -34,16 +47,12 @@ class RunWorkersCommand extends ContainerAwareCommand
         }
 
         try {
-            $logger = $this->getContainer()->get('logger');
-
-            $worker = $this->getContainer()->get('packagist.queue_worker');
-
-            $logger->notice('Worker started successfully');
-            $logger->reset();
+            $this->logger->notice('Worker started successfully');
+            $this->logger->reset();
 
-            $worker->processMessages((int) $input->getOption('messages'));
+            $this->worker->processMessages((int) $input->getOption('messages'));
 
-            $logger->notice('Worker exiting successfully');
+            $this->logger->notice('Worker exiting successfully');
         } finally {
             $lock->release();
         }

src/Packagist/WebBundle/Entity/VersionRepository.php

@@ -157,7 +157,7 @@ class VersionRepository extends EntityRepository
     public function getVersionMetadataForUpdate(Package $package)
     {
         $rows = $this->getEntityManager()->getConnection()->fetchAll(
-            'SELECT id, normalizedVersion, source, softDeletedAt FROM package_version v WHERE v.package_id = :id',
+            'SELECT id, version, normalizedVersion, source, softDeletedAt FROM package_version v WHERE v.package_id = :id',
             ['id' => $package->getId()]
         );
 

src/Packagist/WebBundle/Package/Updater.php

@@ -91,16 +91,12 @@ class Updater
      * @param \Packagist\WebBundle\Entity\Package $package
      * @param RepositoryInterface $repository the repository instance used to update from
      * @param int $flags a few of the constants of this class
-     * @param \DateTime $start
      */
-    public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, \DateTime $start = null): Package
+    public function update(IOInterface $io, Config $config, Package $package, RepositoryInterface $repository, $flags = 0, array $existingVersions = null): Package
     {
         $rfs = new RemoteFilesystem($io, $config);
 
-        if (null === $start) {
-            $start = new \DateTime();
-        }
-        $deleteDate = clone $start;
+        $deleteDate = new \DateTime();
         $deleteDate->modify('-1day');
 
         $em = $this->doctrine->getManager();
@@ -185,7 +181,9 @@ class Updater
             $em->refresh($package);
         }
 
-        $existingVersions = $versionRepository->getVersionMetadataForUpdate($package);
+        if (!$existingVersions) {
+            $existingVersions = $versionRepository->getVersionMetadataForUpdate($package);
+        }
 
         $processedVersions = [];
         $lastProcessed = null;

src/Packagist/WebBundle/Resources/config/services.yml

@@ -5,6 +5,9 @@ services:
         public: false
         bind:
 
+    Packagist\WebBundle\Command\:
+        resource: '../../Command/*'
+        tags: ['console.command']
 
     packagist.twig.extension:
         public: true

src/Packagist/WebBundle/Service/UpdaterWorker.php

@@ -17,6 +17,7 @@ use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Helper\HelperSet;
 use Monolog\Handler\StreamHandler;
 use Packagist\WebBundle\Entity\Package;
+use Packagist\WebBundle\Entity\Version;
 use Packagist\WebBundle\Package\Updater;
 use Packagist\WebBundle\Entity\Job;
 use Packagist\WebBundle\Model\PackageManager;
@@ -83,22 +84,40 @@ class UpdaterWorker
 
         try {
             $flags = 0;
+            $useVersionCache = true;
             if ($job->getPayload()['update_equal_refs'] === true) {
                 $flags = Updater::UPDATE_EQUAL_REFS;
+                $useVersionCache = false;
             }
             if ($job->getPayload()['delete_before'] === true) {
                 $flags = Updater::DELETE_BEFORE;
+                $useVersionCache = false;
             }
 
             // prepare dependencies
             $loader = new ValidatingArrayLoader(new ArrayLoader());
 
+            $versionCache = null;
+            $existingVersions = null;
+
+            if ($useVersionCache) {
+                $existingVersions = $em->getRepository(Version::class)->getVersionMetadataForUpdate($package);
+                $versionCache = new VersionCache($package, $existingVersions);
+            }
+
             // prepare repository
-            $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config);
+            $repository = new VcsRepository(
+                ['url' => $package->getRepository(), 'options' => ['retry-auth-failure' => false]],
+                $io,
+                $config,
+                null,
+                null,
+                $versionCache
+            );
             $repository->setLoader($loader);
 
             // perform the actual update (fetch and re-scan the repository's source)
-            $package = $this->updater->update($io, $config, $package, $repository, $flags);
+            $package = $this->updater->update($io, $config, $package, $repository, $flags, $existingVersions);
 
             // github update downgraded to a git clone, this should not happen, so check through API whether the package still exists
             if (preg_match('{[@/]github.com[:/]([^/]+/[^/]+?)(\.git)?$}i', $package->getRepository(), $match) && 0 === strpos($repository->getDriver()->getUrl(), 'git@')) {
@@ -209,10 +228,18 @@ class UpdaterWorker
         return [
             'status' => Job::STATUS_COMPLETED,
             'message' => 'Update of '.$packageName.' complete',
-            'details' => '<pre>'.$io->getOutput().'</pre>'
+            'details' => '<pre>'.$this->cleanupOutput($io->getOutput()).'</pre>'
         ];
     }
 
+    private function cleanupOutput($str)
+    {
+        return preg_replace('{
+            Reading\ composer.json\ of\ <span(.+?)>(?P<pkg>[^<]+)</span>\ \(<span(.+?)>(?P<version>[^<]+)</span>\)\r?\n
+            (?P<cache>Found\ cached\ composer.json\ of\ <span(.+?)>(?P=pkg)</span>\ \(<span(.+?)>(?P=version)</span>\)\r?\n)
+        }x', '$5', $str);
+    }
+
     private function checkForDeadGitHubPackage(Package $package, $match, $io, $output)
     {
         $rfs = new RemoteFilesystem($io);

src/Packagist/WebBundle/Service/VersionCache.php

@@ -0,0 +1,37 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Composer\Repository\VersionCacheInterface;
+use Packagist\WebBundle\Entity\Package;
+use Packagist\WebBundle\Entity\Version;
+
+class VersionCache implements VersionCacheInterface
+{
+    /** @var Version[] */
+    private $versionCache;
+    private $package;
+
+    public function __construct(Package $package, array $existingVersions)
+    {
+        $this->versionCache = [];
+        foreach ($existingVersions as $version) {
+            $this->versionCache[$version['version']] = $version;
+        }
+        $this->package = $package;
+    }
+
+    public function getVersionPackage($version, $identifier)
+    {
+        if (!empty($this->versionCache[$version]['source']['reference']) && $this->versionCache[$version]['source']['reference'] === $identifier) {
+            return [
+                'name' => $this->package->getName(),
+                'version' => $this->versionCache[$version]['version'],
+                'version_normalized' => $this->versionCache[$version]['normalizedVersion'],
+                'source' => $this->versionCache[$version]['source'],
+            ];
+        }
+
+        return null;
+    }
+}