Add new metadata format

src/Packagist/WebBundle/Controller/PackageController.php

@@ -97,7 +97,7 @@ class PackageController extends Controller
         try {
             $since = new DateTimeImmutable('@'.$since);
         } catch (\Exception $e) {
-            return new JsonResponse(['error' => 'Invalid "since" query parameter, make sure you store the timestamp returned and re-use it in the next query. Use '.$this->generateUrl('updated_packages', ['since' => time() - 60], UrlGeneratorInterface::ABSOLUTE_URL).' to initialize it.'], 400);
+            return new JsonResponse(['error' => 'Invalid "since" query parameter, make sure you store the timestamp returned and re-use it in the next query. Use '.$this->generateUrl('updated_packages', ['since' => time() - 180], UrlGeneratorInterface::ABSOLUTE_URL).' to initialize it.'], 400);
         }
 
         /** @var PackageRepository $repo */
@@ -105,7 +105,9 @@ class PackageController extends Controller
 
         $names = $repo->getPackageNamesUpdatedSince($since);
 
-        return new JsonResponse(['packageNames' => $names, 'timestamp' => $now]);
+        $lastDumpTime = $this->get('snc_redis.default_client')->get('last_metadata_dump_time') ?: (time() - 60);
+
+        return new JsonResponse(['packageNames' => $names, 'timestamp' => $lastDumpTime]);
     }
 
     /**

src/Packagist/WebBundle/Entity/Version.php

@@ -213,10 +213,14 @@ class Version
 
     public function toArray(array $versionData)
     {
-        $tags = array();
-        foreach ($this->getTags() as $tag) {
-            /** @var $tag Tag */
-            $tags[] = $tag->getName();
+        if (isset($versionData[$this->id]['tags'])) {
+            $tags = $versionData[$this->id]['tags'];
+        } else {
+            $tags = array();
+            foreach ($this->getTags() as $tag) {
+                /** @var $tag Tag */
+                $tags[] = $tag->getName();
+            }
         }
 
         if (isset($versionData[$this->id]['authors'])) {
@@ -291,6 +295,15 @@ class Version
         return $data;
     }
 
+    public function toV2Array(array $versionData)
+    {
+        $array = $this->toArray($versionData);
+
+        unset($array['keywords'], $array['authors']);
+
+        return $array;
+    }
+
     public function equals(Version $version)
     {
         return strtolower($version->getName()) === strtolower($this->getName())
@@ -327,18 +340,28 @@ class Version
         return $this->name;
     }
 
-    public function getNames()
+    public function getNames(array $versionData = null)
     {
         $names = array(
             strtolower($this->name) => true
         );
 
-        foreach ($this->getReplace() as $link) {
-            $names[strtolower($link->getPackageName())] = true;
-        }
+        if (isset($versionData[$this->id])) {
+            foreach (($versionData[$this->id]['replace'] ?? []) as $link) {
+                $names[strtolower($link['name'])] = true;
+            }
 
-        foreach ($this->getProvide() as $link) {
-            $names[strtolower($link->getPackageName())] = true;
+            foreach (($versionData[$this->id]['provide'] ?? []) as $link) {
+                $names[strtolower($link['name'])] = true;
+            }
+        } else {
+            foreach ($this->getReplace() as $link) {
+                $names[strtolower($link->getPackageName())] = true;
+            }
+
+            foreach ($this->getProvide() as $link) {
+                $names[strtolower($link->getPackageName())] = true;
+            }
         }
 
         return array_keys($names);

src/Packagist/WebBundle/Entity/VersionRepository.php

@@ -99,6 +99,8 @@ class VersionRepository extends EntityRepository
                 'conflict' => [],
                 'provide' => [],
                 'replace' => [],
+                'authors' => [],
+                'tags' => [],
             ];
         }
 
@@ -124,6 +126,16 @@ class VersionRepository extends EntityRepository
             $result[$versionId]['authors'][] = array_filter($row);
         }
 
+        $rows = $this->getEntityManager()->getConnection()->fetchAll(
+            'SELECT vt.version_id, name FROM tag t JOIN version_tag vt ON vt.tag_id = t.id WHERE vt.version_id IN (:ids)',
+            ['ids' => $versionIds],
+            ['ids' => Connection::PARAM_INT_ARRAY]
+        );
+        foreach ($rows as $row) {
+            $versionId = $row['version_id'];
+            $result[$versionId]['tags'][] = $row['name'];
+        }
+
         return $result;
     }
 

src/Packagist/WebBundle/Model/PackageManager.php

@@ -32,8 +32,9 @@ class PackageManager
     protected $algoliaClient;
     protected $algoliaIndexName;
     protected $githubWorker;
+    protected $metadataDir;
 
-    public function __construct(RegistryInterface $doctrine, \Swift_Mailer $mailer, \Twig_Environment $twig, LoggerInterface $logger, array $options, ProviderManager $providerManager, AlgoliaClient $algoliaClient, string $algoliaIndexName, GitHubUserMigrationWorker $githubWorker)
+    public function __construct(RegistryInterface $doctrine, \Swift_Mailer $mailer, \Twig_Environment $twig, LoggerInterface $logger, array $options, ProviderManager $providerManager, AlgoliaClient $algoliaClient, string $algoliaIndexName, GitHubUserMigrationWorker $githubWorker, string $metadataDir)
     {
         $this->doctrine = $doctrine;
         $this->mailer = $mailer;
@@ -44,6 +45,7 @@ class PackageManager
         $this->algoliaClient = $algoliaClient;
         $this->algoliaIndexName  = $algoliaIndexName;
         $this->githubWorker  = $githubWorker;
+        $this->metadataDir  = $metadataDir;
     }
 
     public function deletePackage(Package $package)
@@ -73,6 +75,14 @@ class PackageManager
         $em->remove($package);
         $em->flush();
 
+        $metadataV2 = $this->metadataDir.'/p2/'.strtolower($packageName).'.json';
+        if (file_exists($metadataV2)) {
+            @unlink($metadataV2);
+        }
+        if (file_exists($metadataV2.'.gz')) {
+            @unlink($metadataV2.'.gz');
+        }
+
         // attempt search index cleanup
         try {
             $indexName = $this->algoliaIndexName;

src/Packagist/WebBundle/Package/SymlinkDumper.php

@@ -18,7 +18,9 @@ use Symfony\Bridge\Doctrine\RegistryInterface;
 use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
 use Symfony\Component\Finder\Finder;
 use Packagist\WebBundle\Entity\Version;
+use Packagist\WebBundle\Entity\Package;
 use Doctrine\DBAL\Connection;
+use Predis\Client;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -56,6 +58,11 @@ class SymlinkDumper
      */
     protected $router;
 
+    /**
+     * @var Client
+     */
+    protected $redis;
+
     /**
      * Data cache
      * @var array
@@ -74,12 +81,24 @@ class SymlinkDumper
      */
     private $individualFiles = array();
 
+    /**
+     * Data cache
+     * @var array
+     */
+    private $individualFilesV2 = array();
+
     /**
      * Modified times of individual files
      * @var array
      */
     private $individualFilesMtime = array();
 
+    /**
+     * Modified times of individual files V2
+     * @var array
+     */
+    private $individualFilesV2Mtime = array();
+
     /**
      * Stores all the disk writes to be replicated in the second build dir after the symlink has been swapped
      * @var array
@@ -102,7 +121,7 @@ class SymlinkDumper
      * @param string                $targetDir
      * @param int                   $compress
      */
-    public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, $webDir, $targetDir, $compress)
+    public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, UrlGeneratorInterface $router, Client $redis, $webDir, $targetDir, $compress)
     {
         $this->doctrine = $doctrine;
         $this->fs = $filesystem;
@@ -111,6 +130,7 @@ class SymlinkDumper
         $this->webDir = realpath($webDir);
         $this->buildDir = $targetDir;
         $this->compress = $compress;
+        $this->redis = $redis;
     }
 
     /**
@@ -129,6 +149,7 @@ class SymlinkDumper
 
         $buildDirA = $this->buildDir.'/a';
         $buildDirB = $this->buildDir.'/b';
+        $buildDirV2 = $this->buildDir.'/p2';
 
         // initialize
         $initialRun = false;
@@ -140,6 +161,9 @@ class SymlinkDumper
             $this->fs->mkdir($buildDirA);
             $this->fs->mkdir($buildDirB);
         }
+        if (!is_dir($buildDirV2)) {
+            $this->fs->mkdir($buildDirV2);
+        }
 
         // set build dir to the not-active one
         if (realpath($webDir.'/p') === realpath($buildDirA)) {
@@ -149,6 +173,7 @@ class SymlinkDumper
             $buildDir = realpath($buildDirA);
             $oldBuildDir = realpath($buildDirB);
         }
+        $buildDirV2 = realpath($buildDirV2);
 
         // copy existing stuff for smooth BC transition
         if ($initialRun && !$force) {
@@ -169,6 +194,7 @@ class SymlinkDumper
         if ($verbose) {
             echo 'Web dir is '.$webDir.'/p ('.realpath($webDir.'/p').')'.PHP_EOL;
             echo 'Build dir is '.$buildDir.PHP_EOL;
+            echo 'Build v2 dir is '.$buildDirV2.PHP_EOL;
         }
 
         // clean the build dir to start over if we are re-dumping everything
@@ -190,6 +216,7 @@ class SymlinkDumper
 
         try {
             $modifiedIndividualFiles = array();
+            $modifiedV2Files = array();
 
             $total = count($packageIds);
             $current = 0;
@@ -238,7 +265,7 @@ class SymlinkDumper
                     }
                     $versionData = $versionRepo->getVersionData($versionIds);
                     foreach ($package->getVersions() as $version) {
-                        foreach (array_slice($version->getNames(), 0, 150) as $versionName) {
+                        foreach (array_slice($version->getNames($versionData), 0, 150) as $versionName) {
                             if (!preg_match('{^[A-Za-z0-9_-][A-Za-z0-9_.-]*/[A-Za-z0-9_-][A-Za-z0-9_.-]*$}', $versionName) || strpos($versionName, '..')) {
                                 continue;
                             }
@@ -249,8 +276,14 @@ class SymlinkDumper
                             $modifiedIndividualFiles[$key] = true;
                             $affectedFiles[$key] = true;
                         }
+
                     }
 
+                    // dump v2 format
+                    $key = $name.'.json';
+                    $this->dumpPackageToV2File($package, $versionData, $key);
+                    $modifiedV2Files[$key] = true;
+
                     // store affected files to clean up properly in the next update
                     $this->fs->mkdir(dirname($buildDir.'/'.$name));
                     $this->writeFile($buildDir.'/'.$name.'.files', json_encode(array_keys($affectedFiles)));
@@ -266,6 +299,7 @@ class SymlinkDumper
                         echo 'Dumping individual files'.PHP_EOL;
                     }
                     $this->dumpIndividualFiles($buildDir);
+                    $this->dumpIndividualFilesV2($buildDirV2);
                 }
             }
 
@@ -304,7 +338,7 @@ class SymlinkDumper
             $this->rootFile['notify'] = str_replace('VND/PKG', '%package%', $url);
             $this->rootFile['notify-batch'] = $this->router->generate('track_download_batch', [], UrlGeneratorInterface::ABSOLUTE_URL);
             $this->rootFile['providers-url'] = $this->router->generate('home', []) . 'p/%package%$%hash%.json';
-            $this->rootFile['metadata-url'] = $this->router->generate('home', []) . 'p/%package%.json'; // TODO make this p2/ once we dump the new format there
+            $this->rootFile['metadata-url'] = $this->router->generate('home', []) . 'p2/%package%.json';
             $this->rootFile['search'] = $this->router->generate('search', ['_format' => 'json'], UrlGeneratorInterface::ABSOLUTE_URL) . '?q=%query%&type=%type%';
 
             if ($verbose) {
@@ -334,6 +368,11 @@ class SymlinkDumper
                 echo 'Putting new files in production'.PHP_EOL;
             }
 
+            if (!file_exists($webDir.'/p2') && !@symlink($buildDirV2, $webDir.'/p2')) {
+                echo 'Warning: Could not symlink the build dir v2 into the web dir';
+                throw new \RuntimeException('Could not symlink the build dir v2 into the web dir');
+            }
+
             // move away old files for BC update
             if ($initialRun && file_exists($webDir.'/p') && !is_link($webDir.'/p')) {
                 rename($webDir.'/p', $webDir.'/p-old');
@@ -416,6 +455,8 @@ class SymlinkDumper
         if ($verbose) {
             echo 'Updating package dump times'.PHP_EOL;
         }
+
+        $maxDumpTime = 0;
         foreach ($dumpTimeUpdates as $dt => $ids) {
             $retries = 5;
             // retry loop in case of a lock timeout
@@ -429,6 +470,7 @@ class SymlinkDumper
                         ],
                         ['ids' => Connection::PARAM_INT_ARRAY]
                     );
+                    break;
                 } catch (\Exception $e) {
                     if (!$retries) {
                         throw $e;
@@ -436,6 +478,18 @@ class SymlinkDumper
                     sleep(2);
                 }
             }
+
+            $maxDumpTime = max($maxDumpTime, strtotime($dt));
+        }
+
+        if ($maxDumpTime !== 0) {
+            $this->redis->set('last_metadata_dump_time', $maxDumpTime + 1);
+
+            // make sure no next dumper has a chance to start and dump things within the same second as $maxDumpTime
+            // as in updatedSince we will return the updates from the next second only (the +1 above) to avoid serving the same updates twice
+            if (time() === $maxDumpTime) {
+                sleep(1);
+            }
         }
 
         // TODO when a package is deleted, it should be removed from provider files, or marked for removal at least
@@ -630,6 +684,57 @@ class SymlinkDumper
         }
     }
 
+    private function dumpIndividualFilesV2($dir)
+    {
+        // dump individual files to build dir
+        foreach ($this->individualFilesV2 as $key => $data) {
+            $path = $dir . '/' . $key;
+            $this->fs->mkdir(dirname($path));
+
+            $json = json_encode($data, JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE);
+            $this->writeFile($path, $json, $this->individualFilesV2Mtime[$key]);
+        }
+
+        $this->individualFilesV2 = array();
+        $this->individualFilesV2Mtime = array();
+    }
+
+    private function dumpPackageToV2File(Package $package, $versionData, string $packageKey)
+    {
+        $deduplicatedVersions = [];
+        $uniqKeys = ['version', 'version_normalized', 'source', 'dist', 'time'];
+        $mtime = 0;
+
+        foreach ($package->getVersions() as $version) {
+            $versionArray = $version->toV2Array($versionData);
+
+            $filtered = $versionArray;
+            foreach ($uniqKeys as $key) {
+                unset($filtered[$key]);
+            }
+            $hash = md5(json_encode($filtered));
+
+            if (isset($deduplicatedVersions[$hash])) {
+                foreach ($uniqKeys as $key) {
+                    $deduplicatedVersions[$hash][$key.'s'][] = $versionArray[$key];
+                }
+            } else {
+                foreach ($uniqKeys as $key) {
+                    $filtered[$key.'s'] = [$versionArray[$key]];
+                }
+                $deduplicatedVersions[$hash] = $filtered;
+            }
+
+            $mtime = max($mtime, $version->getReleasedAt() ? $version->getReleasedAt()->getTimestamp() : time());
+        }
+
+        // get rid of md5 hash keys
+        $deduplicatedVersions = array_values($deduplicatedVersions);
+
+        $this->individualFilesV2[$packageKey]['packages'][strtolower($package->getName())] = $deduplicatedVersions;
+        $this->individualFilesV2Mtime[$packageKey] = $mtime;
+    }
+
     private function clearDirectory($path)
     {
         if (!$this->removeDirectory($path)) {

src/Packagist/WebBundle/Resources/config/services.yml

@@ -35,7 +35,7 @@ services:
 
     packagist.package_dumper:
         class: Packagist\WebBundle\Package\SymlinkDumper
-        arguments: [ '@doctrine', '@filesystem', '@router', '%kernel.root_dir%/../web/', '%packagist_metadata_dir%', '%packagist_dumper_compress%' ]
+        arguments: [ '@doctrine', '@filesystem', '@router', '@snc_redis.default_client', '%kernel.root_dir%/../web/', '%packagist_metadata_dir%', '%packagist_dumper_compress%' ]
 
     packagist.user_provider:
         class: Packagist\WebBundle\Security\Provider\UserProvider
@@ -117,6 +117,7 @@ services:
             - '@packagist.algolia.client'
             - '%algolia.index_name%'
             - '@github_user_migration_worker'
+            - '%packagist_metadata_dir%'
 
     packagist.profile.form.type:
         class: Packagist\WebBundle\Form\Type\ProfileFormType