Add packagist:dump command which dumps a static repo in new smarter format

.gitignore

@@ -1,4 +1,5 @@
 web/bundles/
+web/packages*.json
 app/config/parameters.yml
 app/bootstrap*
 app/cache/*

src/Packagist/WebBundle/Command/DumpPackagesCommand.php

@@ -0,0 +1,67 @@
+<?php
+
+/*
+ * This file is part of Packagist.
+ *
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
+ *     Nils Adermann <naderman@naderman.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Packagist\WebBundle\Command;
+
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ */
+class DumpPackagesCommand extends ContainerAwareCommand
+{
+    /**
+     * {@inheritdoc}
+     */
+    protected function configure()
+    {
+        $this
+            ->setName('packagist:dump')
+            ->setDefinition(array(
+                new InputOption('force', null, InputOption::VALUE_NONE, 'Force a dump of all packages'),
+            ))
+            ->setDescription('Dumps the packages into a packages.json + included files')
+        ;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $force = $input->getOption('force');
+
+        $doctrine = $this->getContainer()->get('doctrine');
+
+        if ($force) {
+            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getFullPackages();
+        } else {
+            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackagesForDumping();
+        }
+
+        $lock = $this->getContainer()->getParameter('kernel.cache_dir').'/composer-dumper.lock';
+        $timeout = 600;
+
+        // another dumper is still active
+        if (file_exists($lock) && filemtime($lock) > time() - $timeout) {
+            return;
+        }
+
+        touch($lock);
+        $this->getContainer()->get('packagist.package_dumper')->dump($packages);
+        unlink($lock);
+    }
+}

src/Packagist/WebBundle/Entity/Package.php

@@ -24,7 +24,12 @@ use Composer\Repository\RepositoryManager;
  * @ORM\Entity(repositoryClass="Packagist\WebBundle\Entity\PackageRepository")
  * @ORM\Table(
  *     name="package",
- *     uniqueConstraints={@ORM\UniqueConstraint(name="name_idx", columns={"name"})}
+ *     uniqueConstraints={@ORM\UniqueConstraint(name="name_idx", columns={"name"})},
+ *     indexes={
+ *         @ORM\Index(name="indexed_idx",columns={"indexedAt"}),
+ *         @ORM\Index(name="crawled_idx",columns={"crawledAt"}),
+ *         @ORM\Index(name="dumped_idx",columns={"dumpedAt"})
+ *     }
  * )
  * @Assert\Callback(methods={"isPackageUnique","isRepositoryValid"})
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -94,6 +99,11 @@ class Package
      */
     private $indexedAt;
 
+    /**
+     * @ORM\Column(type="datetime", nullable=true)
+     */
+    private $dumpedAt;
+
     /**
      * @ORM\Column(type="boolean")
      */
@@ -383,6 +393,26 @@ class Package
         return $this->indexedAt;
     }
 
+    /**
+     * Set dumpedAt
+     *
+     * @param \DateTime $dumpedAt
+     */
+    public function setDumpedAt($dumpedAt)
+    {
+        $this->dumpedAt = $dumpedAt;
+    }
+
+    /**
+     * Get dumpedAt
+     *
+     * @return \DateTime
+     */
+    public function getDumpedAt()
+    {
+        return $this->dumpedAt;
+    }
+
     /**
      * Add maintainers
      *

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -98,6 +98,25 @@ class PackageRepository extends EntityRepository
         return $qb->getQuery()->getResult();
     }
 
+    public function getStalePackagesForDumping()
+    {
+        $qb = $this->getEntityManager()->createQueryBuilder();
+        $qb->select('p, v, t, a, req, rec, sug, rep, con, pro')
+            ->from('Packagist\WebBundle\Entity\Package', 'p')
+            ->leftJoin('p.versions', 'v')
+            ->leftJoin('v.tags', 't')
+            ->leftJoin('v.authors', 'a')
+            ->leftJoin('v.require', 'req')
+            ->leftJoin('v.recommend', 'rec')
+            ->leftJoin('v.suggest', 'sug')
+            ->leftJoin('v.replace', 'rep')
+            ->leftJoin('v.conflict', 'con')
+            ->leftJoin('v.provide', 'pro')
+            ->where('p.dumpedAt IS NULL OR p.dumpedAt < p.crawledAt');
+
+        return $qb->getQuery()->getResult();
+    }
+
     public function findOneByName($name)
     {
         $qb = $this->getBaseQueryBuilder()

src/Packagist/WebBundle/Entity/Version.php

@@ -642,6 +642,14 @@ class Version
         return $this->development;
     }
 
+    /**
+     * @return Boolean
+     */
+    public function isDevelopment()
+    {
+        return $this->getDevelopment();
+    }
+
     /**
      * Add tags
      *

src/Packagist/WebBundle/Package/Dumper.php

@@ -0,0 +1,187 @@
+<?php
+
+/*
+ * This file is part of Packagist.
+ *
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
+ *     Nils Adermann <naderman@naderman.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Packagist\WebBundle\Package;
+
+use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Bridge\Doctrine\RegistryInterface;
+use Packagist\WebBundle\Entity\Version;
+
+/**
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ */
+class Dumper
+{
+    /**
+     * Doctrine
+     * @var RegistryInterface
+     */
+    protected $doctrine;
+
+    /**
+     * @var Filesystem
+     */
+    protected $fs;
+
+    /**
+     * @var string
+     */
+    protected $webDir;
+
+    /**
+     * @var string
+     */
+    protected $cacheDir;
+
+    /**
+     * Data cache
+     * @var array
+     */
+    private $files = array();
+
+    /**
+     * Constructor
+     *
+     * @param RegistryInterface $doctrine
+     * @param string $webDir web root
+     * @param string $cacheDir cache dir
+     */
+    public function __construct(RegistryInterface $doctrine, Filesystem $filesystem, $webDir, $cacheDir)
+    {
+        $this->doctrine = $doctrine;
+        $this->fs = $filesystem;
+        $this->webDir = $webDir;
+        $this->cacheDir = $cacheDir . '/composer-packages-build/';
+    }
+
+    /**
+     * Dump a set of packages to the web root
+     *
+     * @param array $packages
+     * @param Boolean $force
+     */
+    public function dump(array $packages, $force = false)
+    {
+        // prepare build dir
+        $webDir = realpath($this->webDir);
+        $buildDir = realpath($this->cacheDir);
+        $this->fs->remove($buildDir);
+        $this->fs->mkdir($buildDir);
+        if (!$force) {
+            foreach (glob($webDir.'/packages*.json') as $file) {
+                copy($file, $buildDir.'/'.basename($file));
+            }
+        }
+
+        $modifiedFiles = array();
+
+        // prepare packages in memory
+        foreach ($packages as $package) {
+            // clean up all versions of that package
+            foreach (glob($buildDir.'/packages*.json') as $file) {
+                $key = basename($file);
+                $this->loadFile($file);
+                if (isset($this->files[$key]['packages'][$package->getName()])) {
+                    unset($this->files[$key]['packages'][$package->getName()]);
+                    $modifiedFiles[$key] = true;
+                }
+            }
+
+            // (re)write versions
+            foreach ($package->getVersions() as $version) {
+                $file = $buildDir.'/'.$this->getTargetFile($version);
+                $modifiedFiles[basename($file)] = true;
+                $this->dumpVersion($version, $file);
+            }
+
+            $package->setDumpedAt(new \DateTime);
+        }
+
+        // prepare root file
+        $rootFile = $buildDir.'/packages.json';
+        $this->loadFile($rootFile);
+        if (!isset($this->files['packages.json']['packages'])) {
+            $this->files['packages.json']['packages'] = array();
+        }
+
+        // dump files to build dir
+        foreach ($modifiedFiles as $file => $dummy) {
+            $this->dumpFile($file);
+            $this->files['packages.json']['includes'][$file] = array('sha1' => sha1_file($file));
+        }
+        $this->dumpFile($rootFile);
+
+        // put the new files in production
+        foreach ($modifiedFiles as $file => $dummy) {
+            rename($file, $webDir.'/'.$file);
+        }
+        rename($rootFile, $webDir.'/'.basename($rootFile));
+
+        if ($force) {
+            // clear files that were not created in this build
+            foreach (glob($webDir.'/packages-*.json') as $file) {
+                if (!isset($modifiedFiles[basename($file)])) {
+                    unlink($file);
+                }
+            }
+        }
+
+        // update dump dates
+        $this->doctrine->getEntityManager()->flush();
+    }
+
+    private function loadFile($file)
+    {
+        $key = basename($file);
+
+        if (isset($this->files[$key])) {
+            return;
+        }
+
+        if (file_exists($file)) {
+            $this->files[$key] = json_decode(file_get_contents($file), true);
+        } else {
+            $this->files[$key] = array();
+        }
+    }
+
+    private function dumpFile($file)
+    {
+        $key = basename($file);
+
+        // sort all versions and packages to make sha1 consistent
+        ksort($this->files[$key]['packages']);
+        foreach ($this->files[$key]['packages'] as $package => $versions) {
+            ksort($this->files[$key]['packages'][$package]);
+        }
+
+        file_put_contents($file, json_encode($this->files[$key]));
+    }
+
+    private function dumpVersion(Version $version, $file)
+    {
+        $this->loadFile($file);
+        $this->files[basename($file)]['packages'][$version->getName()][$version->getVersion()] = $version->toArray();
+    }
+
+    private function getTargetFile(Version $version)
+    {
+        if ($version->isDevelopment()) {
+            $distribution = 16;
+            return 'packages-dev-' . chr(abs(crc32($version->getName())) % $distribution + 97) . '.json';
+        }
+
+        $date = $version->getReleasedAt();
+
+        return 'packages-' . ($date->format('Y') === date('Y') ? $date->format('Y-m') : $date->format('Y')) . '.json';
+    }
+}

src/Packagist/WebBundle/Resources/config/services.yml

@@ -4,3 +4,7 @@ services:
         arguments: [ @doctrine ]
         tags:
             - { name: twig.extension }
+
+    packagist.package_dumper:
+        class: Packagist\WebBundle\Package\Dumper
+        arguments: [ @doctrine, @filesystem, %kernel.root_dir%/../web/, %kernel.cache_dir% ]