Dump packages 50 at a time to avoid memory issues

src/Packagist/WebBundle/Command/DumpPackagesCommand.php

@@ -42,26 +42,35 @@ class DumpPackagesCommand extends ContainerAwareCommand
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $force = $input->getOption('force');
+        $force = (Boolean) $input->getOption('force');
+        $verbose = (Boolean) $input->getOption('verbose');
 
         $doctrine = $this->getContainer()->get('doctrine');
 
         if ($force) {
-            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getFullPackages();
+            $packages = $doctrine->getEntityManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
         } else {
             $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackagesForDumping();
         }
 
+        $ids = array();
+        foreach ($packages as $package) {
+            $ids[] = $package['id'];
+        }
+
         $lock = $this->getContainer()->getParameter('kernel.cache_dir').'/composer-dumper.lock';
         $timeout = 600;
 
         // another dumper is still active
         if (file_exists($lock) && filemtime($lock) > time() - $timeout) {
+            if ($verbose) {
+                $output->writeln('Aborting, '.$lock.' file present');
+            }
             return;
         }
 
         touch($lock);
-        $this->getContainer()->get('packagist.package_dumper')->dump($packages);
+        $this->getContainer()->get('packagist.package_dumper')->dump($ids, $force, $verbose);
         unlink($lock);
     }
 }

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -100,21 +100,9 @@ class PackageRepository extends EntityRepository
 
     public function getStalePackagesForDumping()
     {
-        $qb = $this->getEntityManager()->createQueryBuilder();
-        $qb->select('p', 'v', 't', 'a', 'req', 'devReq', 'sug', 'rep', 'con', 'pro')
-            ->from('Packagist\WebBundle\Entity\Package', 'p')
-            ->leftJoin('p.versions', 'v')
-            ->leftJoin('v.tags', 't')
-            ->leftJoin('v.authors', 'a')
-            ->leftJoin('v.require', 'req')
-            ->leftJoin('v.devRequire', 'devReq')
-            ->leftJoin('v.suggest', 'sug')
-            ->leftJoin('v.replace', 'rep')
-            ->leftJoin('v.conflict', 'con')
-            ->leftJoin('v.provide', 'pro')
-            ->where('p.dumpedAt IS NULL OR p.dumpedAt < p.crawledAt');
+        $conn = $this->getEntityManager()->getConnection();
 
-        return $qb->getQuery()->getResult();
+        return $conn->fetchAll('SELECT p.id FROM package p WHERE p.dumpedAt IS NULL OR p.dumpedAt < p.crawledAt  ORDER BY p.id ASC');
     }
 
     public function findOneByName($name)
@@ -141,7 +129,7 @@ class PackageRepository extends EntityRepository
         return $qb->getQuery()->getSingleResult();
     }
 
-    public function getFullPackages()
+    public function getFullPackages(array $ids = null)
     {
         $qb = $this->getEntityManager()->createQueryBuilder();
         $qb->select('p', 'v', 't', 'a', 'req', 'devReq', 'sug', 'rep', 'con', 'pro')
@@ -158,6 +146,10 @@ class PackageRepository extends EntityRepository
             ->orderBy('v.development', 'DESC')
             ->addOrderBy('v.releasedAt', 'DESC');
 
+        if (null !== $ids) {
+            $qb->where($qb->expr()->in('p.id', $ids));
+        }
+
         return $qb->getQuery()->getResult();
     }
 

src/Packagist/WebBundle/Package/Dumper.php

@@ -73,10 +73,10 @@ class Dumper
     /**
      * Dump a set of packages to the web root
      *
-     * @param array $packages
+     * @param array $packageIds
      * @param Boolean $force
      */
-    public function dump(array $packages, $force = false)
+    public function dump(array $packageIds, $force = false, $verbose = false)
     {
         // prepare build dir
         $webDir = $this->webDir;
@@ -91,26 +91,41 @@ class Dumper
 
         $modifiedFiles = array();
 
-        // prepare packages in memory
-        foreach ($packages as $package) {
-            // clean up all versions of that package
-            foreach (glob($buildDir.'/packages*.json') as $file) {
-                $key = basename($file);
-                $this->loadFile($file);
-                if (isset($this->files[$key]['packages'][$package->getName()])) {
-                    unset($this->files[$key]['packages'][$package->getName()]);
-                    $modifiedFiles[$key] = true;
-                }
+        while ($packageIds) {
+            $packages = $this->doctrine->getRepository('PackagistWebBundle:Package')->getFullPackages(array_splice($packageIds, 0, 50));
+
+            if ($verbose) {
+                echo 'Processing '.count($packages).' packages...'.PHP_EOL;
             }
 
-            // (re)write versions
-            foreach ($package->getVersions() as $version) {
-                $file = $buildDir.'/'.$this->getTargetFile($version);
-                $modifiedFiles[basename($file)] = true;
-                $this->dumpVersion($version, $file);
+            // prepare packages in memory
+            foreach ($packages as $package) {
+
+                // clean up all versions of that package
+                foreach (glob($buildDir.'/packages*.json') as $file) {
+                    $key = basename($file);
+                    $this->loadFile($file);
+                    if (isset($this->files[$key]['packages'][$package->getName()])) {
+                        unset($this->files[$key]['packages'][$package->getName()]);
+                        $modifiedFiles[$key] = true;
+                    }
+                }
+
+                // (re)write versions
+                foreach ($package->getVersions() as $version) {
+                    $file = $buildDir.'/'.$this->getTargetFile($version);
+                    $modifiedFiles[basename($file)] = true;
+                    $this->dumpVersion($version, $file);
+                }
+
+                $package->setDumpedAt(new \DateTime);
+
             }
 
-            $package->setDumpedAt(new \DateTime);
+            // update dump dates
+            $this->doctrine->getEntityManager()->flush();
+            $this->doctrine->getEntityManager()->clear();
+            unset($packages);
         }
 
         // prepare root file
@@ -143,9 +158,6 @@ class Dumper
                 }
             }
         }
-
-        // update dump dates
-        $this->doctrine->getEntityManager()->flush();
     }
 
     private function loadFile($file)