Make updater work in batches of 50 packages

src/Packagist/WebBundle/Command/UpdatePackagesCommand.php

@@ -57,15 +57,20 @@ class UpdatePackagesCommand extends ContainerAwareCommand
         $flags = 0;
 
         if ($package) {
-            $packages = array($doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package));
+            $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId()));
             $flags = Updater::UPDATE_TAGS;
         } elseif ($force) {
-            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getFullPackages();
+            $packages = $doctrine->getEntityManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
             $flags = Updater::UPDATE_TAGS;
         } else {
             $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackages();
         }
 
+        $ids = array();
+        foreach ($packages as $package) {
+            $ids[] = $package['id'];
+        }
+
         if ($input->getOption('delete-before')) {
             $flags = Updater::DELETE_BEFORE;
         }
@@ -76,16 +81,23 @@ class UpdatePackagesCommand extends ContainerAwareCommand
         $input->setInteractive(false);
         $io = $verbose ? new ConsoleIO($input, $output, $this->getApplication()->getHelperSet()) : new NullIO;
 
-        foreach ($packages as $package) {
-            if ($verbose) {
-                $output->writeln('Importing '.$package->getRepository());
-            }
-            try {
-                $repository = new VcsRepository(array('url' => $package->getRepository()), $io);
-                $updater->update($package, $repository, $flags, $start);
-            } catch (\Exception $e) {
-                $output->writeln('<error>Exception: '.$e->getMessage().' at '.$e->getFile().':'.$e->getLine().', skipping package '.$package->getName().'.</error>');
+        while ($ids) {
+            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getFullPackages(array_splice($ids, 0, 50));
+
+            foreach ($packages as $package) {
+                if ($verbose) {
+                    $output->writeln('Importing '.$package->getRepository());
+                }
+                try {
+                    $repository = new VcsRepository(array('url' => $package->getRepository()), $io);
+                    $updater->update($package, $repository, $flags, $start);
+                } catch (\Exception $e) {
+                    $output->writeln('<error>Exception: '.$e->getMessage().' at '.$e->getFile().':'.$e->getLine().', skipping package '.$package->getName().'.</error>');
+                }
             }
+
+            $doctrine->getEntityManager()->clear();
+            unset($packages);
         }
     }
 }

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -66,17 +66,19 @@ class PackageRepository extends EntityRepository
 
     public function getStalePackages()
     {
-        $qb = $this->getEntityManager()->createQueryBuilder();
-        $qb->select('p', 'v')
-            ->from('Packagist\WebBundle\Entity\Package', 'p')
-            ->leftJoin('p.versions', 'v')
-            ->where('p.crawledAt IS NULL')
-            ->orWhere('(p.autoUpdated = false AND p.crawledAt < :crawled)')
-            ->orWhere('(p.crawledAt < :autocrawled)')
-            ->setParameter('crawled', new \DateTime('-1hour')) // crawl packages by hand once an hour
-            ->setParameter('autocrawled', new \DateTime('-1week')); // crawl auto-updated packages just in case once a week
+        $conn = $this->getEntityManager()->getConnection();
 
-        return $qb->getQuery()->getResult();
+        return $conn->fetchAll(
+            'SELECT p.id FROM package p
+            WHERE p.crawledAt IS NULL
+            OR (p.autoUpdated = 0 AND p.crawledAt < :crawled)
+            OR (p.crawledAt < :autocrawled)
+            ORDER BY p.id ASC',
+            array(
+                'crawled' => date('Y-m-d H:i:s', strtotime('-4hours')),
+                'autocrawled' => date('Y-m-d H:i:s', strtotime('-1week')),
+            )
+        );
     }
 
     public function getStalePackagesForIndexing()