* Nils Adermann * * For the full copyright and license information, please view the LICENSE * file that was distributed with this source code. */ namespace Packagist\WebBundle\Command; use Packagist\WebBundle\Entity\Package; use Packagist\WebBundle\Model\DownloadManager; use Packagist\WebBundle\Model\FavoriteManager; use Solarium_Document_ReadWrite; use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; /** * @author Igor Wiedler */ class IndexPackagesCommand extends ContainerAwareCommand { /** * {@inheritdoc} */ protected function configure() { $this ->setName('packagist:index') ->setDefinition(array( new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-indexing of all packages'), new InputOption('all', null, InputOption::VALUE_NONE, 'Index all packages without clearing the index first'), new InputArgument('package', InputArgument::OPTIONAL, 'Package name to index'), )) ->setDescription('Indexes packages in Solr') ; } /** * {@inheritdoc} */ protected function execute(InputInterface $input, OutputInterface $output) { $verbose = $input->getOption('verbose'); $force = $input->getOption('force'); $indexAll = $input->getOption('all'); $package = $input->getArgument('package'); $deployLock = $this->getContainer()->getParameter('kernel.cache_dir').'/deploy.globallock'; if (file_exists($deployLock)) { if ($verbose) { $output->writeln('Aborting, '.$deployLock.' file present'); } return; } $doctrine = $this->getContainer()->get('doctrine'); $solarium = $this->getContainer()->get('solarium.client'); $redis = $this->getContainer()->get('snc_redis.default'); $downloadManager = $this->getContainer()->get('packagist.download_manager'); $favoriteManager = $this->getContainer()->get('packagist.favorite_manager'); $lock = $this->getContainer()->getParameter('kernel.cache_dir').'/composer-indexer.lock'; $timeout = 600; // another dumper is still active if (file_exists($lock) && filemtime($lock) > time() - $timeout) { if ($verbose) { $output->writeln('Aborting, '.$lock.' file present'); } return; } touch($lock); if ($package) { $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId())); } elseif ($force || $indexAll) { $packages = $doctrine->getManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC'); $doctrine->getManager()->getConnection()->executeQuery('UPDATE package SET indexedAt = NULL'); } else { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackagesForIndexing(); } $ids = array(); foreach ($packages as $row) { $ids[] = $row['id']; } // clear index before a full-update if ($force && !$package) { if ($verbose) { $output->writeln('Deleting existing index'); } $update = $solarium->createUpdate(); $update->addDeleteQuery('*:*'); $update->addCommit(); $solarium->update($update); } $total = count($ids); $current = 0; // update package index while ($ids) { $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($ids, 0, 50)); $update = $solarium->createUpdate(); foreach ($packages as $package) { $current++; if ($verbose) { $output->writeln('['.sprintf('%'.strlen($total).'d', $current).'/'.$total.'] Indexing '.$package->getName()); } try { $document = $update->createDocument(); $this->updateDocumentFromPackage($document, $package, $redis, $downloadManager, $favoriteManager); $update->addDocument($document); $package->setIndexedAt(new \DateTime); } catch (\Exception $e) { $output->writeln('Exception: '.$e->getMessage().', skipping package '.$package->getName().'.'); } foreach ($package->getVersions() as $version) { // abort when a non-dev version shows up since dev ones are ordered first if (!$version->isDevelopment()) { break; } if (count($provide = $version->getProvide())) { foreach ($version->getProvide() as $provide) { try { $document = $update->createDocument(); $document->setField('id', $provide->getPackageName()); $document->setField('name', $provide->getPackageName()); $document->setField('description', ''); $document->setField('type', 'virtual-package'); $document->setField('trendiness', 100); $document->setField('repository', ''); $document->setField('abandoned', 0); $document->setField('replacementPackage', ''); $update->addDocument($document); } catch (\Exception $e) { $output->writeln(''.get_class($e).': '.$e->getMessage().', skipping package '.$package->getName().':provide:'.$provide->getPackageName().''); } } } } } $doctrine->getManager()->flush(); $doctrine->getManager()->clear(); unset($packages); $update->addCommit(); $solarium->update($update); } unlink($lock); } private function updateDocumentFromPackage( Solarium_Document_ReadWrite $document, Package $package, $redis, DownloadManager $downloadManager, FavoriteManager $favoriteManager ) { $document->setField('id', $package->getId()); $document->setField('name', $package->getName()); $document->setField('description', $package->getDescription()); $document->setField('type', $package->getType()); $document->setField('trendiness', $redis->zscore('downloads:trending', $package->getId())); $document->setField('downloads', $downloadManager->getTotalDownloads($package)); $document->setField('favers', $favoriteManager->getFaverCount($package)); $document->setField('repository', $package->getRepository()); if ($package->isAbandoned()) { $document->setField('abandoned', 1); $document->setField('replacementPackage', $package->getReplacementPackage() ?: ''); } else { $document->setField('abandoned', 0); $document->setField('replacementPackage', ''); } $tags = array(); foreach ($package->getVersions() as $version) { foreach ($version->getTags() as $tag) { $tags[mb_strtolower($tag->getName(), 'UTF-8')] = true; } } $document->setField('tags', array_keys($tags)); } }