Make search take downloads into account

doc/schema.xml

@@ -236,6 +236,7 @@
    <field name="description" type="text_general_rev" indexed="true" stored="true"/>
    <field name="tags" type="text_general_rev" indexed="true" stored="true" multiValued="true"/>
    <field name="type" type="text_general_rev" indexed="true" stored="true"/>
+   <field name="trendiness" type="float" indexed="true" stored="true" />
 
    <!-- catchall field, containing all other searchable text fields (implemented
         via copyField further on in this schema  -->

src/Packagist/WebBundle/Command/CompileStatsCommand.php

@@ -117,6 +117,43 @@ class CompileStatsCommand extends ContainerAwareCommand
 
             $date = $nextDay;
         }
+
+        // fetch existing ids
+        $doctrine = $this->getContainer()->get('doctrine');
+        $packages = $doctrine->getEntityManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
+        $ids = array();
+        foreach ($packages as $row) {
+            $ids[] = $row['id'];
+        }
+
+        // add downloads from the last 5 days to the solr index
+        $solarium = $this->getContainer()->get('solarium.client');
+
+        if ($verbose) {
+            $output->writeln('Writing new trendiness data into redis');
+        }
+
+        while ($id = array_shift($ids)) {
+            $trendiness = $this->sumLastNDays(5, $id, $yesterday);
+
+            $redis->zadd('downloads:trending:new', $trendiness, $id);
+            $redis->zadd('downloads:absolute:new', $redis->get('dl:'.$id), $id);
+        }
+
+        $redis->rename('downloads:trending:new', 'downloads:trending');
+        $redis->rename('downloads:absolute:new', 'downloads:absolute');
+    }
+
+    protected function sumLastNDays($days, $id, \DateTime $yesterday)
+    {
+        $date = clone $yesterday;
+        $keys = array();
+        for ($i = 0; $i < $days; $i++) {
+            $keys[] = 'dl:'.$id.':'.$date->format('Ymd');
+            $date->modify('-1day');
+        }
+
+        return array_sum($this->redis->mget($keys));
     }
 
     protected function sum($date, array $ids)

src/Packagist/WebBundle/Command/IndexPackagesCommand.php

@@ -34,6 +34,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
             ->setName('packagist:index')
             ->setDefinition(array(
                 new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-indexing of all packages'),
+                new InputOption('all', null, InputOption::VALUE_NONE, 'Index all packages without clearing the index first'),
                 new InputArgument('package', InputArgument::OPTIONAL, 'Package name to index'),
             ))
             ->setDescription('Indexes packages in Solr')
@@ -47,6 +48,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
     {
         $verbose = $input->getOption('verbose');
         $force = $input->getOption('force');
+        $indexAll = $input->getOption('all');
         $package = $input->getArgument('package');
 
         $deployLock = $this->getContainer()->getParameter('kernel.cache_dir').'/deploy.globallock';
@@ -59,6 +61,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
 
         $doctrine = $this->getContainer()->get('doctrine');
         $solarium = $this->getContainer()->get('solarium.client');
+        $redis = $this->getContainer()->get('snc_redis.default');
 
         $lock = $this->getContainer()->getParameter('kernel.cache_dir').'/composer-indexer.lock';
         $timeout = 600;
@@ -75,7 +78,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
 
         if ($package) {
             $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId()));
-        } elseif ($force) {
+        } elseif ($force || $indexAll) {
             $packages = $doctrine->getEntityManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
             $doctrine->getEntityManager()->getConnection()->executeQuery('UPDATE package SET indexedAt = NULL');
         } else {
@@ -116,7 +119,7 @@ class IndexPackagesCommand extends ContainerAwareCommand
 
                 try {
                     $document = $update->createDocument();
-                    $this->updateDocumentFromPackage($document, $package);
+                    $this->updateDocumentFromPackage($document, $package, $redis);
                     $update->addDocument($document);
 
                     $package->setIndexedAt(new \DateTime);
@@ -136,12 +139,13 @@ class IndexPackagesCommand extends ContainerAwareCommand
         unlink($lock);
     }
 
-    private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package)
+    private function updateDocumentFromPackage(\Solarium_Document_ReadWrite $document, Package $package, $redis)
     {
         $document->setField('id', $package->getId());
         $document->setField('name', $package->getName());
         $document->setField('description', $package->getDescription());
         $document->setField('type', $package->getType());
+        $document->setField('trendiness', $redis->zscore('downloads:trending', $package->getId()));
 
         $tags = array();
         foreach ($package->getVersions() as $version) {

src/Packagist/WebBundle/Controller/WebController.php

@@ -141,8 +141,9 @@ class WebController extends Controller
                     $select->addFilterQuery($filterQuery);
                 }
                 $dismax = $select->getDisMax();
-                $dismax->setQueryFields(array('name^2', 'description', 'tags', 'text', 'text_ngram', 'name_split^1.5'));
+                $dismax->setQueryFields(array('name^4', 'description', 'tags', 'text', 'text_ngram', 'name_split^2'));
                 $dismax->setPhraseFields(array('description'));
+                $dismax->setBoostFunctions(array('log(trendiness)^10'));
                 //this is very lenient, and may want to be refined
                 $dismax->setMinimumMatch(1);
                 $dismax->setQueryParser('edismax');