|
|
@@ -0,0 +1,147 @@
|
|
|
+<?php
|
|
|
+
|
|
|
+/*
|
|
|
+ * This file is part of Packagist.
|
|
|
+ *
|
|
|
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
|
|
|
+ * Nils Adermann <naderman@naderman.de>
|
|
|
+ *
|
|
|
+ * For the full copyright and license information, please view the LICENSE
|
|
|
+ * file that was distributed with this source code.
|
|
|
+ */
|
|
|
+
|
|
|
+namespace Packagist\WebBundle\Command;
|
|
|
+
|
|
|
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
|
|
|
+use Symfony\Component\Console\Input\InputInterface;
|
|
|
+use Symfony\Component\Console\Input\InputOption;
|
|
|
+use Symfony\Component\Console\Input\InputArgument;
|
|
|
+use Symfony\Component\Console\Output\OutputInterface;
|
|
|
+use Packagist\WebBundle\Package\Updater;
|
|
|
+use Composer\Repository\VcsRepository;
|
|
|
+use Composer\Factory;
|
|
|
+use Composer\Package\Loader\ValidatingArrayLoader;
|
|
|
+use Composer\Package\Loader\ArrayLoader;
|
|
|
+use Composer\IO\BufferIO;
|
|
|
+use Composer\IO\ConsoleIO;
|
|
|
+use Composer\Repository\InvalidRepositoryException;
|
|
|
+
|
|
|
+/**
|
|
|
+ * @author Jordi Boggiano <j.boggiano@seld.be>
|
|
|
+ */
|
|
|
+class CompileStatsCommand extends ContainerAwareCommand
|
|
|
+{
|
|
|
+ protected $redis;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * {@inheritdoc}
|
|
|
+ */
|
|
|
+ protected function configure()
|
|
|
+ {
|
|
|
+ $this
|
|
|
+ ->setName('packagist:stats:compile')
|
|
|
+ ->setDefinition(array(
|
|
|
+ new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-build of all stats'),
|
|
|
+ ))
|
|
|
+ ->setDescription('Updates the redis stats indices')
|
|
|
+ ;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * {@inheritdoc}
|
|
|
+ */
|
|
|
+ protected function execute(InputInterface $input, OutputInterface $output)
|
|
|
+ {
|
|
|
+ $verbose = $input->getOption('verbose');
|
|
|
+ $force = $input->getOption('force');
|
|
|
+
|
|
|
+ $doctrine = $this->getContainer()->get('doctrine');
|
|
|
+ $this->redis = $redis = $this->getContainer()->get('snc_redis.default');
|
|
|
+
|
|
|
+ $minMax = $doctrine->getEntityManager()->getConnection()->fetchAssoc('SELECT MAX(id) maxId, MIN(id) minId FROM package');
|
|
|
+ if (!isset($minMax['minId'])) {
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ $ids = range($minMax['minId'], $minMax['maxId']);
|
|
|
+ $res = $doctrine->getEntityManager()->getConnection()->fetchAssoc('SELECT MIN(createdAt) minDate FROM package');
|
|
|
+ $date = new \DateTime($res['minDate']);
|
|
|
+ $date->modify('00:00:00');
|
|
|
+ $yesterday = new \DateTime('yesterday 00:00:00');
|
|
|
+
|
|
|
+ if ($force) {
|
|
|
+ if ($verbose) {
|
|
|
+ $output->writeln('Clearing aggregated DB');
|
|
|
+ }
|
|
|
+ $clearDate = clone $date;
|
|
|
+ $keys = array();
|
|
|
+ while ($clearDate <= $yesterday) {
|
|
|
+ $keys['downloads:'.$clearDate->format('Ymd')] = true;
|
|
|
+ $keys['downloads:'.$clearDate->format('Ym')] = true;
|
|
|
+ $clearDate->modify('+1day');
|
|
|
+ }
|
|
|
+ $redis->del(array_keys($keys));
|
|
|
+ }
|
|
|
+
|
|
|
+ while ($date <= $yesterday) {
|
|
|
+ // skip months already computed
|
|
|
+ if (null !== $this->getMonthly($date) && $date->format('m') !== $yesterday->format('m')) {
|
|
|
+ $date->setDate($date->format('Y'), $date->format('m')+1, 1);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ // skip days already computed
|
|
|
+ if (null !== $this->getDaily($date) && $date != $yesterday) {
|
|
|
+ $date->modify('+1day');
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ $sum = $this->sum($date->format('Ymd'), $ids);
|
|
|
+ $redis->set('downloads:'.$date->format('Ymd'), $sum);
|
|
|
+
|
|
|
+ if ($verbose) {
|
|
|
+ $output->writeln('Wrote daily data for '.$date->format('Y-m-d').': '.$sum);
|
|
|
+ }
|
|
|
+
|
|
|
+ $nextDay = clone $date;
|
|
|
+ $nextDay->modify('+1day');
|
|
|
+ // update the monthly total if we just computed the last day of the month or the last known day
|
|
|
+ if ($date->format('Ymd') === $yesterday->format('Ymd') || $date->format('Ym') !== $nextDay->format('Ym')) {
|
|
|
+ $sum = $this->sum($date->format('Ym'), $ids);
|
|
|
+ $redis->set('downloads:'.$date->format('Ym'), $sum);
|
|
|
+
|
|
|
+ if ($verbose) {
|
|
|
+ $output->writeln('Wrote monthly data for '.$date->format('Y-m').': '.$sum);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ $date = $nextDay;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ protected function sum($date, array $ids)
|
|
|
+ {
|
|
|
+ $sum = 0;
|
|
|
+
|
|
|
+ while ($ids) {
|
|
|
+ $batch = array_splice($ids, 0, 500);
|
|
|
+ $keys = array();
|
|
|
+ foreach ($batch as $id) {
|
|
|
+ $keys[] = 'dl:'.$id.':'.$date;
|
|
|
+ }
|
|
|
+ $sum += array_sum($res = $this->redis->mget($keys));
|
|
|
+ }
|
|
|
+
|
|
|
+ return $sum;
|
|
|
+ }
|
|
|
+
|
|
|
+ protected function getMonthly(\DateTime $date)
|
|
|
+ {
|
|
|
+ return $this->redis->get('downloads:'.$date->format('Ym'));
|
|
|
+ }
|
|
|
+
|
|
|
+ protected function getDaily(\DateTime $date)
|
|
|
+ {
|
|
|
+ return $this->redis->get('downloads:'.$date->format('Ymd'));
|
|
|
+ }
|
|
|
+}
|
Jordi Boggiano