Migrate download history to MySQL

src/Packagist/Redis/DownloadsIncr.php

@@ -30,7 +30,7 @@ local successful = 0;
 for i, key in ipairs(KEYS) do
     if i <= 3 then
         -- nothing
-    elseif ((i - 4) % 7) == 0 then
+    elseif ((i - 4) % 4) == 0 then
         local requests = tonumber(redis.call("ZINCRBY", key, 1, ARGV[1]));
         if 1 == requests then
             redis.call("EXPIRE", key, 86400);

src/Packagist/WebBundle/Command/CompileStatsCommand.php

@@ -16,6 +16,7 @@ use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Packagist\WebBundle\Entity\Download;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -32,7 +33,6 @@ class CompileStatsCommand extends ContainerAwareCommand
         $this
             ->setName('packagist:stats:compile')
             ->setDefinition(array(
-                new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-build of all stats'),
             ))
             ->setDescription('Updates the redis stats indices')
         ;
@@ -44,37 +44,26 @@ class CompileStatsCommand extends ContainerAwareCommand
     protected function execute(InputInterface $input, OutputInterface $output)
     {
         $verbose = $input->getOption('verbose');
-        $force = $input->getOption('force');
 
         $doctrine = $this->getContainer()->get('doctrine');
+        $conn = $doctrine->getManager()->getConnection();
         $this->redis = $redis = $this->getContainer()->get('snc_redis.default');
 
-        $minMax = $doctrine->getManager()->getConnection()->fetchAssoc('SELECT MAX(id) maxId, MIN(id) minId FROM package');
+        // TODO delete this whole block mid-august 2018
+        $minMax = $conn->fetchAssoc('SELECT MAX(id) maxId, MIN(id) minId FROM package');
         if (!isset($minMax['minId'])) {
             return 0;
         }
 
         $ids = range($minMax['minId'], $minMax['maxId']);
-        $res = $doctrine->getManager()->getConnection()->fetchAssoc('SELECT MIN(createdAt) minDate FROM package');
+        $res = $conn->fetchAssoc('SELECT MIN(createdAt) minDate FROM package');
         $date = new \DateTime($res['minDate']);
         $date->modify('00:00:00');
         $yesterday = new \DateTime('yesterday 00:00:00');
 
-        if ($force) {
-            if ($verbose) {
-                $output->writeln('Clearing aggregated DB');
-            }
-            $clearDate = clone $date;
-            $keys = array();
-            while ($clearDate <= $yesterday) {
-                $keys['downloads:'.$clearDate->format('Ymd')] = true;
-                $keys['downloads:'.$clearDate->format('Ym')] = true;
-                $clearDate->modify('+1day');
-            }
-            $redis->del(array_keys($keys));
-        }
-
-        while ($date <= $yesterday) {
+        // after this date no need to compute anymore
+        $cutoffDate = new \DateTime('2018-07-31 23:59:59');
+        while ($date <= $yesterday && $date <= $cutoffDate) {
             // skip months already computed
             if (null !== $this->getMonthly($date) && $date->format('m') !== $yesterday->format('m')) {
                 $date->setDate($date->format('Y'), $date->format('m')+1, 1);
@@ -108,13 +97,14 @@ class CompileStatsCommand extends ContainerAwareCommand
 
             $date = $nextDay;
         }
+        // TODO end delete here
 
         // fetch existing ids
         $doctrine = $this->getContainer()->get('doctrine');
-        $packages = $doctrine->getManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
+        $packages = $conn->fetchAll('SELECT id FROM package ORDER BY id ASC');
         $ids = array();
         foreach ($packages as $row) {
-            $ids[] = $row['id'];
+            $ids[] = (int) $row['id'];
         }
 
         if ($verbose) {
@@ -122,29 +112,40 @@ class CompileStatsCommand extends ContainerAwareCommand
         }
 
         while ($id = array_shift($ids)) {
-            $trendiness = $this->sumLastNDays(7, $id, $yesterday);
+            $total = (int) $redis->get('dl:'.$id);
+            if ($total > 10) {
+                $trendiness = $this->sumLastNDays(7, $id, $yesterday, $conn);
+            } else {
+                $trendiness = 0;
+            }
 
             $redis->zadd('downloads:trending:new', $trendiness, $id);
-            $redis->zadd('downloads:absolute:new', $redis->get('dl:'.$id), $id);
+            $redis->zadd('downloads:absolute:new', $total, $id);
         }
 
         $redis->rename('downloads:trending:new', 'downloads:trending');
         $redis->rename('downloads:absolute:new', 'downloads:absolute');
     }
 
-    // TODO could probably run faster with lua scripting
-    protected function sumLastNDays($days, $id, \DateTime $yesterday)
+    protected function sumLastNDays($days, $id, \DateTime $yesterday, $conn)
     {
         $date = clone $yesterday;
-        $keys = array();
+        $row = $conn->fetchAssoc('SELECT data FROM download WHERE id = :id AND type = :type', ['id' => $id, 'type' => Download::TYPE_PACKAGE]);
+        if (!$row) {
+            return 0;
+        }
+
+        $data = json_decode($row['data'], true);
+        $sum = 0;
         for ($i = 0; $i < $days; $i++) {
-            $keys[] = 'dl:'.$id.':'.$date->format('Ymd');
+            $sum += $data[$date->format('Ymd')] ?? 0;
             $date->modify('-1day');
         }
 
-        return array_sum($this->redis->mget($keys));
+        return $sum;
     }
 
+    // TODO delete all below as well once july data is computed
     protected function sum($date, array $ids)
     {
         $sum = 0;

src/Packagist/WebBundle/Command/MigrateDownloadCountsCommand.php

@@ -0,0 +1,60 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Command;
+
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Filesystem\LockHandler;
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Seld\Signal\SignalHandler;
+use Packagist\WebBundle\Entity\Package;
+
+class MigrateDownloadCountsCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('packagist:migrate-download-counts')
+            ->setDescription('Migrates download counts from redis to mysql')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $locker = $this->getContainer()->get('locker');
+        $logger = $this->getContainer()->get('logger');
+
+        // another migrate command is still active
+        $lockAcquired = $locker->lockCommand($this->getName());
+        if (!$lockAcquired) {
+            if ($input->getOption('verbose')) {
+                $output->writeln('Aborting, another task is running already');
+            }
+            return;
+        }
+
+        $signal = SignalHandler::create(null, $logger);
+        $downloadManager = $this->getContainer()->get('packagist.download_manager');
+        $doctrine = $this->getContainer()->get('doctrine');
+        $packageRepo = $doctrine->getRepository(Package::class);
+
+        try {
+            $packagesToProcess = $packageRepo->iterateStaleDownloadCountPackageIds();
+            foreach ($packagesToProcess as $packageDetails) {
+                $packageId = $packageDetails['id'];
+                $logger->debug('Processing package #'.$packageId);
+                $package = $packageRepo->findOneById($packageId);
+                $downloadManager->transferDownloadsToDb($package, $packageDetails['lastUpdated']);
+
+                $doctrine->getManager()->clear();
+
+                if ($signal->isTriggered()) {
+                    break;
+                }
+            }
+        } finally {
+            $locker->unlockCommand($this->getName());
+        }
+    }
+}

src/Packagist/WebBundle/Controller/PackageController.php

@@ -12,6 +12,7 @@ use Composer\Repository\VcsRepository;
 use Doctrine\ORM\NoResultException;
 use Packagist\WebBundle\Entity\PackageRepository;
 use Packagist\WebBundle\Entity\VersionRepository;
+use Packagist\WebBundle\Entity\Download;
 use Packagist\WebBundle\Form\Model\MaintainerRequest;
 use Packagist\WebBundle\Form\Type\AbandonedType;
 use Packagist\WebBundle\Entity\Package;
@@ -944,36 +945,28 @@ class PackageController extends Controller
         }
         $average = $req->query->get('average', $this->guessStatsAverage($from, $to));
 
+        if ($version) {
+            $downloads = $this->getDoctrine()->getRepository('PackagistWebBundle:Download')->findOneBy(['id' => $version->getId(), 'type' => Download::TYPE_VERSION]);
+        } else {
+            $downloads = $this->getDoctrine()->getRepository('PackagistWebBundle:Download')->findOneBy(['id' => $package->getId(), 'type' => Download::TYPE_PACKAGE]);
+        }
+
         $datePoints = $this->createDatePoints($from, $to, $average, $package, $version);
+        $dlData = $downloads ? $downloads->getData() : [];
 
-        $redis = $this->get('snc_redis.default');
-        if ($average === 'daily') {
-            $datePoints = array_map(function ($vals) {
-                return $vals[0];
-            }, $datePoints);
-
-            if (count($datePoints)) {
-                $datePoints = array(
-                    'labels' => array_keys($datePoints),
-                    'values' => array_map(function ($val) {
-                        return (int) $val;
-                    }, $redis->mget(array_values($datePoints))),
-                );
-            } else {
-                $datePoints = [
-                    'labels' => [],
-                    'values' => [],
-                ];
+        foreach ($datePoints as $label => $values) {
+            $value = 0;
+            foreach ($values as $valueKey) {
+                $value += $dlData[$valueKey] ?? 0;
             }
-        } else {
-            $datePoints = array(
-                'labels' => array_keys($datePoints),
-                'values' => array_values(array_map(function ($vals) use ($redis) {
-                    return array_sum($redis->mget(array_values($vals)));
-                }, $datePoints))
-            );
+            $datePoints[$label] = $value;
         }
 
+        $datePoints = array(
+            'labels' => array_keys($datePoints),
+            'values' => array_values($datePoints),
+        );
+
         $datePoints['average'] = $average;
 
         if ($average !== 'daily') {
@@ -1080,11 +1073,12 @@ class PackageController extends Controller
     {
         $interval = $this->getStatsInterval($average);
 
-        $dateKey = $average === 'monthly' ? 'Ym' : 'Ymd';
+        $dateKey = 'Ymd';
         $dateFormat = $average === 'monthly' ? 'Y-m' : 'Y-m-d';
-        $dateJump = $average === 'monthly' ? '+1month' : '+1day';
+        $dateJump = '+1day';
         if ($average === 'monthly') {
-            $to = new DateTimeImmutable('last day of '.$to->format('Y-m'));
+            $from = new DateTimeImmutable('first day of ' . $from->format('Y-m'));
+            $to = new DateTimeImmutable('last day of ' . $to->format('Y-m'));
         }
 
         $nextDataPointLabel = $from->format($dateFormat);
@@ -1092,7 +1086,7 @@ class PackageController extends Controller
 
         $datePoints = [];
         while ($from <= $to) {
-            $datePoints[$nextDataPointLabel][] = 'dl:'.$package->getId().($version ? '-' . $version->getId() : '').':'.$from->format($dateKey);
+            $datePoints[$nextDataPointLabel][] = $from->format($dateKey);
 
             $from = $from->modify($dateJump);
             if ($from >= $nextDataPoint) {

src/Packagist/WebBundle/Entity/Download.php

@@ -0,0 +1,126 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Entity;
+
+use Doctrine\ORM\Mapping as ORM;
+use DateTimeInterface;
+
+/**
+ * @ORM\Entity(repositoryClass="Packagist\WebBundle\Entity\DownloadRepository")
+ * @ORM\Table(
+ *     name="download",
+ *     indexes={
+ *         @ORM\Index(name="last_updated_idx",columns={"lastUpdated"}),
+ *         @ORM\Index(name="total_idx",columns={"total"}),
+ *         @ORM\Index(name="package_idx",columns={"package_id"})
+ *     }
+ * )
+ */
+class Download
+{
+    const TYPE_PACKAGE = 1;
+    const TYPE_VERSION = 2;
+
+    /**
+     * @ORM\Id
+     * @ORM\Column(type="bigint")
+     */
+    public $id;
+
+    /**
+     * @ORM\Id
+     * @ORM\Column(type="smallint")
+     */
+    public $type;
+
+    /**
+     * @ORM\Column(type="json_array")
+     */
+    public $data;
+
+    /**
+     * @ORM\Column(type="integer")
+     */
+    public $total;
+
+    /**
+     * @ORM\Column(type="datetime")
+     */
+    public $lastUpdated;
+
+    /**
+     * @ORM\ManyToOne(targetEntity="Packagist\WebBundle\Entity\Package", inversedBy="downloads")
+     */
+    public $package;
+
+    public function __construct()
+    {
+        $this->data = [];
+        $this->total = 0;
+    }
+
+    public function computeSum()
+    {
+        $this->total = array_sum($this->data);
+    }
+
+    public function setId(int $id)
+    {
+        $this->id = $id;
+    }
+
+    public function getId(): int
+    {
+        return $this->id;
+    }
+
+    public function setType(int $type)
+    {
+        $this->type = $type;
+    }
+
+    public function getType(): int
+    {
+        return $this->type;
+    }
+
+    public function setData(array $data)
+    {
+        $this->data = $data;
+    }
+
+    public function setDataPoint($key, $value)
+    {
+        $this->data[$key] = $value;
+    }
+
+    public function getData(): array
+    {
+        return $this->data;
+    }
+
+    public function getTotal(): int
+    {
+        return $this->total;
+    }
+
+    public function setLastUpdated(DateTimeInterface $lastUpdated)
+    {
+        $this->lastUpdated = $lastUpdated;
+    }
+
+    public function getLastUpdated(): DateTimeInterface
+    {
+        return $this->lastUpdated;
+    }
+
+    public function setPackage(Package $package)
+    {
+        $this->package = $package;
+    }
+
+    public function getPackage(): Package
+    {
+        return $this->package;
+    }
+}

src/Packagist/WebBundle/Entity/DownloadRepository.php

@@ -0,0 +1,15 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Entity;
+
+use Doctrine\ORM\EntityRepository;
+
+class DownloadRepository extends EntityRepository
+{
+    public function deletePackageDownloads(Package $package)
+    {
+        $conn = $this->getEntityManager()->getConnection();
+
+        $conn->executeUpdate('DELETE FROM download WHERE package_id = :id', ['package_id' => $package->getId()]);
+    }
+}

src/Packagist/WebBundle/Entity/Package.php

@@ -137,6 +137,11 @@ class Package
      */
     private $dumpedAt;
 
+    /**
+     * @ORM\OneToMany(targetEntity="Packagist\WebBundle\Entity\Download", mappedBy="package")
+     */
+    private $downloads;
+
     /**
      * @ORM\Column(type="boolean")
      */

src/Packagist/WebBundle/Entity/PackageRepository.php

@@ -159,6 +159,22 @@ class PackageRepository extends EntityRepository
         return $conn->fetchAll('SELECT p.id FROM package p WHERE p.dumpedAt IS NULL OR p.dumpedAt <= p.crawledAt AND p.crawledAt < NOW() ORDER BY p.id ASC');
     }
 
+    public function iterateStaleDownloadCountPackageIds()
+    {
+        $qb = $this->createQueryBuilder('p');
+        $res = $qb
+            ->select('p.id, d.lastUpdated, p.createdAt')
+            ->leftJoin('p.downloads', 'd')
+            ->where('((d.type = :type AND d.lastUpdated < :time) OR d.lastUpdated IS NULL)')
+            ->setParameters(['type' => Download::TYPE_PACKAGE, 'time' => new \DateTime('-20hours')])
+            ->getQuery()
+            ->getResult();
+
+        foreach ($res as $row) {
+            yield ['id' => $row['id'], 'lastUpdated' => is_null($row['lastUpdated']) ? new \DateTimeImmutable($row['createdAt']->format('r')) : new \DateTimeImmutable($row['lastUpdated']->format('r'))];
+        }
+    }
+
     public function getPartialPackageByNameWithVersions($name)
     {
         // first fetch a partial package including joined versions/maintainers, that way

src/Packagist/WebBundle/Entity/VersionRepository.php

@@ -52,6 +52,7 @@ class VersionRepository extends EntityRepository
         $em->getConnection()->executeQuery('DELETE FROM link_provide WHERE version_id=:id', array('id' => $version->getId()));
         $em->getConnection()->executeQuery('DELETE FROM link_require_dev WHERE version_id=:id', array('id' => $version->getId()));
         $em->getConnection()->executeQuery('DELETE FROM link_require WHERE version_id=:id', array('id' => $version->getId()));
+        $em->getConnection()->executeQuery('DELETE FROM download WHERE id=:id AND type = :type', ['id' => $version->getId(), 'type' => Download::TYPE_VERSION]);
 
         $em->remove($version);
     }

src/Packagist/WebBundle/Model/DownloadManager.php

@@ -12,9 +12,12 @@
 
 namespace Packagist\WebBundle\Model;
 
+use Doctrine\Common\Persistence\ManagerRegistry;
 use Packagist\WebBundle\Entity\Package;
 use Packagist\WebBundle\Entity\Version;
+use Packagist\WebBundle\Entity\Download;
 use Predis\Client;
+use DateTimeImmutable;
 
 /**
  * Manages the download counts for packages.
@@ -22,11 +25,13 @@ use Predis\Client;
 class DownloadManager
 {
     protected $redis;
+    protected $doctrine;
     protected $redisCommandLoaded = false;
 
-    public function __construct(Client $redis)
+    public function __construct(Client $redis, ManagerRegistry $doctrine)
     {
         $this->redis = $redis;
+        $this->doctrine = $doctrine;
     }
 
     /**
@@ -46,25 +51,47 @@ class DownloadManager
             $version = $version->getId();
         }
 
+        $type = Download::TYPE_PACKAGE;
+        $id = $package;
+        $keyBase = 'dl:'.$package;
+
         if ($version !== null) {
-            $version = '-'.$version;
+            $id = $version;
+            $type = Download::TYPE_PACKAGE;
+            $keyBase .= '-'.$version;
         }
 
+        $record = $this->doctrine->getRepository(Download::class)->findOneBy(['id' => $id, 'type' => $type]);
+        $dlData = $record ? $record->data : [];
+
+        $keyBase .= ':';
         $date = new \DateTime();
-        $keys = array('dl:'.$package . $version);
+        $todayDate = $date->format('Ymd');
+        $yesterdayDate = date('Ymd', $date->format('U') - 86400);
+
+        // fetch today, yesterday and the latest total from redis
+        $redisData = $this->redis->mget([$keyBase.$todayDate, $keyBase.$yesterdayDate, 'dl:'.$package]);
+        $monthly = 0;
         for ($i = 0; $i < 30; $i++) {
-            $keys[] = 'dl:' . $package . $version . ':' . $date->format('Ymd');
+            // current day and previous day might not be in db yet or incomplete, so we take the data from redis if there is still data there
+            if ($i <= 1) {
+                $monthly += $redisData[$i] ?? $dlData[$date->format('Ymd')] ?? 0;
+            } else {
+                $monthly += $dlData[$date->format('Ymd')] ?? 0;
+            }
             $date->modify('-1 day');
         }
 
-        $vals = $this->redis->mget($keys);
-        $result = array(
-            'total' => (int) array_shift($vals) ?: 0,
-            'monthly' => (int) array_sum($vals) ?: 0,
-            'daily' => (int) $vals[0] ?: 0,
-        );
+        $total = $redisData[2];
 
-        return $result;
+        // how much of yesterday to add to today to make it a whole day (sort of..)
+        $dayRatio = (2400 - (int) date('Hi')) / 2400;
+
+        return [
+            'total' => $total,
+            'monthly' => $monthly,
+            'daily' => round(($redisData[0] ?? $dlData[$todayDate] ?? 0) + (($redisData[1] ?? $dlData[$yesterdayDate] ?? 0) * $dayRatio)),
+        ];
     }
 
     /**
@@ -131,10 +158,7 @@ class DownloadManager
             $args[] = 'throttle:'.$package.':'.$day;
             // stats keys
             $args[] = 'dl:'.$package;
-            $args[] = 'dl:'.$package.':'.$month;
             $args[] = 'dl:'.$package.':'.$day;
-            $args[] = 'dl:'.$package.'-'.$version;
-            $args[] = 'dl:'.$package.'-'.$version.':'.$month;
             $args[] = 'dl:'.$package.'-'.$version.':'.$day;
         }
 
@@ -142,4 +166,142 @@ class DownloadManager
 
         $this->redis->downloadsIncr(...$args);
     }
+
+    public function transferDownloadsToDb(Package $package, DateTimeImmutable $lastUpdated)
+    {
+        // might be a large dataset coming through here especially on first run due to historical data
+        ini_set('memory_limit', '1G');
+
+        $packageId = $package->getId();
+        $rows = $this->doctrine->getManager()->getConnection()->fetchAll('SELECT id FROM package_version WHERE package_id = :id', ['id' => $packageId]);
+        $versionIds = [];
+        foreach ($rows as $row) {
+            $versionIds[] = $row['id'];
+        }
+
+        $now = new DateTimeImmutable();
+        $keys = [];
+        $firstIteration = true;
+        while ($lastUpdated < $now) {
+            // TODO delete once the redis db has been purged
+            if ($firstIteration || $lastUpdated->format('d') === '01') {
+                $firstIteration = false;
+                // dl:$package:Ym
+                $keys[] = 'dl:'.$packageId.':'.$lastUpdated->format('Ym');
+                foreach ($versionIds as $id) {
+                    // dl:$package-$version and dl:$package-$version:Ym
+                    $keys[] = 'dl:'.$packageId.'-'.$id;
+                    $keys[] = 'dl:'.$packageId.'-'.$id.':'.$lastUpdated->format('Ym');
+                }
+            }
+
+            // dl:$package:Ymd
+            $keys[] = 'dl:'.$packageId.':'.$lastUpdated->format('Ymd');
+            foreach ($versionIds as $id) {
+                // dl:$package-$version:Ymd
+                $keys[] = 'dl:'.$packageId.'-'.$id.':'.$lastUpdated->format('Ymd');
+            }
+
+            $lastUpdated = $lastUpdated->modify('+1day');
+        }
+
+        sort($keys);
+
+        $buffer = [];
+        $toDelete = [];
+        $lastPrefix = '';
+
+        foreach ($keys as $key) {
+            // ignore IP keys temporarily until they all switch to throttle:* prefix
+            if (preg_match('{^dl:\d+:(\d+\.|[0-9a-f]+:[0-9a-f]+:)}', $key)) {
+                continue;
+            }
+
+            // delete version totals when we find one
+            if (preg_match('{^dl:\d+-\d+$}', $key)) {
+                $toDelete[] = $key;
+                continue;
+            }
+
+            $prefix = preg_replace('{:\d+$}', ':', $key);
+
+            if ($lastPrefix && $prefix !== $lastPrefix && $buffer) {
+                $toDelete = $this->createDbRecordsForKeys($package, $buffer, $toDelete, $now);
+                $buffer = [];
+            }
+
+            $buffer[] = $key;
+            $lastPrefix = $prefix;
+        }
+
+        if ($buffer) {
+            $toDelete = $this->createDbRecordsForKeys($package, $buffer, $toDelete, $now);
+        }
+
+        $this->doctrine->getManager()->flush();
+
+        while ($toDelete) {
+            $batch = array_splice($toDelete, 0, 1000);
+            $this->redis->del($batch);
+        }
+    }
+
+    private function createDbRecordsForKeys(Package $package, array $keys, array $toDelete, DateTimeImmutable $now): array
+    {
+        list($id, $type) = $this->getKeyInfo($keys[0]);
+        $record = $this->doctrine->getRepository(Download::class)->findOneBy(['id' => $id, 'type' => $type]);
+        $isNewRecord = false;
+        if (!$record) {
+            $record = new Download();
+            $record->setId($id);
+            $record->setType($type);
+            $record->setPackage($package);
+            $isNewRecord = true;
+        }
+
+        $today = date('Ymd');
+        $record->setLastUpdated($now);
+
+        $values = $this->redis->mget($keys);
+        foreach ($keys as $index => $key) {
+            $date = preg_replace('{^.*?:(\d+)$}', '$1', $key);
+
+            // monthly data point, discard
+            if (strlen($date) === 6) {
+                $toDelete[] = $key;
+                continue;
+            }
+
+            $val = (int) $values[$index];
+            if ($val) {
+                $record->setDataPoint($date, $val);
+            }
+            // today's value is not deleted yet as it might not be complete and we want to update it when its complete
+            if ($date !== $today) {
+                $toDelete[] = $key;
+            }
+        }
+
+        // only store records for packages or for versions that have had downloads to avoid storing empty records
+        if ($isNewRecord && ($type === Download::TYPE_PACKAGE || count($record->getData()) > 0)) {
+            $this->doctrine->getManager()->persist($record);
+        }
+
+        $record->computeSum();
+
+        return $toDelete;
+    }
+
+    private function getKeyInfo(string $key): array
+    {
+        if (preg_match('{^dl:(\d+):}', $key, $match)) {
+            return [(int) $match[1], Download::TYPE_PACKAGE];
+        }
+
+        if (preg_match('{^dl:\d+-(\d+):}', $key, $match)) {
+            return [(int) $match[1], Download::TYPE_VERSION];
+        }
+
+        throw new \LogicException('Invalid key given: '.$key);
+    }
 }

src/Packagist/WebBundle/Model/PackageManager.php

@@ -52,6 +52,9 @@ class PackageManager
             $versionRepo->remove($version);
         }
 
+        $downloadRepo = $this->doctrine->getRepository('PackagistWebBundle:Download');
+        $downloadRepo->deletePackageDownloads($package);
+
         $this->providerManager->deletePackage($package);
         $packageName = $package->getName();
 

src/Packagist/WebBundle/Resources/config/services.yml

@@ -86,6 +86,7 @@ services:
         class: Packagist\WebBundle\Model\DownloadManager
         arguments:
             - '@snc_redis.default_client'
+            - '@doctrine'
 
     packagist.provider_manager:
         class: Packagist\WebBundle\Model\ProviderManager

src/Packagist/WebBundle/Resources/views/Package/stats.html.twig

@@ -6,6 +6,18 @@
 
 {% block title %}{{ 'stats.title'|trans }} - {{ package.name }} - {{ parent() }}{% endblock %}
 
+{% block content_header %}
+    <section class="wrapper wrapper-white">
+        <div class="container flash-container">
+            <div class="alert alert-error">
+                <p>We are currently doing some background processing of download statistics, they might be unavailable on some packages for the next few days. Thanks for your patience.</p>
+            </div>
+        </div>
+    </section>
+
+    {{ parent() }}
+{% endblock %}
+
 {% block content %}
     <div class="row">
         <div class="col-xs-12 package">