Merge pull request #231 from Seldaek/newrepo

[WIP] Add dumping of one json file per provider

.gitignore

@@ -1,5 +1,6 @@
 web/bundles/
 web/packages*.json
+web/p/
 app/config/parameters.yml
 app/bootstrap*
 app/cache/*

src/Packagist/WebBundle/Command/DumpPackagesCommand.php

@@ -61,6 +61,8 @@ class DumpPackagesCommand extends ContainerAwareCommand
         $lock = $this->getContainer()->getParameter('kernel.cache_dir').'/composer-dumper.lock';
         $timeout = 600;
 
+        ini_set('memory_limit', -1);
+
         // another dumper is still active
         if (file_exists($lock) && filemtime($lock) > time() - $timeout) {
             if ($verbose) {

src/Packagist/WebBundle/Entity/Version.php

@@ -307,6 +307,23 @@ class Version
         return $this->name;
     }
 
+    public function getNames()
+    {
+        $names = array(
+            strtolower($this->name) => true
+        );
+
+        foreach ($this->getReplace() as $link) {
+            $names[strtolower($link->getPackageName())] = true;
+        }
+
+        foreach ($this->getProvide() as $link) {
+            $names[strtolower($link->getPackageName())] = true;
+        }
+
+        return array_keys($names);
+    }
+
     /**
      * Set description
      *

src/Packagist/WebBundle/Package/Dumper.php

@@ -15,6 +15,7 @@ namespace Packagist\WebBundle\Package;
 use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Bridge\Doctrine\RegistryInterface;
 use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
+use Symfony\Component\Finder\Finder;
 use Packagist\WebBundle\Entity\Version;
 
 /**
@@ -86,30 +87,76 @@ class Dumper
         $buildDir = $this->buildDir;
         $this->fs->remove($buildDir);
         $this->fs->mkdir($buildDir);
+        $this->fs->mkdir($webDir.'/p/');
+
         if (!$force) {
             foreach (glob($webDir.'/packages*.json') as $file) {
                 copy($file, $buildDir.'/'.basename($file));
             }
+
+            $this->fs->mirror($webDir.'/p/', $buildDir.'/p/', null, array('override' => true));
         }
 
         $modifiedFiles = array();
+        $modifiedIndividualFiles = array();
 
+        $total = count($packageIds);
+        $current = 0;
+        $step = 50;
         while ($packageIds) {
-            $packages = $this->doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($packageIds, 0, 50));
+            $packages = $this->doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($packageIds, 0, $step));
 
             if ($verbose) {
-                echo 'Processing '.count($packages).' packages...'.PHP_EOL;
+                echo '['.sprintf('%'.strlen($total).'d', $current).'/'.$total.'] Processing '.$step.' packages'.PHP_EOL;
             }
 
+            $current += $step;
+
             // prepare packages in memory
             foreach ($packages as $package) {
+                $affectedFiles = array();
+                $name = strtolower($package->getName());
+
+                // clean up versions in individual files
+                if (file_exists($buildDir.'/p/'.$name.'.files')) {
+                    $files = json_decode(file_get_contents($buildDir.'/p/'.$name.'.files.json'));
+
+                    foreach ($files as $file) {
+                        $key = $this->getIndividualFileKey($buildDir.'/'.$file);
+                        $this->loadIndividualFile($buildDir.'/'.$file);
+                        if (isset($this->individualFiles[$key]['packages'][$name])) {
+                            unset($this->individualFiles[$key]['packages'][$name]);
+                            $modifiedIndividualFiles[$key] = true;
+                        }
+                    }
+                }
+
+                // (re)write versions in individual files
+                foreach ($package->getVersions() as $version) {
+                    foreach ($version->getNames() as $versionName) {
+                        if (!preg_match('{^[A-Za-z0-9_-][A-Za-z0-9_.-]+/[A-Za-z0-9_-][A-Za-z0-9_.-]+?$}', $versionName) || strpos($versionName, '..')) {
+                            continue;
+                        }
+
+                        $file = $buildDir.'/p/'.$versionName.'.json';
+                        $key = $this->getIndividualFileKey($file);
+                        $this->dumpVersionToIndividualFile($version, $file, $key);
+                        $modifiedIndividualFiles[$key] = true;
+                        $affectedFiles[$key] = true;
+                    }
+                }
+
+                // store affected files to clean up properly in the next update
+                $this->fs->mkdir(dirname($buildDir.'/p/'.$name));
+                file_put_contents($buildDir.'/p/'.$name.'.files.json', json_encode(array_keys($affectedFiles)));
+                $modifiedIndividualFiles['p/'.$name.'.files.json'] = true;
 
                 // clean up all versions of that package
                 foreach (glob($buildDir.'/packages*.json') as $file) {
                     $key = basename($file);
                     $this->loadFile($file);
-                    if (isset($this->files[$key]['packages'][$package->getName()])) {
-                        unset($this->files[$key]['packages'][$package->getName()]);
+                    if (isset($this->files[$key]['packages'][$name])) {
+                        unset($this->files[$key]['packages'][$name]);
                         $modifiedFiles[$key] = true;
                     }
                 }
@@ -129,6 +176,19 @@ class Dumper
             $this->doctrine->getEntityManager()->flush();
             $this->doctrine->getEntityManager()->clear();
             unset($packages);
+
+            if ($current % 250 === 0 || !$packageIds) {
+                if ($verbose) {
+                    echo 'Dumping individual files'.PHP_EOL;
+                }
+
+                // dump individual files to build dir
+                foreach ($this->individualFiles as $file => $dummy) {
+                    $this->dumpIndividualFile($buildDir.'/'.$file, $file);
+                }
+
+                $this->individualFiles = array();
+            }
         }
 
         // prepare root file
@@ -139,6 +199,11 @@ class Dumper
         }
         $url = $this->router->generate('track_download', array('name' => 'VND/PKG'));
         $this->files['packages_root.json']['notify'] = str_replace('VND/PKG', '%package%', $url);
+        $this->files['packages_root.json']['providers'] = '/p/%package%.json';
+
+        if ($verbose) {
+            echo 'Dumping complete files'.PHP_EOL;
+        }
 
         // dump files to build dir
         foreach ($modifiedFiles as $file => $dummy) {
@@ -147,19 +212,46 @@ class Dumper
         }
         $this->dumpFile($rootFile);
 
+        if ($verbose) {
+            echo 'Putting new files in production'.PHP_EOL;
+        }
+
         // put the new files in production
         foreach ($modifiedFiles as $file => $dummy) {
             rename($buildDir.'/'.$file, $webDir.'/'.$file);
         }
         rename($rootFile, $webDir.'/'.basename($rootFile));
 
+        // put new individual files in production
+        foreach ($modifiedIndividualFiles as $file => $dummy) {
+            $this->fs->mkdir(dirname($webDir.'/'.$file));
+            rename($buildDir.'/'.$file, $webDir.'/'.$file);
+        }
+
         if ($force) {
+            if ($verbose) {
+                echo 'Cleaning up outdated files'.PHP_EOL;
+            }
+
             // clear files that were not created in this build
             foreach (glob($webDir.'/packages-*.json') as $file) {
                 if (!isset($modifiedFiles[basename($file)])) {
                     unlink($file);
                 }
             }
+
+            $finder = Finder::create()
+                ->files()
+                ->ignoreVCS(true)
+                ->name('*.json')
+                ->in($webDir.'/p/');
+
+            foreach ($finder as $file) {
+                $key = $this->getIndividualFileKey(strtr($file, '\\', '/'));
+                if (!isset($modifiedIndividualFiles[$key])) {
+                    unlink($file);
+                }
+            }
         }
     }
 
@@ -191,12 +283,46 @@ class Dumper
         file_put_contents($file, json_encode($this->files[$key]));
     }
 
+    private function loadIndividualFile($path, $key)
+    {
+        if (isset($this->individualFiles[$key])) {
+            return;
+        }
+
+        if (file_exists($path)) {
+            $this->individualFiles[$key] = json_decode(file_get_contents($path), true);
+        } else {
+            $this->individualFiles[$key] = array();
+        }
+    }
+
+    private function dumpIndividualFile($path, $key)
+    {
+        // sort all versions and packages to make sha1 consistent
+        ksort($this->individualFiles[$key]['packages']);
+        foreach ($this->individualFiles[$key]['packages'] as $package => $versions) {
+            ksort($this->individualFiles[$key]['packages'][$package]);
+        }
+
+        $this->fs->mkdir(dirname($path));
+
+        file_put_contents($path, json_encode($this->individualFiles[$key]));
+    }
+
     private function dumpVersion(Version $version, $file)
     {
         $this->loadFile($file);
         $this->files[basename($file)]['packages'][$version->getName()][$version->getVersion()] = $version->toArray();
     }
 
+    private function dumpVersionToIndividualFile(Version $version, $file, $key)
+    {
+        $this->loadIndividualFile($file, $key);
+        $data = $version->toArray();
+        $data['uid'] = $version->getId();
+        $this->individualFiles[$key]['packages'][strtolower($version->getName())][$version->getVersion()] = $data;
+    }
+
     private function getTargetFile(Version $version)
     {
         if ($version->isDevelopment()) {
@@ -208,4 +334,9 @@ class Dumper
 
         return 'packages-' . ($date->format('Y') === date('Y') ? $date->format('Y-m') : $date->format('Y')) . '.json';
     }
+
+    private function getIndividualFileKey($path)
+    {
+        return preg_replace('{^.*?[/\\\\](p[/\\\\].+?(?:\.files)?\.json)$}', '$1', $path);
+    }
 }