More refactoring

src/Packagist/WebBundle/Command/UpdatePackagesCommand.php

@@ -62,106 +62,116 @@ EOF
             ->setParameters(array(date('Y-m-d H:i:s', time() - 3600)));
 
         foreach ($qb->getQuery()->getResult() as $package) {
-            $repo = $provider->getRepository($package->getRepository());
+            $repository = $provider->getRepository($package->getRepository());
 
-            if (!$repo) {
+            if (!$repository) {
                 $output->writeln('<error>Unsupported repository: '.$package->getRepository().'</error>');
                 continue;
             }
 
-            $output->writeln('Importing '.$repo->getUrl());
+            $output->writeln('Importing '.$repository->getUrl());
 
             try {
-                $files = $repo->getAllComposerFiles();
+                foreach ($repository->getTags() as $tag => $identifier) {
+                    // TODO parse tag name (or fetch composer file?) w/ composer version parser, if no match, ignore the tag
+                    $this->fetchInformation($output, $em, $package, $repository, $identifier);
+                }
+
+                foreach ($repository->getBranches() as $branch => $identifier) {
+                    // TODO parse branch name, matching a "$num.x.x" version scheme, + the master one
+                    // use for all "x.y.z-dev" versions, usable through "latest-dev"
+                    $this->fetchInformation($output, $em, $package, $repository, $identifier);
+                }
+
+                $package->setUpdatedAt(new \DateTime);
+                $package->setCrawledAt(new \DateTime);
+                $em->flush();
             } catch (\Exception $e) {
-                $output->writeln('<error>Could not fetch data from: '.$repo->getUrl().', skipping.</error>');
+                $output->writeln('<error>Exception: '.$e->getMessage().', skipping package.</error>');
                 continue;
             }
+        }
+    }
 
-            foreach ($files as $uniqid => $data) {
-                // check if we have that version yet
-                foreach ($package->getVersions() as $version) {
-                    if ($version->getVersion() === $data['version']) {
-                        continue 2;
-                    }
-                }
+    protected function fetchInformation(OutputInterface $output, $em, $package, $repository, $identifier)
+    {
+        $data = $repository->getComposerInformation($identifier);
 
-                if ($data['name'] !== $package->getName()) {
-                    $output->writeln('<error>Package name seems to have changed for '.$repo->getUrl().'@'.$uniqid.', skipping</error>');
-                    continue;
-                }
+        // check if we have that version yet
+        foreach ($package->getVersions() as $version) {
+            if ($version->getVersion() === $data['version']) {
+                return;
+            }
+        }
 
-                $version = new Version();
-                $em->persist($version);
+        if ($data['name'] !== $package->getName()) {
+            $output->writeln('<error>Package name seems to have changed for '.$repository->getUrl().'@'.$identifier.', skipping tag.</error>');
+            return;
+        }
 
-                foreach (array('name', 'description', 'homepage', 'license', 'version') as $field) {
-                    if (isset($data[$field])) {
-                        $version->{'set'.$field}($data[$field]);
-                    }
-                }
+        $version = new Version();
+        $em->persist($version);
 
-                $version->setPackage($package);
-                $version->setUpdatedAt(new \DateTime);
-                $version->setReleasedAt(new \DateTime($data['time']));
-                $version->setSource(array('type' => $repo->getType(), 'url' => $repo->getUrl()));
+        foreach (array('name', 'description', 'homepage', 'license', 'version') as $field) {
+            if (isset($data[$field])) {
+                $version->{'set'.$field}($data[$field]);
+            }
+        }
 
-                $checksum = hash_file('sha1', $data['download']);
-                $version->setDist(array('type' => 'zip', 'url' => $data['download'], 'shasum' => $checksum ?: ''));
+        $version->setPackage($package);
+        $version->setUpdatedAt(new \DateTime);
+        $version->setReleasedAt(new \DateTime($data['time']));
+        $version->setSource(array('type' => $repository->getType(), 'url' => $repository->getUrl()));
+        $version->setDist($repository->getDist($identifier));
 
-                if (isset($data['keywords'])) {
-                    foreach ($data['keywords'] as $keyword) {
-                        $version->addTags(Tag::getByName($em, $keyword, true));
-                    }
+        if (isset($data['keywords'])) {
+            foreach ($data['keywords'] as $keyword) {
+                $version->addTags(Tag::getByName($em, $keyword, true));
+            }
+        }
+
+        if (isset($data['authors'])) {
+            foreach ($data['authors'] as $authorData) {
+                $author = null;
+                // skip authors with no information
+                if (!isset($authorData['email']) && !isset($authorData['name'])) {
+                    continue;
                 }
-                if (isset($data['authors'])) {
-                    foreach ($data['authors'] as $authorData) {
-                        $author = null;
-                        // skip authors with no information
-                        if (!isset($authorData['email']) && !isset($authorData['name'])) {
-                            continue;
-                        }
-
-                        if (isset($authorData['email'])) {
-                            $qb = $em->createQueryBuilder();
-                            $qb->select('a')
-                                ->from('Packagist\WebBundle\Entity\Author', 'a')
-                                ->where('a.email = ?0')
-                                ->setParameters(array($authorData['email']))
-                                ->setMaxResults(1);
-                            $author = $qb->getQuery()->getOneOrNullResult();
-                        }
-
-                        if (!$author) {
-                            $author = new Author();
-                            $em->persist($author);
-                        }
-                        foreach (array('email', 'name', 'homepage') as $field) {
-                            if (isset($authorData[$field])) {
-                                $author->{'set'.$field}($authorData[$field]);
-                            }
-                        }
-                        $author->setUpdatedAt(new \DateTime);
-                        $version->addAuthors($author);
-                        $author->addVersions($version);
-                    }
+
+                if (isset($authorData['email'])) {
+                    $qb = $em->createQueryBuilder();
+                    $qb->select('a')
+                        ->from('Packagist\WebBundle\Entity\Author', 'a')
+                        ->where('a.email = ?0')
+                        ->setParameters(array($authorData['email']))
+                        ->setMaxResults(1);
+                    $author = $qb->getQuery()->getOneOrNullResult();
+                }
+
+                if (!$author) {
+                    $author = new Author();
+                    $em->persist($author);
                 }
-                if (isset($data['require'])) {
-                    foreach ($data['require'] as $requireName => $requireVersion) {
-                        $requirement = new Requirement();
-                        $em->persist($requirement);
-                        $requirement->setPackageName($requireName);
-                        $requirement->setPackageVersion($requireVersion);
-                        $version->addRequirements($requirement);
-                        $requirement->setVersion($version);
+                foreach (array('email', 'name', 'homepage') as $field) {
+                    if (isset($authorData[$field])) {
+                        $author->{'set'.$field}($authorData[$field]);
                     }
                 }
+                $author->setUpdatedAt(new \DateTime);
+                $version->addAuthors($author);
+                $author->addVersions($version);
             }
+        }
 
-            // TODO parse composer.json on every branch matching a "$num.x.x" version scheme, + the master one, for all "x.y.z-dev" versions, usable through "latest-dev"
-
-            $package->setUpdatedAt(new \DateTime);
-            $package->setCrawledAt(new \DateTime);
-            $em->flush();
+        if (isset($data['require'])) {
+            foreach ($data['require'] as $requireName => $requireVersion) {
+                $requirement = new Requirement();
+                $em->persist($requirement);
+                $requirement->setPackageName($requireName);
+                $requirement->setPackageVersion($requireVersion);
+                $version->addRequirements($requirement);
+                $requirement->setVersion($version);
+            }
         }
     }
 }

src/Packagist/WebBundle/Repository/Repository/GitHubRepository.php

@@ -6,7 +6,9 @@ class GitHubRepository implements RepositoryInterface
 {
     protected $owner;
     protected $repository;
-    protected $data;
+    protected $repositoryData;
+    protected $tags;
+    protected $branches;
 
     public function __construct($url)
     {
@@ -15,60 +17,87 @@ class GitHubRepository implements RepositoryInterface
         $this->repository = $match[2];
     }
 
-    protected function getRepoData()
-    {
-        if (null === $this->data) {
-            $url = 'http://github.com/api/v2/json/repos/show/'.$this->owner.'/'.$this->repository;
-            $this->data = json_decode(@file_get_contents($url), true);
-            if (!$this->data) {
-                throw new \UnexpectedValueException('Failed to download from '.$url);
-            }
-        }
-        return $this->data;
-    }
-
+    /**
+     * {@inheritDoc}
+     */
     public function getType()
     {
         return 'git';
     }
 
+    /**
+     * {@inheritDoc}
+     */
     public function getUrl()
     {
         return 'http://github.com/'.$this->owner.'/'.$this->repository.'.git';
     }
 
-    protected function getDist($tag)
+    /**
+     * {@inheritDoc}
+     */
+    public function getDist($identifier)
     {
-        $repoData = $this->getRepoData();
+        $repoData = $this->getRepositoryData();
         if ($repoData['repository']['has_downloads']) {
-            return 'https://github.com/'.$this->owner.'/'.$this->repository.'/zipball/'.$tag;
-        } else {
-            // TODO clone the repo and build/host a zip ourselves. Not sure if this can happen, but it'll be needed for non-GitHub repos anyway
+            $label = array_search($identifier, (array) $this->tags) ?: array_search($identifier, (array) $this->branches) ?: $identifier;
+            $url = 'https://github.com/'.$this->owner.'/'.$this->repository.'/zipball/'.$label;
+            $checksum = hash_file('sha1', $url);
+            return array('type' => 'zip', 'url' => $url, 'shasum' => $checksum ?: '');
         }
+
+        // TODO clone the repo and build/host a zip ourselves. Not sure if this can happen, but it'll be needed for non-GitHub repos anyway
+        throw new \LogicException('Not implemented yet.');
     }
 
-    public function getAllComposerFiles()
+    /**
+     * {@inheritDoc}
+     */
+    public function getComposerInformation($identifier)
     {
-        $repoData = $this->getRepoData();
+        $composer = json_decode(@file_get_contents('https://raw.github.com/'.$this->owner.'/'.$this->repository.'/'.$identifier.'/composer.json'), true);
+        if (!$composer) {
+            throw new \UnexpectedValueException('Failed to download retrieve composer information for identifier '.$identifier.' in '.$this->getUrl());
+        }
 
-        $files = array();
+        if (!isset($composer['time'])) {
+            $commit = json_decode(file_get_contents('http://github.com/api/v2/json/commits/show/'.$this->owner.'/'.$this->repository.'/'.$identifier), true);
+            $composer['time'] = $commit['commit']['committed_date'];
+        }
 
-        $tagsData = json_decode(file_get_contents('http://github.com/api/v2/json/repos/show/'.$this->owner.'/'.$this->repository.'/tags'), true);
-        foreach ($tagsData['tags'] as $tag => $hash) {
-            if ($file = json_decode(file_get_contents('https://raw.github.com/'.$this->owner.'/'.$this->repository.'/'.$hash.'/composer.json'), true)) {
-                if (!isset($file['time'])) {
-                    $commit = json_decode(file_get_contents('http://github.com/api/v2/json/commits/show/'.$this->owner.'/'.$this->repository.'/'.$tag), true);
-                    $file['time'] = $commit['commit']['committed_date'];
-                }
+        return $composer;
+    }
 
-                // TODO parse $data['version'] w/ composer version parser, if no match, ignore the tag
+    /**
+     * {@inheritDoc}
+     */
+    public function getTags()
+    {
+        if (null === $this->tags) {
+            $tagsData = json_decode(file_get_contents('http://github.com/api/v2/json/repos/show/'.$this->owner.'/'.$this->repository.'/tags'), true);
+            $this->tags = $tagsData['tags'];
+        }
+        return $this->tags;
+    }
 
-                $file['download'] = $this->getDist($tag);
+    /**
+     * {@inheritDoc}
+     */
+    public function getBranches()
+    {
+        // TODO implement
+        return array();
+    }
 
-                $files[$tag] = $file;
+    protected function getRepositoryData()
+    {
+        if (null === $this->repositoryData) {
+            $url = 'http://github.com/api/v2/json/repos/show/'.$this->owner.'/'.$this->repository;
+            $this->repositoryData = json_decode(@file_get_contents($url), true);
+            if (!$this->repositoryData) {
+                throw new \UnexpectedValueException('Failed to download from '.$url);
             }
         }
-
-        return $files;
+        return $this->repositoryData;
     }
 }

src/Packagist/WebBundle/Repository/Repository/RepositoryInterface.php

@@ -5,17 +5,46 @@ namespace Packagist\WebBundle\Repository\Repository;
 interface RepositoryInterface
 {
     /**
-     * Return an array of all composer files (by tag).
+     * Return the composer.json file information
+     *
+     * @param string $identifier Any identifier to a specific branch/tag/commit
+     * @return array containing all infos from the composer.json file
      */
-    public function getAllComposerFiles();
+    function getComposerInformation($identifier);
 
     /**
-     * Return the URL of the Repository
+     * Return list of branches in the repository
+     *
+     * @return array Branch names as keys, identifiers as values
      */
-    public function getUrl();
+    function getBranches();
+
+    /**
+     * Return list of tags in the repository
+     *
+     * @return array Tag names as keys, identifiers as values
+     */
+    function getTags();
+
+    /**
+     * Return the URL of the repository
+     *
+     * @param string $identifier Any identifier to a specific branch/tag/commit
+     * @return array With type, url and shasum properties.
+     */
+    function getDist($identifier);
+
+    /**
+     * Return the URL of the repository
+     *
+     * @return string
+     */
+    function getUrl();
 
     /**
      * Return the type of the repository
+     *
+     * @return string
      */
-    public function getType();
+    function getType();
 }