Implemented update package command (very rough but working, needs refactoring)

src/Packagist/WebBundle/Command/UpdatePackagesCommand.php

@@ -18,6 +18,10 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\HttpKernel\KernelInterface;
 use Symfony\Component\Finder\Finder;
+use Packagist\WebBundle\Entity\Version;
+use Packagist\WebBundle\Entity\Tag;
+use Packagist\WebBundle\Entity\Author;
+use Packagist\WebBundle\Entity\Requirement;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -30,7 +34,7 @@ class UpdatePackagesCommand extends ContainerAwareCommand
     protected function configure()
     {
         $this
-            ->setName('pkg:update-packages')
+            ->setName('pkg:update')
             ->setDefinition(array(
             ))
             ->setDescription('Updates packages')
@@ -45,18 +49,141 @@ EOF
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
-        $em = $this->container->get('doctrine')->getEntityManager();
+        $em = $this->getContainer()->get('doctrine')->getEntityManager();
+        $logger = $this->getContainer()->get('logger');
 
         $qb = $em->createQueryBuilder();
-        $qb->select('p')
+        $qb->select('p, v')
             ->from('Packagist\WebBundle\Entity\Package', 'p')
-            ->where('p.status = ?1')
-            ->andWhere('p.lastUpdate IS NULL')
-            ->setParameter(1, 'active');
+            ->leftJoin('p.versions', 'v')
+            ->where('p.crawledAt IS NULL OR p.crawledAt < ?0')
+            ->setParameters(array(date('Y-m-d H:i:s', time() - 3600)));
 
         foreach ($qb->getQuery()->getResult() as $package) {
-            //$package->lastUpdate = new \DateTime;
-            //$em->flush();
+            $repo = $package->getRepository();
+
+            // Process GitHub via API
+            if (preg_match('#^(?:https?|git)://github\.com/([^/]+)/(.+?)(?:\.git)?$#', $repo, $match)) {
+                $owner = $match[1];
+                $repository = $match[2];
+                $output->writeln('Importing '.$owner.'/'.$repository);
+
+                $repoData = json_decode(file_get_contents('http://github.com/api/v2/json/repos/show/'.$owner.'/'.$repository), true);
+                if (!$repoData) {
+                    $output->writeln('Err: Could not fetch data from: '.$repo.', skipping.');
+                    continue;
+                }
+
+                $tagsData = json_decode(file_get_contents('http://github.com/api/v2/json/repos/show/'.$owner.'/'.$repository.'/tags'), true);
+
+                foreach ($tagsData['tags'] as $tag => $hash) {
+                    $data = json_decode(file_get_contents('https://raw.github.com/'.$owner.'/'.$repository.'/'.$hash.'/composer.json'), true);
+
+                    // silently skip tags without composer.json, this is expected.
+                    if (!$data) {
+                        continue;
+                    }
+
+                    // TODO parse $data['version'] w/ composer version parser, if no match, ignore the tag
+
+                    // check if we have that version yet
+                    foreach ($package->getVersions() as $version) {
+                        if ($version->getVersion() === $data['version']) {
+                            continue 2;
+                        }
+                    }
+
+                    if ($data['name'] !== $package->getName()) {
+                        $output->writeln('Err: Package name seems to have changed for '.$repo.'@'.$tag.' '.$hash.', skipping');
+                        continue;
+                    }
+
+                    $version = new Version();
+                    $em->persist($version);
+
+                    foreach (array('name', 'description', 'homepage', 'license', 'version') as $field) {
+                        if (isset($data[$field])) {
+                            $version->{'set'.$field}($data[$field]);
+                        }
+                    }
+
+                    // fetch date from the commit if not specified
+                    if (!isset($data['time'])) {
+                        $commit = json_decode(file_get_contents('http://github.com/api/v2/json/commits/show/'.$owner.'/'.$repository.'/'.$hash), true);
+                        $data['time'] = $commit['commit']['committed_date'];
+                    }
+
+                    $version->setPackage($package);
+                    $version->setUpdatedAt(new \DateTime);
+                    $version->setReleasedAt(new \DateTime($data['time']));
+                    $version->setSource(array('type' => 'git', 'url' => 'http://github.com/'.$owner.'/'.$repository.'.git'));
+
+                    if ($repoData['repository']['has_downloads']) {
+                        $downloadUrl = 'https://github.com/'.$owner.'/'.$repository.'/zipball/'.$tag;
+                        $checksum = hash_file('sha1', $downloadUrl);
+                        $version->setDist(array('type' => 'zip', 'url' => $downloadUrl, 'shasum' => $checksum ?: ''));
+                    } else {
+                        // TODO clone the repo and build/host a zip ourselves. Not sure if this can happen, but it'll be needed for non-GitHub repos anyway
+                    }
+
+                    if (isset($data['keywords'])) {
+                        foreach ($data['keywords'] as $keyword) {
+                            $version->addTags(Tag::getByName($em, $keyword, true));
+                        }
+                    }
+                    if (isset($data['authors'])) {
+                        foreach ($data['authors'] as $authorData) {
+                            $author = null;
+                            // skip authors with no information
+                            if (!isset($authorData['email']) && !isset($authorData['name'])) {
+                                continue;
+                            }
+
+                            if (isset($authorData['email'])) {
+                                $qb = $em->createQueryBuilder();
+                                $qb->select('a')
+                                    ->from('Packagist\WebBundle\Entity\Author', 'a')
+                                    ->where('a.email = ?0')
+                                    ->setParameters(array($authorData['email']))
+                                    ->setMaxResults(1);
+                                $author = $qb->getQuery()->getOneOrNullResult();
+                            }
+
+                            if (!$author) {
+                                $author = new Author();
+                                $em->persist($author);
+                            }
+                            foreach (array('email', 'name', 'homepage') as $field) {
+                                if (isset($authorData[$field])) {
+                                    $author->{'set'.$field}($authorData[$field]);
+                                }
+                            }
+                            $author->setUpdatedAt(new \DateTime);
+                            $version->addAuthors($author);
+                            $author->addVersions($version);
+                        }
+                    }
+                    if (isset($data['require'])) {
+                        foreach ($data['require'] as $requireName => $requireVersion) {
+                            $requirement = new Requirement();
+                            $em->persist($requirement);
+                            $requirement->setPackageName($requireName);
+                            $requirement->setPackageVersion($requireVersion);
+                            $version->addRequirements($requirement);
+                            $requirement->setVersion($version);
+                        }
+                    }
+                }
+
+                // TODO parse composer.json on every branch matching a "$num.x.x" version scheme, + the master one, for all "x.y.z-dev" versions, usable through "latest-dev"
+            } else {
+                // TODO support other repos
+                $output->writeln('Err: unsupported repository: '.$repo);
+                continue;
+            }
+            $package->setUpdatedAt(new \DateTime);
+            $package->setCrawledAt(new \DateTime);
+            $em->flush();
         }
     }
 }