Add HttpDownloader to wrap/replace RemoteFilesystem with a new curl multi implementation

composer.json

@@ -34,7 +34,8 @@
         "symfony/console": "^2.7 || ^3.0 || ^4.0",
         "symfony/filesystem": "^2.7 || ^3.0 || ^4.0",
         "symfony/finder": "^2.7 || ^3.0 || ^4.0",
-        "symfony/process": "^2.7 || ^3.0 || ^4.0"
+        "symfony/process": "^2.7 || ^3.0 || ^4.0",
+        "react/promise": "^1.2"
     },
     "conflict": {
         "symfony/console": "2.8.38"

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "e46280c4cfd37bf3ec8be36095feb20e",
+    "content-hash": "d356b92e869790db1e9d2c0f4b10935e",
     "packages": [
         {
             "name": "composer/ca-bundle",
@@ -342,6 +342,50 @@
             ],
             "time": "2018-11-20T15:27:04+00:00"
         },
+        {
+            "name": "react/promise",
+            "version": "v1.2.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/reactphp/promise.git",
+                "reference": "eefff597e67ff66b719f8171480add3c91474a1e"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/reactphp/promise/zipball/eefff597e67ff66b719f8171480add3c91474a1e",
+                "reference": "eefff597e67ff66b719f8171480add3c91474a1e",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.3.3"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.1-dev"
+                }
+            },
+            "autoload": {
+                "psr-0": {
+                    "React\\Promise": "src/"
+                },
+                "files": [
+                    "src/React/Promise/functions_include.php"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jan Sorgalla",
+                    "email": "jsorgalla@gmail.com"
+                }
+            ],
+            "description": "A lightweight implementation of CommonJS Promises/A for PHP",
+            "time": "2016-03-07T13:46:50+00:00"
+        },
         {
             "name": "seld/jsonlint",
             "version": "1.7.1",

src/Composer/Downloader/FileDownloader.php

@@ -24,7 +24,7 @@ use Composer\Plugin\PluginEvents;
 use Composer\Plugin\PreFileDownloadEvent;
 use Composer\EventDispatcher\EventDispatcher;
 use Composer\Util\Filesystem;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\Util\Url as UrlUtil;
 
 /**
@@ -39,7 +39,7 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
 {
     protected $io;
     protected $config;
-    protected $rfs;
+    protected $httpDownloader;
     protected $filesystem;
     protected $cache;
     protected $outputProgress = true;
@@ -52,16 +52,16 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
      * @param IOInterface      $io              The IO instance
      * @param Config           $config          The config
      * @param EventDispatcher  $eventDispatcher The event dispatcher
-     * @param Cache            $cache           Optional cache instance
-     * @param RemoteFilesystem $rfs             The remote filesystem
+     * @param Cache            $cache           Cache instance
+     * @param HttpDownloader   $httpDownloader  The remote filesystem
      * @param Filesystem       $filesystem      The filesystem
      */
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, RemoteFilesystem $rfs = null, Filesystem $filesystem = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, Cache $cache, HttpDownloader $httpDownloader, Filesystem $filesystem = null)
     {
         $this->io = $io;
         $this->config = $config;
         $this->eventDispatcher = $eventDispatcher;
-        $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $config);
+        $this->httpDownloader = $httpDownloader;
         $this->filesystem = $filesystem ?: new Filesystem();
         $this->cache = $cache;
 
@@ -125,13 +125,12 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
         $fileName = $this->getFileName($package, $path);
 
         $processedUrl = $this->processUrl($package, $url);
-        $hostname = parse_url($processedUrl, PHP_URL_HOST);
 
-        $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $processedUrl);
+        $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $processedUrl);
         if ($this->eventDispatcher) {
             $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
         }
-        $rfs = $preFileDownloadEvent->getRemoteFilesystem();
+        $httpDownloader = $preFileDownloadEvent->getHttpDownloader();
 
         try {
             $checksum = $package->getDistSha1Checksum();
@@ -150,7 +149,8 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
                 $retries = 3;
                 while ($retries--) {
                     try {
-                        $rfs->copy($hostname, $processedUrl, $fileName, $this->outputProgress, $package->getTransportOptions());
+                        // TODO handle this->outputProgress
+                        $httpDownloader->copy($processedUrl, $fileName, $package->getTransportOptions());
                         break;
                     } catch (TransportException $e) {
                         // if we got an http response with a proper code, then requesting again will probably not help, abort

src/Composer/Downloader/GzipDownloader.php

@@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher;
 use Composer\Package\PackageInterface;
 use Composer\Util\Platform;
 use Composer\Util\ProcessExecutor;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\IO\IOInterface;
 
 /**
@@ -30,10 +30,10 @@ class GzipDownloader extends ArchiveDownloader
 {
     protected $process;
 
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null)
     {
         $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $eventDispatcher, $cache, $rfs);
+        parent::__construct($io, $config, $eventDispatcher, $cache, $downloader);
     }
 
     protected function extract($file, $path)

src/Composer/Downloader/RarDownloader.php

@@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher;
 use Composer\Util\IniHelper;
 use Composer\Util\Platform;
 use Composer\Util\ProcessExecutor;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\IO\IOInterface;
 use RarArchive;
 
@@ -33,10 +33,10 @@ class RarDownloader extends ArchiveDownloader
 {
     protected $process;
 
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null)
     {
         $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $eventDispatcher, $cache, $rfs);
+        parent::__construct($io, $config, $eventDispatcher, $cache, $downloader);
     }
 
     protected function extract($file, $path)

src/Composer/Downloader/XzDownloader.php

@@ -17,7 +17,7 @@ use Composer\Cache;
 use Composer\EventDispatcher\EventDispatcher;
 use Composer\Package\PackageInterface;
 use Composer\Util\ProcessExecutor;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\IO\IOInterface;
 
 /**
@@ -30,11 +30,11 @@ class XzDownloader extends ArchiveDownloader
 {
     protected $process;
 
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null)
     {
         $this->process = $process ?: new ProcessExecutor($io);
 
-        parent::__construct($io, $config, $eventDispatcher, $cache, $rfs);
+        parent::__construct($io, $config, $eventDispatcher, $cache, $downloader);
     }
 
     protected function extract($file, $path)

src/Composer/Downloader/ZipDownloader.php

@@ -19,7 +19,7 @@ use Composer\Package\PackageInterface;
 use Composer\Util\IniHelper;
 use Composer\Util\Platform;
 use Composer\Util\ProcessExecutor;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\IO\IOInterface;
 use Symfony\Component\Process\ExecutableFinder;
 use ZipArchive;
@@ -36,10 +36,10 @@ class ZipDownloader extends ArchiveDownloader
     protected $process;
     private $zipArchiveObject;
 
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null)
     {
         $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $eventDispatcher, $cache, $rfs);
+        parent::__construct($io, $config, $eventDispatcher, $cache, $downloader);
     }
 
     /**

src/Composer/Factory.php

@@ -23,7 +23,7 @@ use Composer\Repository\WritableRepositoryInterface;
 use Composer\Util\Filesystem;
 use Composer\Util\Platform;
 use Composer\Util\ProcessExecutor;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\Util\Silencer;
 use Composer\Plugin\PluginEvents;
 use Composer\EventDispatcher\Event;
@@ -325,7 +325,7 @@ class Factory
             $io->loadConfiguration($config);
         }
 
-        $rfs = self::createRemoteFilesystem($io, $config);
+        $rfs = self::createHttpDownloader($io, $config);
 
         // initialize event dispatcher
         $dispatcher = new EventDispatcher($composer, $io);
@@ -451,7 +451,7 @@ class Factory
      * @param  EventDispatcher            $eventDispatcher
      * @return Downloader\DownloadManager
      */
-    public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null)
+    public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null)
     {
         $cache = null;
         if ($config->get('cache-files-ttl') > 0) {
@@ -579,10 +579,10 @@ class Factory
     /**
      * @param  IOInterface      $io      IO instance
      * @param  Config           $config  Config instance
-     * @param  array            $options Array of options passed directly to RemoteFilesystem constructor
-     * @return RemoteFilesystem
+     * @param  array            $options Array of options passed directly to HttpDownloader constructor
+     * @return HttpDownloader
      */
-    public static function createRemoteFilesystem(IOInterface $io, Config $config = null, $options = array())
+    public static function createHttpDownloader(IOInterface $io, Config $config = null, $options = array())
     {
         static $warned = false;
         $disableTls = false;
@@ -607,7 +607,7 @@ class Factory
             $remoteFilesystemOptions = array_replace_recursive($remoteFilesystemOptions, $options);
         }
         try {
-            $remoteFilesystem = new RemoteFilesystem($io, $config, $remoteFilesystemOptions, $disableTls);
+            $remoteFilesystem = new HttpDownloader($io, $config, $remoteFilesystemOptions, $disableTls);
         } catch (TransportException $e) {
             if (false !== strpos($e->getMessage(), 'cafile')) {
                 $io->write('<error>Unable to locate a valid CA certificate file. You must set a valid \'cafile\' option.</error>');

src/Composer/Plugin/PreFileDownloadEvent.php

@@ -13,7 +13,7 @@
 namespace Composer\Plugin;
 
 use Composer\EventDispatcher\Event;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 
 /**
  * The pre file download event.
@@ -23,7 +23,7 @@ use Composer\Util\RemoteFilesystem;
 class PreFileDownloadEvent extends Event
 {
     /**
-     * @var RemoteFilesystem
+     * @var HttpDownloader
      */
     private $rfs;
 
@@ -36,10 +36,10 @@ class PreFileDownloadEvent extends Event
      * Constructor.
      *
      * @param string           $name         The event name
-     * @param RemoteFilesystem $rfs
+     * @param HttpDownloader $rfs
      * @param string           $processedUrl
      */
-    public function __construct($name, RemoteFilesystem $rfs, $processedUrl)
+    public function __construct($name, HttpDownloader $rfs, $processedUrl)
     {
         parent::__construct($name);
         $this->rfs = $rfs;
@@ -47,21 +47,17 @@ class PreFileDownloadEvent extends Event
     }
 
     /**
-     * Returns the remote filesystem
-     *
-     * @return RemoteFilesystem
+     * @return HttpDownloader
      */
-    public function getRemoteFilesystem()
+    public function getHttpDownloader()
     {
         return $this->rfs;
     }
 
     /**
-     * Sets the remote filesystem
-     *
-     * @param RemoteFilesystem $rfs
+     * @param HttpDownloader $rfs
      */
-    public function setRemoteFilesystem(RemoteFilesystem $rfs)
+    public function setHttpDownloader(HttpDownloader $rfs)
     {
         $this->rfs = $rfs;
     }

src/Composer/Repository/ComposerRepository.php

@@ -21,7 +21,7 @@ use Composer\Cache;
 use Composer\Config;
 use Composer\Factory;
 use Composer\IO\IOInterface;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\Plugin\PluginEvents;
 use Composer\Plugin\PreFileDownloadEvent;
 use Composer\EventDispatcher\EventDispatcher;
@@ -40,7 +40,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
     protected $url;
     protected $baseUrl;
     protected $io;
-    protected $rfs;
+    protected $httpDownloader;
     protected $cache;
     protected $notifyUrl;
     protected $searchUrl;
@@ -60,8 +60,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
     private $rootData;
     private $hasPartialPackages;
     private $partialPackagesByName;
+    private $versionParser;
 
-    public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null)
+    public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader)
     {
         parent::__construct();
         if (!preg_match('{^[\w.]+\??://}', $repoConfig['url'])) {
@@ -98,12 +99,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         $this->baseUrl = rtrim(preg_replace('{(?:/[^/\\\\]+\.json)?(?:[?#].*)?$}', '', $this->url), '/');
         $this->io = $io;
         $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$');
-        $this->loader = new ArrayLoader();
-        if ($rfs && $this->options) {
-            $rfs = clone $rfs;
-            $rfs->setOptions($this->options);
-        }
-        $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $this->config, $this->options);
+        $this->versionParser = new VersionParser();
+        $this->loader = new ArrayLoader($this->versionParser);
+        if ($httpDownloader && $this->options) {
+            // TODO solve this somehow - should be sent a request time not on the instance
+            $httpDownloader = clone $httpDownloader;
+            $httpDownloader->setOptions($this->options);
+        }
+        $this->httpDownloader = $httpDownloader;
         $this->eventDispatcher = $eventDispatcher;
         $this->repoConfig = $repoConfig;
     }
@@ -129,8 +132,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
 
         $name = strtolower($name);
         if (!$constraint instanceof ConstraintInterface) {
-            $versionParser = new VersionParser();
-            $constraint = $versionParser->parseConstraints($constraint);
+            $constraint = $this->versionParser->parseConstraints($constraint);
         }
 
         foreach ($this->getProviderNames() as $providerName) {
@@ -161,8 +163,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         $name = strtolower($name);
 
         if (null !== $constraint && !$constraint instanceof ConstraintInterface) {
-            $versionParser = new VersionParser();
-            $constraint = $versionParser->parseConstraints($constraint);
+            $constraint = $this->versionParser->parseConstraints($constraint);
         }
 
         $packages = array();
@@ -196,8 +197,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
 
     public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable)
     {
+        if ($this->lazyProvidersUrl) {
+            return $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable);
+        }
         if (!$this->hasProviders()) {
-            // TODO build more efficient version of this
             return parent::loadPackages($packageNameMap, $isPackageAcceptableCallable);
         }
 
@@ -235,9 +238,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         if ($this->searchUrl && $mode === self::SEARCH_FULLTEXT) {
             $url = str_replace(array('%query%', '%type%'), array($query, $type), $this->searchUrl);
 
-            $hostname = parse_url($url, PHP_URL_HOST) ?: $url;
-            $json = $this->rfs->getContents($hostname, $url, false);
-            $search = JsonFile::parseJson($json, $url);
+            $search = $this->httpDownloader->get($url)->decodeJson();
 
             if (empty($search['results'])) {
                 return array();
@@ -496,6 +497,85 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         $this->configurePackageTransportOptions($package);
     }
 
+    private function loadAsyncPackages(array $packageNames, $isPackageAcceptableCallable)
+    {
+        $this->loadRootServerFile();
+
+        $packages = array();
+        $repo = $this;
+
+        $createPackageIfAcceptable = function ($version, $constraint) use (&$packages, $isPackageAcceptableCallable, $repo) {
+            if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) {
+                return;
+            }
+
+            if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) {
+                return;
+            }
+
+            // load acceptable packages in the providers
+            $package = $this->createPackage($version, 'Composer\Package\CompletePackage');
+            $package->setRepository($repo);
+
+            // if there was no version_normalized, then we need to check now for the constraint
+            if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) {
+                $packages[spl_object_hash($package)] = $package;
+                if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) {
+                    $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf();
+                }
+            }
+        };
+
+        if ($this->lazyProvidersUrl) {
+            foreach ($packageNames as $name => $constraint) {
+                $url = str_replace('%package%', $name, $this->lazyProvidersUrl);
+                $cacheKey = 'provider-'.strtr($name, '/', '$').'.json';
+
+                $lastModified = null;
+                if ($contents = $this->cache->read($cacheKey)) {
+                    $contents = json_decode($contents, true);
+                    $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null;
+                }
+
+                $this->asyncFetchFile($url, $cacheKey, $lastModified)
+                    ->then(function ($response) use (&$packages, $contents, $name, $constraint, $createPackageIfAcceptable) {
+                        if (true === $response) {
+                            $response = $contents;
+                        }
+
+                        $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time');
+                        foreach ($response['packages'][$name] as $version) {
+                            if (isset($version['versions'])) {
+                                $baseVersion = $version;
+                                foreach ($uniqKeys as $key) {
+                                    unset($baseVersion[$key.'s']);
+                                }
+
+                                foreach ($version['versions'] as $index => $dummy) {
+                                    $unpackedVersion = $baseVersion;
+                                    foreach ($uniqKeys as $key) {
+                                        $unpackedVersion[$key] = $version[$key.'s'][$index];
+                                    }
+
+                                    $createPackageIfAcceptable($unpackedVersion, $constraint);
+                                }
+                            } else {
+                                $createPackageIfAcceptable($version, $constraint);
+                            }
+                        }
+                    }, function ($e) {
+                        // TODO use ->done() above instead with react/promise 2.0
+                        var_dump('Uncaught Ex', $e->getMessage());
+                    });
+            }
+        }
+
+        $this->httpDownloader->wait();
+
+        return $packages;
+        // RepositorySet should call loadMetadata, getMetadata when all promises resolved, then metadataComplete when done so we can GC the loaded json and whatnot then as needed
+    }
+
     protected function loadRootServerFile()
     {
         if (null !== $this->rootData) {
@@ -691,15 +771,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         $retries = 3;
         while ($retries--) {
             try {
-                $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename);
-                if ($this->eventDispatcher) {
-                    $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
-                }
+                $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename);
+                $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
 
-                $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename;
-                $rfs = $preFileDownloadEvent->getRemoteFilesystem();
+                $httpDownloader = $preFileDownloadEvent->getHttpDownloader();
 
-                $json = $rfs->getContents($hostname, $filename, false);
+                $response = $httpDownloader->get($filename);
+                $json = $response->getBody();
                 if ($sha256 && $sha256 !== hash('sha256', $json)) {
                     // undo downgrade before trying again if http seems to be hijacked or modifying content somehow
                     if ($this->allowSslDowngrade) {
@@ -718,7 +796,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                     throw new RepositorySecurityException('The contents of '.$filename.' do not match its signature. This could indicate a man-in-the-middle attack or e.g. antivirus software corrupting files. Try running composer again and report this if you think it is a mistake.');
                 }
 
-                $data = JsonFile::parseJson($json, $filename);
+                $data = $response->decodeJson();
                 if (!empty($data['warning'])) {
                     $this->io->writeError('<warning>Warning from '.$this->url.': '.$data['warning'].'</warning>');
                 }
@@ -728,7 +806,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
 
                 if ($cacheKey) {
                     if ($storeLastModifiedTime) {
-                        $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified');
+                        $lastModifiedDate = $response->getHeader('last-modified');
                         if ($lastModifiedDate) {
                             $data['last-modified'] = $lastModifiedDate;
                             $json = json_encode($data);
@@ -737,8 +815,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                     $this->cache->write($cacheKey, $json);
                 }
 
+                $response->collect();
+
                 break;
             } catch (\Exception $e) {
+                if ($e instanceof \LogicException) {
+                    throw $e;
+                }
+
                 if ($e instanceof TransportException && $e->getStatusCode() === 404) {
                     throw $e;
                 }
@@ -775,20 +859,18 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         $retries = 3;
         while ($retries--) {
             try {
-                $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename);
-                if ($this->eventDispatcher) {
-                    $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
-                }
+                $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename);
+                $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
 
-                $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename;
-                $rfs = $preFileDownloadEvent->getRemoteFilesystem();
+                $httpDownloader = $preFileDownloadEvent->getHttpDownloader();
                 $options = array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime)));
-                $json = $rfs->getContents($hostname, $filename, false, $options);
-                if ($json === '' && $rfs->findStatusCode($rfs->getLastHeaders()) === 304) {
+                $response = $httpDownloader->get($filename, $options);
+                $json = $response->getBody();
+                if ($json === '' && $response->getStatusCode() === 304) {
                     return true;
                 }
 
-                $data = JsonFile::parseJson($json, $filename);
+                $data = $response->decodeJson();
                 if (!empty($data['warning'])) {
                     $this->io->writeError('<warning>Warning from '.$this->url.': '.$data['warning'].'</warning>');
                 }
@@ -796,7 +878,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
                     $this->io->writeError('<info>Info from '.$this->url.': '.$data['info'].'</info>');
                 }
 
-                $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified');
+                $lastModifiedDate = $response->getHeader('last-modified');
+                $response->collect();
                 if ($lastModifiedDate) {
                     $data['last-modified'] = $lastModifiedDate;
                     $json = json_encode($data);
@@ -805,6 +888,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
 
                 return $data;
             } catch (\Exception $e) {
+                if ($e instanceof \LogicException) {
+                    throw $e;
+                }
+
                 if ($e instanceof TransportException && $e->getStatusCode() === 404) {
                     throw $e;
                 }
@@ -825,6 +912,69 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito
         }
     }
 
+    protected function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null)
+    {
+        $retries = 3;
+        $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename);
+        $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent);
+
+        $httpDownloader = $preFileDownloadEvent->getHttpDownloader();
+        $options = $lastModifiedTime ? array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))) : array();
+
+        $io = $this->io;
+        $url = $this->url;
+        $cache = $this->cache;
+        $degradedMode =& $this->degradedMode;
+
+        $accept = function ($response) use ($io, $url, $cache, $cacheKey) {
+            $json = $response->getBody();
+            if ($json === '' && $response->getStatusCode() === 304) {
+                return true;
+            }
+
+            $data = $response->decodeJson();
+            if (!empty($data['warning'])) {
+                $io->writeError('<warning>Warning from '.$url.': '.$data['warning'].'</warning>');
+            }
+            if (!empty($data['info'])) {
+                $io->writeError('<info>Info from '.$url.': '.$data['info'].'</info>');
+            }
+
+            $lastModifiedDate = $response->getHeader('last-modified');
+            $response->collect();
+            if ($lastModifiedDate) {
+                $data['last-modified'] = $lastModifiedDate;
+                $json = JsonFile::encode($data, JsonFile::JSON_UNESCAPED_SLASHES | JsonFile::JSON_UNESCAPED_UNICODE);
+            }
+            $cache->write($cacheKey, $json);
+
+            return $data;
+        };
+
+        $reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, $cache, &$degradedMode) {
+            var_dump('Caught8', $e->getMessage());
+            if ($e instanceof TransportException && $e->getStatusCode() === 404) {
+                return false;
+            }
+
+            if (--$retries) {
+                usleep(100000);
+
+                return $httpDownloader->add($filename, $options)->then($accept, $reject);
+            }
+
+            if (!$degradedMode) {
+                $io->writeError('<warning>'.$e->getMessage().'</warning>');
+                $io->writeError('<warning>'.$url.' could not be fully loaded, package information was loaded from the local cache and may be out of date</warning>');
+            }
+            $degradedMode = true;
+
+            return true;
+        };
+
+        return $httpDownloader->add($filename, $options)->then($accept, $reject);
+    }
+
     /**
      * This initializes the packages key of a partial packages.json that contain some packages inlined + a providers-lazy-url
      *

src/Composer/Repository/RepositoryFactory.php

@@ -16,7 +16,7 @@ use Composer\Factory;
 use Composer\IO\IOInterface;
 use Composer\Config;
 use Composer\EventDispatcher\EventDispatcher;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 use Composer\Json\JsonFile;
 
 /**
@@ -108,10 +108,10 @@ class RepositoryFactory
      * @param  IOInterface       $io
      * @param  Config            $config
      * @param  EventDispatcher   $eventDispatcher
-     * @param  RemoteFilesystem  $rfs
+     * @param  HttpDownloader  $rfs
      * @return RepositoryManager
      */
-    public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null)
+    public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null)
     {
         $rm = new RepositoryManager($io, $config, $eventDispatcher, $rfs);
         $rm->setRepositoryClass('composer', 'Composer\Repository\ComposerRepository');

src/Composer/Repository/RepositoryManager.php

@@ -16,7 +16,7 @@ use Composer\IO\IOInterface;
 use Composer\Config;
 use Composer\EventDispatcher\EventDispatcher;
 use Composer\Package\PackageInterface;
-use Composer\Util\RemoteFilesystem;
+use Composer\Util\HttpDownloader;
 
 /**
  * Repositories manager.
@@ -35,7 +35,7 @@ class RepositoryManager
     private $eventDispatcher;
     private $rfs;
 
-    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null)
+    public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null)
     {
         $this->io = $io;
         $this->config = $config;
@@ -127,7 +127,7 @@ class RepositoryManager
 
         $reflMethod = new \ReflectionMethod($class, '__construct');
         $params = $reflMethod->getParameters();
-        if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\RemoteFilesystem') {
+        if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\HttpDownloader') {
             return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->rfs);
         }
 

src/Composer/Util/Http/CurlDownloader.php

@@ -0,0 +1,282 @@
+<?php
+
+/*
+ * This file is part of Composer.
+ *
+ * (c) Nils Adermann <naderman@naderman.de>
+ *     Jordi Boggiano <j.boggiano@seld.be>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Composer\Util\Http;
+
+use Composer\Config;
+use Composer\IO\IOInterface;
+use Composer\Downloader\TransportException;
+use Composer\CaBundle\CaBundle;
+use Psr\Log\LoggerInterface;
+use React\Promise\Promise;
+
+/**
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ * @author Nicolas Grekas <p@tchwork.com>
+ */
+class CurlDownloader
+{
+    private $multiHandle;
+    private $shareHandle;
+    private $jobs = array();
+    private $io;
+    private $selectTimeout = 5.0;
+    protected $multiErrors = array(
+        CURLM_BAD_HANDLE      => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'),
+        CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."),
+        CURLM_OUT_OF_MEMORY   => array('CURLM_OUT_OF_MEMORY', 'You are doomed.'),
+        CURLM_INTERNAL_ERROR  => array('CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!')
+    );
+
+    private static $options = array(
+        'http' => array(
+            'method' => CURLOPT_CUSTOMREQUEST,
+            'content' => CURLOPT_POSTFIELDS,
+            'proxy' => CURLOPT_PROXY,
+        ),
+        'ssl' => array(
+            'ciphers' => CURLOPT_SSL_CIPHER_LIST,
+            'cafile' => CURLOPT_CAINFO,
+            'capath' => CURLOPT_CAPATH,
+        ),
+    );
+
+    private static $timeInfo = array(
+        'total_time' => true,
+        'namelookup_time' => true,
+        'connect_time' => true,
+        'pretransfer_time' => true,
+        'starttransfer_time' => true,
+        'redirect_time' => true,
+    );
+
+    public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false)
+    {
+        $this->io = $io;
+
+        $this->multiHandle = $mh = curl_multi_init();
+        if (function_exists('curl_multi_setopt')) {
+            curl_multi_setopt($mh, CURLMOPT_PIPELINING, /*CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX*/ 3);
+            if (defined('CURLMOPT_MAX_HOST_CONNECTIONS')) {
+                curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8);
+            }
+        }
+
+        if (function_exists('curl_share_init')) {
+            $this->shareHandle = $sh = curl_share_init();
+            curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
+            curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
+            curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
+        }
+    }
+
+    public function download($resolve, $reject, $origin, $url, $options, $copyTo = null)
+    {
+        $ch = curl_init();
+        $hd = fopen('php://temp/maxmemory:32768', 'w+b');
+
+        // TODO auth & other context
+        // TODO cleanup
+
+        if ($copyTo && !$fd = @fopen($copyTo.'~', 'w+b')) {
+            // TODO throw here probably?
+            $copyTo = null;
+        }
+        if (!$copyTo) {
+            $fd = @fopen('php://temp/maxmemory:524288', 'w+b');
+        }
+
+        if (!isset($options['http']['header'])) {
+            $options['http']['header'] = array();
+        }
+
+        $headers = array_diff($options['http']['header'], array('Connection: close'));
+
+        // TODO
+        $degradedMode = false;
+        if ($degradedMode) {
+            curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+        } else {
+            $headers[] = 'Connection: keep-alive';
+            $version = curl_version();
+            $features = $version['features'];
+            if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) {
+                curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+            }
+        }
+
+        curl_setopt($ch, CURLOPT_URL, $url);
+        curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+        //curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, false);
+        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
+        curl_setopt($ch, CURLOPT_TIMEOUT, 10); // TODO increase
+        curl_setopt($ch, CURLOPT_WRITEHEADER, $hd);
+        curl_setopt($ch, CURLOPT_FILE, $fd);
+        if (function_exists('curl_share_init')) {
+            curl_setopt($ch, CURLOPT_SHARE, $this->shareHandle);
+        }
+
+        foreach (self::$options as $type => $curlOptions) {
+            foreach ($curlOptions as $name => $curlOption) {
+                if (isset($options[$type][$name])) {
+                    curl_setopt($ch, $curlOption, $options[$type][$name]);
+                }
+            }
+        }
+
+        $progress = array_diff_key(curl_getinfo($ch), self::$timeInfo);
+
+        $this->jobs[(int) $ch] = array(
+            'progress' => $progress,
+            'ch' => $ch,
+            //'callback' => $params['notification'],
+            'file' => $copyTo,
+            'hd' => $hd,
+            'fd' => $fd,
+            'resolve' => $resolve,
+            'reject' => $reject,
+        );
+
+        $this->io->write('Downloading '.$url, true, IOInterface::DEBUG);
+
+        $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $ch));
+        //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false);
+    }
+
+    public function tick()
+    {
+        // TODO check we have active handles before doing this
+        if (!$this->jobs) {
+            return;
+        }
+
+        $active = true;
+        try {
+            $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active));
+            if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) {
+                // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select
+                usleep(150);
+            }
+
+            while ($progress = curl_multi_info_read($this->multiHandle)) {
+                $h = $progress['handle'];
+                $i = (int) $h;
+                if (!isset($this->jobs[$i])) {
+                    continue;
+                }
+                $progress = array_diff_key(curl_getinfo($h), self::$timeInfo);
+                $job = $this->jobs[$i];
+                unset($this->jobs[$i]);
+                curl_multi_remove_handle($this->multiHandle, $h);
+                $error = curl_error($h);
+                $errno = curl_errno($h);
+                curl_close($h);
+
+                try {
+                    //$this->onProgress($h, $job['callback'], $progress, $job['progress']);
+                    if ('' !== $error) {
+                        throw new TransportException(curl_error($h));
+                    }
+
+                    if ($job['file']) {
+                        if (CURLE_OK === $errno) {
+                            fclose($job['fd']);
+                            rename($job['file'].'~', $job['file']);
+                            call_user_func($job['resolve'], true);
+                        }
+                        // TODO otherwise show error?
+                    } else {
+                        rewind($job['hd']);
+                        $headers = explode("\r\n", rtrim(stream_get_contents($job['hd'])));
+                        fclose($job['hd']);
+                        rewind($job['fd']);
+                        $contents = stream_get_contents($job['fd']);
+                        fclose($job['fd']);
+                        $this->io->writeError('['.$progress['http_code'].'] '.$progress['url'], true, IOInterface::DEBUG);
+                        call_user_func($job['resolve'], new Response(array('url' => $progress['url']), $progress['http_code'], $headers, $contents));
+                    }
+                } catch (TransportException $e) {
+                    fclose($job['hd']);
+                    fclose($job['fd']);
+                    if ($job['file']) {
+                        @unlink($job['file'].'~');
+                    }
+                    call_user_func($job['reject'], $e);
+                }
+            }
+
+            foreach ($this->jobs as $i => $h) {
+                if (!isset($this->jobs[$i])) {
+                    continue;
+                }
+                $h = $this->jobs[$i]['ch'];
+                $progress = array_diff_key(curl_getinfo($h), self::$timeInfo);
+
+                if ($this->jobs[$i]['progress'] !== $progress) {
+                    $previousProgress = $this->jobs[$i]['progress'];
+                    $this->jobs[$i]['progress'] = $progress;
+                    try {
+                        //$this->onProgress($h, $this->jobs[$i]['callback'], $progress, $previousProgress);
+                    } catch (TransportException $e) {
+                        var_dump('Caught '.$e->getMessage());die;
+                        unset($this->jobs[$i]);
+                        curl_multi_remove_handle($this->multiHandle, $h);
+                        curl_close($h);
+
+                        fclose($job['hd']);
+                        fclose($job['fd']);
+                        if ($job['file']) {
+                            @unlink($job['file'].'~');
+                        }
+                        call_user_func($job['reject'], $e);
+                    }
+                }
+            }
+        } catch (\Exception $e) {
+            var_dump('Caught2', get_class($e), $e->getMessage(), $e);die;
+        }
+
+// TODO finalize / resolve
+//            if ($copyTo && !isset($this->exceptions[(int) $ch])) {
+//                $fd = fopen($copyTo, 'rb');
+//            }
+//
+    }
+
+    private function onProgress($ch, callable $notify, array $progress, array $previousProgress)
+    {
+        if (300 <= $progress['http_code'] && $progress['http_code'] < 400) {
+            return;
+        }
+        if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) {
+            $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE;
+            $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($ch), $progress['http_code'], 0, 0, false);
+        }
+        if ($previousProgress['download_content_length'] < $progress['download_content_length']) {
+            $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false);
+        }
+        if ($previousProgress['size_download'] < $progress['size_download']) {
+            $notify(STREAM_NOTIFY_PROGRESS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, (int) $progress['size_download'], (int) $progress['download_content_length'], false);
+        }
+    }
+
+    private function checkCurlResult($code)
+    {
+        if ($code != CURLM_OK && $code != CURLM_CALL_MULTI_PERFORM) {
+            throw new \RuntimeException(isset($this->multiErrors[$code])
+                ? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}"
+                : 'Unexpected cURL error: ' . $code
+            );
+        }
+    }
+}

src/Composer/Util/Http/Response.php

@@ -0,0 +1,75 @@
+<?php
+
+/*
+ * This file is part of Composer.
+ *
+ * (c) Nils Adermann <naderman@naderman.de>
+ *     Jordi Boggiano <j.boggiano@seld.be>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Composer\Util\Http;
+
+use Composer\Json\JsonFile;
+
+class Response
+{
+    private $request;
+    private $code;
+    private $headers;
+    private $body;
+
+    public function __construct(array $request, $code, array $headers, $body)
+    {
+        $this->request = $request;
+        $this->code = $code;
+        $this->headers = $headers;
+        $this->body = $body;
+    }
+
+    public function getStatusCode()
+    {
+        return $this->code;
+    }
+
+    public function getHeaders()
+    {
+        return $this->headers;
+    }
+
+    public function getHeader($name)
+    {
+        $value = null;
+        foreach ($this->headers as $header) {
+            if (preg_match('{^'.$name.':\s*(.+?)\s*$}i', $header, $match)) {
+                $value = $match[1];
+            } elseif (preg_match('{^HTTP/}i', $header)) {
+                // TODO ideally redirects would be handled in CurlDownloader/RemoteFilesystem and this becomes unnecessary
+                //
+                // In case of redirects, http_response_headers contains the headers of all responses
+                // so we reset the flag when a new response is being parsed as we are only interested in the last response
+                $value = null;
+            }
+        }
+
+        return $value;
+    }
+
+
+    public function getBody()
+    {
+        return $this->body;
+    }
+
+    public function decodeJson()
+    {
+        return JsonFile::parseJson($this->body, $this->request['url']);
+    }
+
+    public function collect()
+    {
+        $this->request = $this->code = $this->headers = $this->body = null;
+    }
+}

src/Composer/Util/HttpDownloader.php

@@ -0,0 +1,246 @@
+<?php
+
+/*
+ * This file is part of Composer.
+ *
+ * (c) Nils Adermann <naderman@naderman.de>
+ *     Jordi Boggiano <j.boggiano@seld.be>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Composer\Util;
+
+use Composer\Config;
+use Composer\IO\IOInterface;
+use Composer\Downloader\TransportException;
+use Composer\CaBundle\CaBundle;
+use Psr\Log\LoggerInterface;
+use React\Promise\Promise;
+
+/**
+ * @author Jordi Boggiano <j.boggiano@seld.be>
+ */
+class HttpDownloader
+{
+    const STATUS_QUEUED = 1;
+    const STATUS_STARTED = 2;
+    const STATUS_COMPLETED = 3;
+    const STATUS_FAILED = 4;
+
+    private $io;
+    private $config;
+    private $jobs = array();
+    private $index;
+    private $progress;
+    private $lastProgress;
+    private $disableTls = false;
+    private $curl;
+    private $rfs;
+    private $idGen = 0;
+
+    /**
+     * Constructor.
+     *
+     * @param IOInterface $io         The IO instance
+     * @param Config      $config     The config
+     * @param array       $options    The options
+     * @param bool        $disableTls
+     */
+    public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false)
+    {
+        $this->io = $io;
+
+        // Setup TLS options
+        // The cafile option can be set via config.json
+        if ($disableTls === false) {
+            $logger = $io instanceof LoggerInterface ? $io : null;
+            $this->options = StreamContextFactory::getTlsDefaults($options, $logger);
+        } else {
+            $this->disableTls = true;
+        }
+
+        // handle the other externally set options normally.
+        $this->options = array_replace_recursive($this->options, $options);
+        $this->config = $config;
+
+        if (extension_loaded('curl')) {
+            $this->curl = new Http\CurlDownloader($io, $config, $options, $disableTls);
+        }
+
+        $this->rfs = new RemoteFilesystem($io, $config, $options, $disableTls);
+    }
+
+    public function get($url, $options = array())
+    {
+        list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false), true);
+        $this->wait($job['id']);
+
+        return $this->getResponse($job['id']);
+    }
+
+    public function add($url, $options = array())
+    {
+        list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false));
+
+        return $promise;
+    }
+
+    public function copy($url, $to, $options = array())
+    {
+        list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to), true);
+        $this->wait($job['id']);
+
+        return $this->getResponse($job['id']);
+    }
+
+    public function addCopy($url, $to, $options = array())
+    {
+        list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to));
+
+        return $promise;
+    }
+
+    private function addJob($request, $sync = false)
+    {
+        $job = array(
+            'id' => $this->idGen++,
+            'status' => self::STATUS_QUEUED,
+            'request' => $request,
+            'sync' => $sync,
+        );
+
+        $curl = $this->curl;
+        $rfs = $this->rfs;
+        $io = $this->io;
+
+        $origin = $this->getOrigin($job['request']['url']);
+
+        // TODO only send http/https through curl
+        if ($curl) {
+            $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) {
+                // start job
+                $url = $job['request']['url'];
+                $options = $job['request']['options'];
+
+                $job['status'] = HttpDownloader::STATUS_STARTED;
+
+                if ($job['request']['copyTo']) {
+                    $curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']);
+                } else {
+                    $curl->download($resolve, $reject, $origin, $url, $options);
+                }
+            };
+        } else {
+            $resolver = function ($resolve, $reject) use (&$job, $rfs, $curl, $origin) {
+                // start job
+                $url = $job['request']['url'];
+                $options = $job['request']['options'];
+
+                $job['status'] = HttpDownloader::STATUS_STARTED;
+
+                if ($job['request']['copyTo']) {
+                    if ($curl) {
+                        $result = $curl->download($origin, $url, $options, $job['request']['copyTo']);
+                    } else {
+                        $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options);
+                    }
+
+                    $resolve($result);
+                } else {
+                    $body = $rfs->getContents($origin, $url, false /* TODO progress */, $options);
+                    $headers = $rfs->getLastHeaders();
+                    $response = new Http\Response($job['request'], $rfs->findStatusCode($headers), $headers, $body);
+
+                    $resolve($response);
+                }
+            };
+        }
+
+        $canceler = function () {};
+
+        $promise = new Promise($resolver, $canceler);
+        $promise->then(function ($response) use (&$job) {
+            $job['status'] = HttpDownloader::STATUS_COMPLETED;
+            $job['response'] = $response;
+            // TODO look for more jobs to start once we throttle to max X jobs
+        }, function ($e) use ($io, &$job) {
+            var_dump(__CLASS__ . __LINE__);
+            var_dump(gettype($e));
+            var_dump($e->getMessage());
+            die;
+            $job['status'] = HttpDownloader::STATUS_FAILED;
+            $job['exception'] = $e;
+        });
+        $this->jobs[$job['id']] =& $job;
+
+        return array($job, $promise);
+    }
+
+    public function wait($index = null, $progress = false)
+    {
+        while (true) {
+            if ($this->curl) {
+                $this->curl->tick();
+            }
+
+            if (null !== $index) {
+                if ($this->jobs[$index]['status'] === self::STATUS_COMPLETED || $this->jobs[$index]['status'] === self::STATUS_FAILED) {
+                    return;
+                }
+            } else {
+                $done = true;
+                foreach ($this->jobs as $job) {
+                    if (!in_array($job['status'], array(self::STATUS_COMPLETED, self::STATUS_FAILED), true)) {
+                        $done = false;
+                        break;
+                    } elseif (!$job['sync']) {
+                        unset($this->jobs[$job['id']]);
+                    }
+                }
+                if ($done) {
+                    return;
+                }
+            }
+
+            usleep(1000);
+        }
+    }
+
+    private function getResponse($index)
+    {
+        if (!isset($this->jobs[$index])) {
+            throw new \LogicException('Invalid request id');
+        }
+
+        if ($this->jobs[$index]['status'] === self::STATUS_FAILED) {
+            throw $this->jobs[$index]['exception'];
+        }
+
+        if (!isset($this->jobs[$index]['response'])) {
+            throw new \LogicException('Response not available yet, call wait() first');
+        }
+
+        $resp = $this->jobs[$index]['response'];
+
+        unset($this->jobs[$index]);
+
+        return $resp;
+    }
+
+    private function getOrigin($url)
+    {
+        $origin = parse_url($url, PHP_URL_HOST);
+
+        if ($origin === 'api.github.com') {
+            return 'github.com';
+        }
+
+        if ($origin === 'repo.packagist.org') {
+            return 'packagist.org';
+        }
+
+        return $origin ?: $url;
+    }
+}

src/Composer/Util/RemoteFilesystem.php

@@ -60,7 +60,8 @@ class RemoteFilesystem
         // Setup TLS options
         // The cafile option can be set via config.json
         if ($disableTls === false) {
-            $this->options = $this->getTlsDefaults($options);
+            $logger = $io instanceof LoggerInterface ? $io : null;
+            $this->options = StreamContextFactory::getTlsDefaults($options, $logger);
         } else {
             $this->disableTls = true;
         }
@@ -891,111 +892,6 @@ class RemoteFilesystem
         return false;
     }
 
-    /**
-     * @param array $options
-     *
-     * @return array
-     */
-    private function getTlsDefaults(array $options)
-    {
-        $ciphers = implode(':', array(
-            'ECDHE-RSA-AES128-GCM-SHA256',
-            'ECDHE-ECDSA-AES128-GCM-SHA256',
-            'ECDHE-RSA-AES256-GCM-SHA384',
-            'ECDHE-ECDSA-AES256-GCM-SHA384',
-            'DHE-RSA-AES128-GCM-SHA256',
-            'DHE-DSS-AES128-GCM-SHA256',
-            'kEDH+AESGCM',
-            'ECDHE-RSA-AES128-SHA256',
-            'ECDHE-ECDSA-AES128-SHA256',
-            'ECDHE-RSA-AES128-SHA',
-            'ECDHE-ECDSA-AES128-SHA',
-            'ECDHE-RSA-AES256-SHA384',
-            'ECDHE-ECDSA-AES256-SHA384',
-            'ECDHE-RSA-AES256-SHA',
-            'ECDHE-ECDSA-AES256-SHA',
-            'DHE-RSA-AES128-SHA256',
-            'DHE-RSA-AES128-SHA',
-            'DHE-DSS-AES128-SHA256',
-            'DHE-RSA-AES256-SHA256',
-            'DHE-DSS-AES256-SHA',
-            'DHE-RSA-AES256-SHA',
-            'AES128-GCM-SHA256',
-            'AES256-GCM-SHA384',
-            'AES128-SHA256',
-            'AES256-SHA256',
-            'AES128-SHA',
-            'AES256-SHA',
-            'AES',
-            'CAMELLIA',
-            'DES-CBC3-SHA',
-            '!aNULL',
-            '!eNULL',
-            '!EXPORT',
-            '!DES',
-            '!RC4',
-            '!MD5',
-            '!PSK',
-            '!aECDH',
-            '!EDH-DSS-DES-CBC3-SHA',
-            '!EDH-RSA-DES-CBC3-SHA',
-            '!KRB5-DES-CBC3-SHA',
-        ));
-
-        /**
-         * CN_match and SNI_server_name are only known once a URL is passed.
-         * They will be set in the getOptionsForUrl() method which receives a URL.
-         *
-         * cafile or capath can be overridden by passing in those options to constructor.
-         */
-        $defaults = array(
-            'ssl' => array(
-                'ciphers' => $ciphers,
-                'verify_peer' => true,
-                'verify_depth' => 7,
-                'SNI_enabled' => true,
-                'capture_peer_cert' => true,
-            ),
-        );
-
-        if (isset($options['ssl'])) {
-            $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']);
-        }
-
-        $caBundleLogger = $this->io instanceof LoggerInterface ? $this->io : null;
-
-        /**
-         * Attempt to find a local cafile or throw an exception if none pre-set
-         * The user may go download one if this occurs.
-         */
-        if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) {
-            $result = CaBundle::getSystemCaRootBundlePath($caBundleLogger);
-
-            if (is_dir($result)) {
-                $defaults['ssl']['capath'] = $result;
-            } else {
-                $defaults['ssl']['cafile'] = $result;
-            }
-        }
-
-        if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $caBundleLogger))) {
-            throw new TransportException('The configured cafile was not valid or could not be read.');
-        }
-
-        if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) {
-            throw new TransportException('The configured capath was not valid or could not be read.');
-        }
-
-        /**
-         * Disable TLS compression to prevent CRIME attacks where supported.
-         */
-        if (PHP_VERSION_ID >= 50413) {
-            $defaults['ssl']['disable_compression'] = true;
-        }
-
-        return $defaults;
-    }
-
     /**
      * Fetch certificate common name and fingerprint for validation of SAN.
      *

src/Composer/Util/StreamContextFactory.php

@@ -13,6 +13,8 @@
 namespace Composer\Util;
 
 use Composer\Composer;
+use Composer\CaBundle\CaBundle;
+use Psr\Log\LoggerInterface;
 
 /**
  * Allows the creation of a basic context supporting http proxy
@@ -153,6 +155,109 @@ final class StreamContextFactory
         return stream_context_create($options, $defaultParams);
     }
 
+    /**
+     * @param array $options
+     *
+     * @return array
+     */
+    public static function getTlsDefaults(array $options, LoggerInterface $logger = null)
+    {
+        $ciphers = implode(':', array(
+            'ECDHE-RSA-AES128-GCM-SHA256',
+            'ECDHE-ECDSA-AES128-GCM-SHA256',
+            'ECDHE-RSA-AES256-GCM-SHA384',
+            'ECDHE-ECDSA-AES256-GCM-SHA384',
+            'DHE-RSA-AES128-GCM-SHA256',
+            'DHE-DSS-AES128-GCM-SHA256',
+            'kEDH+AESGCM',
+            'ECDHE-RSA-AES128-SHA256',
+            'ECDHE-ECDSA-AES128-SHA256',
+            'ECDHE-RSA-AES128-SHA',
+            'ECDHE-ECDSA-AES128-SHA',
+            'ECDHE-RSA-AES256-SHA384',
+            'ECDHE-ECDSA-AES256-SHA384',
+            'ECDHE-RSA-AES256-SHA',
+            'ECDHE-ECDSA-AES256-SHA',
+            'DHE-RSA-AES128-SHA256',
+            'DHE-RSA-AES128-SHA',
+            'DHE-DSS-AES128-SHA256',
+            'DHE-RSA-AES256-SHA256',
+            'DHE-DSS-AES256-SHA',
+            'DHE-RSA-AES256-SHA',
+            'AES128-GCM-SHA256',
+            'AES256-GCM-SHA384',
+            'AES128-SHA256',
+            'AES256-SHA256',
+            'AES128-SHA',
+            'AES256-SHA',
+            'AES',
+            'CAMELLIA',
+            'DES-CBC3-SHA',
+            '!aNULL',
+            '!eNULL',
+            '!EXPORT',
+            '!DES',
+            '!RC4',
+            '!MD5',
+            '!PSK',
+            '!aECDH',
+            '!EDH-DSS-DES-CBC3-SHA',
+            '!EDH-RSA-DES-CBC3-SHA',
+            '!KRB5-DES-CBC3-SHA',
+        ));
+
+        /**
+         * CN_match and SNI_server_name are only known once a URL is passed.
+         * They will be set in the getOptionsForUrl() method which receives a URL.
+         *
+         * cafile or capath can be overridden by passing in those options to constructor.
+         */
+        $defaults = array(
+            'ssl' => array(
+                'ciphers' => $ciphers,
+                'verify_peer' => true,
+                'verify_depth' => 7,
+                'SNI_enabled' => true,
+                'capture_peer_cert' => true,
+            ),
+        );
+
+        if (isset($options['ssl'])) {
+            $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']);
+        }
+
+        /**
+         * Attempt to find a local cafile or throw an exception if none pre-set
+         * The user may go download one if this occurs.
+         */
+        if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) {
+            $result = CaBundle::getSystemCaRootBundlePath($logger);
+
+            if (is_dir($result)) {
+                $defaults['ssl']['capath'] = $result;
+            } else {
+                $defaults['ssl']['cafile'] = $result;
+            }
+        }
+
+        if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $logger))) {
+            throw new TransportException('The configured cafile was not valid or could not be read.');
+        }
+
+        if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) {
+            throw new TransportException('The configured capath was not valid or could not be read.');
+        }
+
+        /**
+         * Disable TLS compression to prevent CRIME attacks where supported.
+         */
+        if (PHP_VERSION_ID >= 50413) {
+            $defaults['ssl']['disable_compression'] = true;
+        }
+
+        return $defaults;
+    }
+
     /**
      * A bug in PHP prevents the headers from correctly being sent when a content-type header is present and
      * NOT at the end of the array