First version of background jobs for updates

app/config/config_dev.yml

@@ -26,6 +26,21 @@ monolog:
         firephp:
             type:  firephp
             level: info
+        console:
+            type:   console
+            bubble: false
+            verbosity_levels:
+                VERBOSITY_VERBOSE: INFO
+                VERBOSITY_VERY_VERBOSE: DEBUG
+            channels: ["!doctrine"]
+        console_very_verbose:
+            type:   console
+            bubble: false
+            verbosity_levels:
+                VERBOSITY_VERBOSE: NOTICE
+                VERBOSITY_VERY_VERBOSE: NOTICE
+                VERBOSITY_DEBUG: DEBUG
+            channels: ["doctrine"]
 
 hwi_oauth:
     http_client:

app/config/config_prod.yml

@@ -31,6 +31,15 @@ monolog:
             level: debug
             include_stacktraces: true
 
+        console_debug:
+            type:   console
+            verbosity_levels:
+                VERBOSITY_NORMAL: EMERGENCY
+                VERBOSITY_VERBOSE: INFO
+                VERBOSITY_VERY_VERBOSE: NOTICE
+                VERBOSITY_DEBUG: DEBUG
+            formatter: packagist.console_stack_trace_line_formatter
+
 snc_redis:
     clients:
         default:

composer.json

@@ -55,7 +55,8 @@
         "ezyang/htmlpurifier": "^4.6",
         "nelmio/cors-bundle": "^1.4",
         "cebe/markdown": "^1.1",
-        "algolia/algoliasearch-client-php": "^1.18"
+        "algolia/algoliasearch-client-php": "^1.18",
+        "seld/signal-handler": "^1.1"
     },
     "_comment": ["fos user bundle 2.0.0 tag needed"],
     "require-dev": {
@@ -77,5 +78,10 @@
     "extra": {
         "symfony-app-dir": "app",
         "symfony-web-dir": "web"
+    },
+    "config": {
+        "platform": {
+            "php": "7.0.27"
+        }
     }
 }

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "b7a08f27034774dc2a4fc3be0a5cfc9b",
+    "content-hash": "2a9c7c3a98f7dfd96273ff57a9ea7c22",
     "packages": [
         {
             "name": "algolia/algoliasearch-client-php",
@@ -179,12 +179,12 @@
             "source": {
                 "type": "git",
                 "url": "https://github.com/composer/composer.git",
-                "reference": "4dc81db069d2622bfbdf889b92b181a81bd67be3"
+                "reference": "ef46a8afa4f5d845befbc7be832432c4b30d6313"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/composer/composer/zipball/4dc81db069d2622bfbdf889b92b181a81bd67be3",
-                "reference": "4dc81db069d2622bfbdf889b92b181a81bd67be3",
+                "url": "https://api.github.com/repos/composer/composer/zipball/ef46a8afa4f5d845befbc7be832432c4b30d6313",
+                "reference": "ef46a8afa4f5d845befbc7be832432c4b30d6313",
                 "shasum": ""
             },
             "require": {
@@ -248,7 +248,7 @@
                 "dependency",
                 "package"
             ],
-            "time": "2018-01-21T16:40:29+00:00"
+            "time": "2018-02-11T10:02:41+00:00"
         },
         {
             "name": "composer/semver",
@@ -402,35 +402,35 @@
         },
         {
             "name": "doctrine/annotations",
-            "version": "v1.2.7",
+            "version": "v1.4.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/annotations.git",
-                "reference": "f25c8aab83e0c3e976fd7d19875f198ccf2f7535"
+                "reference": "54cacc9b81758b14e3ce750f205a393d52339e97"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/annotations/zipball/f25c8aab83e0c3e976fd7d19875f198ccf2f7535",
-                "reference": "f25c8aab83e0c3e976fd7d19875f198ccf2f7535",
+                "url": "https://api.github.com/repos/doctrine/annotations/zipball/54cacc9b81758b14e3ce750f205a393d52339e97",
+                "reference": "54cacc9b81758b14e3ce750f205a393d52339e97",
                 "shasum": ""
             },
             "require": {
                 "doctrine/lexer": "1.*",
-                "php": ">=5.3.2"
+                "php": "^5.6 || ^7.0"
             },
             "require-dev": {
                 "doctrine/cache": "1.*",
-                "phpunit/phpunit": "4.*"
+                "phpunit/phpunit": "^5.7"
             },
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.3.x-dev"
+                    "dev-master": "1.4.x-dev"
                 }
             },
             "autoload": {
-                "psr-0": {
-                    "Doctrine\\Common\\Annotations\\": "lib/"
+                "psr-4": {
+                    "Doctrine\\Common\\Annotations\\": "lib/Doctrine/Common/Annotations"
                 }
             },
             "notification-url": "https://packagist.org/downloads/",
@@ -466,20 +466,20 @@
                 "docblock",
                 "parser"
             ],
-            "time": "2015-08-31T12:32:49+00:00"
+            "time": "2017-02-24T16:22:25+00:00"
         },
         {
             "name": "doctrine/cache",
-            "version": "v1.6.0",
+            "version": "v1.6.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/cache.git",
-                "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6"
+                "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/cache/zipball/f8af318d14bdb0eff0336795b428b547bd39ccb6",
-                "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6",
+                "url": "https://api.github.com/repos/doctrine/cache/zipball/eb152c5100571c7a45470ff2a35095ab3f3b900b",
+                "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b",
                 "shasum": ""
             },
             "require": {
@@ -536,32 +536,33 @@
                 "cache",
                 "caching"
             ],
-            "time": "2015-12-31T16:37:02+00:00"
+            "time": "2017-07-22T12:49:21+00:00"
         },
         {
             "name": "doctrine/collections",
-            "version": "v1.3.0",
+            "version": "v1.4.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/collections.git",
-                "reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a"
+                "reference": "1a4fb7e902202c33cce8c55989b945612943c2ba"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/collections/zipball/6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
-                "reference": "6c1e4eef75f310ea1b3e30945e9f06e652128b8a",
+                "url": "https://api.github.com/repos/doctrine/collections/zipball/1a4fb7e902202c33cce8c55989b945612943c2ba",
+                "reference": "1a4fb7e902202c33cce8c55989b945612943c2ba",
                 "shasum": ""
             },
             "require": {
-                "php": ">=5.3.2"
+                "php": "^5.6 || ^7.0"
             },
             "require-dev": {
-                "phpunit/phpunit": "~4.0"
+                "doctrine/coding-standard": "~0.1@dev",
+                "phpunit/phpunit": "^5.7"
             },
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.2.x-dev"
+                    "dev-master": "1.3.x-dev"
                 }
             },
             "autoload": {
@@ -602,20 +603,20 @@
                 "collections",
                 "iterator"
             ],
-            "time": "2015-04-14T22:21:58+00:00"
+            "time": "2017-01-03T10:49:41+00:00"
         },
         {
             "name": "doctrine/common",
-            "version": "v2.6.1",
+            "version": "v2.7.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/common.git",
-                "reference": "a579557bc689580c19fee4e27487a67fe60defc0"
+                "reference": "4acb8f89626baafede6ee5475bc5844096eba8a9"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/common/zipball/a579557bc689580c19fee4e27487a67fe60defc0",
-                "reference": "a579557bc689580c19fee4e27487a67fe60defc0",
+                "url": "https://api.github.com/repos/doctrine/common/zipball/4acb8f89626baafede6ee5475bc5844096eba8a9",
+                "reference": "4acb8f89626baafede6ee5475bc5844096eba8a9",
                 "shasum": ""
             },
             "require": {
@@ -624,10 +625,10 @@
                 "doctrine/collections": "1.*",
                 "doctrine/inflector": "1.*",
                 "doctrine/lexer": "1.*",
-                "php": "~5.5|~7.0"
+                "php": "~5.6|~7.0"
             },
             "require-dev": {
-                "phpunit/phpunit": "~4.8|~5.0"
+                "phpunit/phpunit": "^5.4.6"
             },
             "type": "library",
             "extra": {
@@ -675,24 +676,24 @@
                 "persistence",
                 "spl"
             ],
-            "time": "2015-12-25T13:18:31+00:00"
+            "time": "2017-07-22T08:35:12+00:00"
         },
         {
             "name": "doctrine/dbal",
-            "version": "v2.5.5",
+            "version": "v2.5.13",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/dbal.git",
-                "reference": "9f8c05cd5225a320d56d4bfdb4772f10d045a0c9"
+                "reference": "729340d8d1eec8f01bff708e12e449a3415af873"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/dbal/zipball/9f8c05cd5225a320d56d4bfdb4772f10d045a0c9",
-                "reference": "9f8c05cd5225a320d56d4bfdb4772f10d045a0c9",
+                "url": "https://api.github.com/repos/doctrine/dbal/zipball/729340d8d1eec8f01bff708e12e449a3415af873",
+                "reference": "729340d8d1eec8f01bff708e12e449a3415af873",
                 "shasum": ""
             },
             "require": {
-                "doctrine/common": ">=2.4,<2.7-dev",
+                "doctrine/common": ">=2.4,<2.8-dev",
                 "php": ">=5.3.2"
             },
             "require-dev": {
@@ -746,41 +747,45 @@
                 "persistence",
                 "queryobject"
             ],
-            "time": "2016-09-09T19:13:33+00:00"
+            "time": "2017-07-22T20:44:48+00:00"
         },
         {
             "name": "doctrine/doctrine-bundle",
-            "version": "1.6.4",
+            "version": "1.8.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/DoctrineBundle.git",
-                "reference": "dd40b0a7fb16658cda9def9786992b8df8a49be7"
+                "reference": "eb6e4fb904a459be28872765ab6e2d246aac7c87"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/DoctrineBundle/zipball/dd40b0a7fb16658cda9def9786992b8df8a49be7",
-                "reference": "dd40b0a7fb16658cda9def9786992b8df8a49be7",
+                "url": "https://api.github.com/repos/doctrine/DoctrineBundle/zipball/eb6e4fb904a459be28872765ab6e2d246aac7c87",
+                "reference": "eb6e4fb904a459be28872765ab6e2d246aac7c87",
                 "shasum": ""
             },
             "require": {
-                "doctrine/dbal": "~2.3",
-                "doctrine/doctrine-cache-bundle": "~1.0",
-                "jdorn/sql-formatter": "~1.1",
-                "php": ">=5.3.2",
-                "symfony/console": "~2.3|~3.0",
-                "symfony/dependency-injection": "~2.3|~3.0",
-                "symfony/doctrine-bridge": "~2.2|~3.0",
-                "symfony/framework-bundle": "~2.3|~3.0"
+                "doctrine/dbal": "^2.5.12",
+                "doctrine/doctrine-cache-bundle": "~1.2",
+                "jdorn/sql-formatter": "^1.2.16",
+                "php": "^5.5.9|^7.0",
+                "symfony/console": "~2.7|~3.0|~4.0",
+                "symfony/dependency-injection": "~2.7|~3.0|~4.0",
+                "symfony/doctrine-bridge": "~2.7|~3.0|~4.0",
+                "symfony/framework-bundle": "~2.7|~3.0|~4.0"
+            },
+            "conflict": {
+                "symfony/http-foundation": "<2.6"
             },
             "require-dev": {
                 "doctrine/orm": "~2.3",
-                "phpunit/phpunit": "~4",
-                "satooshi/php-coveralls": "~0.6.1",
-                "symfony/phpunit-bridge": "~2.7|~3.0",
-                "symfony/property-info": "~2.8|~3.0",
-                "symfony/validator": "~2.2|~3.0",
-                "symfony/yaml": "~2.2|~3.0",
-                "twig/twig": "~1.10"
+                "phpunit/phpunit": "^4.8.36|^5.7|^6.4",
+                "satooshi/php-coveralls": "^1.0",
+                "symfony/phpunit-bridge": "~2.7|~3.0|~4.0",
+                "symfony/property-info": "~2.8|~3.0|~4.0",
+                "symfony/validator": "~2.7|~3.0|~4.0",
+                "symfony/web-profiler-bundle": "~2.7|~3.0|~4.0",
+                "symfony/yaml": "~2.7|~3.0|~4.0",
+                "twig/twig": "~1.26|~2.0"
             },
             "suggest": {
                 "doctrine/orm": "The Doctrine ORM integration is optional in the bundle.",
@@ -789,7 +794,7 @@
             "type": "symfony-bundle",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.6.x-dev"
+                    "dev-master": "1.8.x-dev"
                 }
             },
             "autoload": {
@@ -827,27 +832,27 @@
                 "orm",
                 "persistence"
             ],
-            "time": "2016-08-10T15:35:22+00:00"
+            "time": "2017-11-24T13:09:19+00:00"
         },
         {
             "name": "doctrine/doctrine-cache-bundle",
-            "version": "1.3.0",
+            "version": "1.3.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/DoctrineCacheBundle.git",
-                "reference": "18c600a9b82f6454d2e81ca4957cdd56a1cf3504"
+                "reference": "9baecbd6bfdd1123b0cf8c1b88fee0170a84ddd1"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/DoctrineCacheBundle/zipball/18c600a9b82f6454d2e81ca4957cdd56a1cf3504",
-                "reference": "18c600a9b82f6454d2e81ca4957cdd56a1cf3504",
+                "url": "https://api.github.com/repos/doctrine/DoctrineCacheBundle/zipball/9baecbd6bfdd1123b0cf8c1b88fee0170a84ddd1",
+                "reference": "9baecbd6bfdd1123b0cf8c1b88fee0170a84ddd1",
                 "shasum": ""
             },
             "require": {
                 "doctrine/cache": "^1.4.2",
                 "doctrine/inflector": "~1.0",
                 "php": ">=5.3.2",
-                "symfony/doctrine-bridge": "~2.2|~3.0"
+                "symfony/doctrine-bridge": "~2.2|~3.0|~4.0"
             },
             "require-dev": {
                 "instaclick/coding-standard": "~1.1",
@@ -855,15 +860,15 @@
                 "instaclick/symfony2-coding-standard": "dev-remaster",
                 "phpunit/phpunit": "~4",
                 "predis/predis": "~0.8",
-                "satooshi/php-coveralls": "~0.6.1",
+                "satooshi/php-coveralls": "^1.0",
                 "squizlabs/php_codesniffer": "~1.5",
-                "symfony/console": "~2.2|~3.0",
-                "symfony/finder": "~2.2|~3.0",
-                "symfony/framework-bundle": "~2.2|~3.0",
-                "symfony/phpunit-bridge": "~2.7|~3.0",
+                "symfony/console": "~2.2|~3.0|~4.0",
+                "symfony/finder": "~2.2|~3.0|~4.0",
+                "symfony/framework-bundle": "~2.2|~3.0|~4.0",
+                "symfony/phpunit-bridge": "~2.7|~3.0|~4.0",
                 "symfony/security-acl": "~2.3|~3.0",
-                "symfony/validator": "~2.2|~3.0",
-                "symfony/yaml": "~2.2|~3.0"
+                "symfony/validator": "~2.2|~3.0|~4.0",
+                "symfony/yaml": "~2.2|~3.0|~4.0"
             },
             "suggest": {
                 "symfony/security-acl": "For using this bundle to cache ACLs"
@@ -871,7 +876,7 @@
             "type": "symfony-bundle",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.2.x-dev"
+                    "dev-master": "1.3.x-dev"
                 }
             },
             "autoload": {
@@ -915,37 +920,37 @@
                 "cache",
                 "caching"
             ],
-            "time": "2016-01-26T17:28:51+00:00"
+            "time": "2017-10-12T17:23:29+00:00"
         },
         {
             "name": "doctrine/inflector",
-            "version": "v1.1.0",
+            "version": "v1.2.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/inflector.git",
-                "reference": "90b2128806bfde671b6952ab8bea493942c1fdae"
+                "reference": "e11d84c6e018beedd929cff5220969a3c6d1d462"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/inflector/zipball/90b2128806bfde671b6952ab8bea493942c1fdae",
-                "reference": "90b2128806bfde671b6952ab8bea493942c1fdae",
+                "url": "https://api.github.com/repos/doctrine/inflector/zipball/e11d84c6e018beedd929cff5220969a3c6d1d462",
+                "reference": "e11d84c6e018beedd929cff5220969a3c6d1d462",
                 "shasum": ""
             },
             "require": {
-                "php": ">=5.3.2"
+                "php": "^7.0"
             },
             "require-dev": {
-                "phpunit/phpunit": "4.*"
+                "phpunit/phpunit": "^6.2"
             },
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.1.x-dev"
+                    "dev-master": "1.2.x-dev"
                 }
             },
             "autoload": {
-                "psr-0": {
-                    "Doctrine\\Common\\Inflector\\": "lib/"
+                "psr-4": {
+                    "Doctrine\\Common\\Inflector\\": "lib/Doctrine/Common/Inflector"
                 }
             },
             "notification-url": "https://packagist.org/downloads/",
@@ -982,7 +987,7 @@
                 "singularize",
                 "string"
             ],
-            "time": "2015-11-06T14:35:42+00:00"
+            "time": "2017-07-22T12:18:28+00:00"
         },
         {
             "name": "doctrine/instantiator",
@@ -1094,31 +1099,31 @@
         },
         {
             "name": "doctrine/orm",
-            "version": "v2.5.4",
+            "version": "v2.5.14",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/doctrine2.git",
-                "reference": "bc4ddbfb0114cb33438cc811c9a740d8aa304aab"
+                "reference": "810a7baf81462a5ddf10e8baa8cb94b6eec02754"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/doctrine2/zipball/bc4ddbfb0114cb33438cc811c9a740d8aa304aab",
-                "reference": "bc4ddbfb0114cb33438cc811c9a740d8aa304aab",
+                "url": "https://api.github.com/repos/doctrine/doctrine2/zipball/810a7baf81462a5ddf10e8baa8cb94b6eec02754",
+                "reference": "810a7baf81462a5ddf10e8baa8cb94b6eec02754",
                 "shasum": ""
             },
             "require": {
                 "doctrine/cache": "~1.4",
                 "doctrine/collections": "~1.2",
-                "doctrine/common": ">=2.5-dev,<2.7-dev",
-                "doctrine/dbal": ">=2.5-dev,<2.6-dev",
-                "doctrine/instantiator": "~1.0.1",
+                "doctrine/common": ">=2.5-dev,<2.9-dev",
+                "doctrine/dbal": ">=2.5-dev,<2.7-dev",
+                "doctrine/instantiator": "^1.0.1",
                 "ext-pdo": "*",
                 "php": ">=5.4",
-                "symfony/console": "~2.5|~3.0"
+                "symfony/console": "~2.5|~3.0|~4.0"
             },
             "require-dev": {
                 "phpunit/phpunit": "~4.0",
-                "symfony/yaml": "~2.3|~3.0"
+                "symfony/yaml": "~2.3|~3.0|~4.0"
             },
             "suggest": {
                 "symfony/yaml": "If you want to use YAML Metadata Mapping Driver"
@@ -1166,7 +1171,7 @@
                 "database",
                 "orm"
             ],
-            "time": "2016-01-05T21:34:58+00:00"
+            "time": "2017-12-17T02:57:51+00:00"
         },
         {
             "name": "ezyang/htmlpurifier",
@@ -2673,6 +2678,54 @@
             ],
             "time": "2015-10-13T18:44:15+00:00"
         },
+        {
+            "name": "seld/signal-handler",
+            "version": "1.1.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/Seldaek/signal-handler.git",
+                "reference": "86f791965478b33f76878139df78bc3f369f182c"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/Seldaek/signal-handler/zipball/86f791965478b33f76878139df78bc3f369f182c",
+                "reference": "86f791965478b33f76878139df78bc3f369f182c",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=5.4.0"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^4.8",
+                "psr/log": "^1.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Seld\\Signal\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Jordi Boggiano",
+                    "email": "j.boggiano@seld.be",
+                    "homepage": "http://seld.be"
+                }
+            ],
+            "description": "Simple unix signal handler that silently fails on windows for easy cross-platform development",
+            "keywords": [
+                "posix",
+                "sigint",
+                "signal",
+                "sigterm",
+                "unix"
+            ],
+            "time": "2016-09-14T09:32:56+00:00"
+        },
         {
             "name": "sensio/distribution-bundle",
             "version": "v5.0.9",
@@ -5668,5 +5721,8 @@
     "platform": {
         "php": ">=7.0"
     },
-    "platform-dev": []
+    "platform-dev": [],
+    "platform-overrides": {
+        "php": "7.0.27"
+    }
 }

src/Packagist/WebBundle/Command/RunWorkersCommand.php

@@ -0,0 +1,51 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Command;
+
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Filesystem\LockHandler;
+use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
+use Seld\Signal\SignalHandler;
+
+class RunWorkersCommand extends ContainerAwareCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('packagist:run-workers')
+            ->setDescription('Run worker services')
+            ->addOption('messages', null, InputOption::VALUE_OPTIONAL, 'Amount of messages to process before exiting', 5000)
+            ->addOption('worker-id', 'w', InputOption::VALUE_OPTIONAL, 'Unique worker ID', '1')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $lock = new LockHandler('packagist_run_' . $input->getOption('worker-id'));
+
+        // another dumper is still active
+        if (!$lock->lock()) {
+            if ($input->getOption('verbose')) {
+                $output->writeln('Aborting, another of the same worker is still active');
+            }
+            return;
+        }
+
+        try {
+            $logger = $this->getContainer()->get('logger');
+
+            $worker = $this->getContainer()->get('packagist.queue_worker');
+
+            $logger->notice('Worker started successfully');
+            $this->getContainer()->get('packagist.log_resetter')->reset();
+
+            $worker->processMessages((int) $input->getOption('messages'));
+
+            $logger->notice('Worker exiting successfully');
+        } finally {
+            $lock->release();
+        }
+    }
+}

src/Packagist/WebBundle/Command/UpdatePackagesCommand.php

@@ -41,7 +41,6 @@ class UpdatePackagesCommand extends ContainerAwareCommand
             ->setDefinition(array(
                 new InputOption('force', null, InputOption::VALUE_NONE, 'Force a re-crawl of all packages, or if a package name is given forces an update of all versions'),
                 new InputOption('delete-before', null, InputOption::VALUE_NONE, 'Force deletion of all versions before an update'),
-                new InputOption('notify-failures', null, InputOption::VALUE_NONE, 'Notify failures to maintainers by email'),
                 new InputOption('update-equal-refs', null, InputOption::VALUE_NONE, 'Force update of all versions even when they already exist'),
                 new InputArgument('package', InputArgument::OPTIONAL, 'Package name to update'),
             ))
@@ -60,77 +59,66 @@ class UpdatePackagesCommand extends ContainerAwareCommand
 
         $doctrine = $this->getContainer()->get('doctrine');
         $router = $this->getContainer()->get('router');
-
-        $flags = 0;
+        $deleteBefore = false;
+        $updateEqualRefs = false;
+        $randomTimes = true;
+
+        $locker = $this->getContainer()->get('locker');
+        if (!$locker->lockCommand($this->getName())) {
+            if ($verbose) {
+                $output->writeln('Aborting, another task is running already');
+            }
+            return 0;
+        }
 
         if ($package) {
             $packages = array(array('id' => $doctrine->getRepository('PackagistWebBundle:Package')->findOneByName($package)->getId()));
-            $flags = $force ? Updater::UPDATE_EQUAL_REFS : 0;
+            if ($force) {
+                $updateEqualRefs = true;
+            }
+            $randomTimes = false;
         } elseif ($force) {
             $packages = $doctrine->getManager()->getConnection()->fetchAll('SELECT id FROM package ORDER BY id ASC');
-            $flags = Updater::UPDATE_EQUAL_REFS;
+            $updateEqualRefs = true;
         } else {
             $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getStalePackages();
         }
 
         $ids = array();
         foreach ($packages as $package) {
-            $ids[] = $package['id'];
+            $ids[] = (int) $package['id'];
         }
 
         if ($input->getOption('delete-before')) {
-            $flags = Updater::DELETE_BEFORE;
-        } elseif ($input->getOption('update-equal-refs')) {
-            $flags = Updater::UPDATE_EQUAL_REFS;
+            $deleteBefore = true;
         }
-
-        $updater = $this->getContainer()->get('packagist.package_updater');
-        $start = new \DateTime();
-
-        if ($verbose && $input->getOption('notify-failures')) {
-            throw new \LogicException('Failures can not be notified in verbose mode since the output is piped to the CLI');
+        if ($input->getOption('update-equal-refs')) {
+            $updateEqualRefs = true;
         }
 
-        $input->setInteractive(false);
-        $config = Factory::createConfig();
-        $io = $verbose ? new ConsoleIO($input, $output, $this->getApplication()->getHelperSet()) : new BufferIO('');
-        $io->loadConfiguration($config);
-        $loader = new ValidatingArrayLoader(new ArrayLoader());
-        $auths = $io->getAuthentications();
+        $scheduler = $this->getContainer()->get('scheduler');
 
         while ($ids) {
-            $packages = $doctrine->getRepository('PackagistWebBundle:Package')->getPackagesWithVersions(array_splice($ids, 0, 50));
+            $idsGroup = array_splice($ids, 0, 100);
 
-            foreach ($packages as $package) {
-                if ($verbose) {
-                    $output->writeln('Importing '.$package->getRepository());
-                }
-                try {
-                    if (null === $io || $io instanceof BufferIO) {
-                        $io = new BufferIO('');
-                        $io->loadConfiguration($config);
-                    } else {
-                        foreach ($auths as $domain => $auth) {
-                            $io->setAuthentication($domain, $auth['username'], $auth['password']);
-                        }
-                    }
-                    $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config);
-                    $repository->setLoader($loader);
-                    $updater->update($io, $config, $package, $repository, $flags, $start);
-                } catch (InvalidRepositoryException $e) {
-                    $output->writeln('<error>Broken repository in '.$router->generate('view_package', array('name' => $package->getName()), true).': '.$e->getMessage().'</error>');
-                    if ($input->getOption('notify-failures')) {
-                        if (!$this->getContainer()->get('packagist.package_manager')->notifyUpdateFailure($package, $e, $io->getOutput())) {
-                            $output->writeln('<error>Failed to notify maintainers</error>');
-                        }
+            foreach ($idsGroup as $id) {
+                if ($scheduler->hasPendingUpdateJob($id, $updateEqualRefs, $deleteBefore)) {
+                    if ($verbose) {
+                        $output->writeln('Package '.$id.' already has a pending job, skipping');
                     }
-                } catch (\Exception $e) {
-                    $output->writeln('<error>Error updating '.$router->generate('view_package', array('name' => $package->getName()), true).' ['.get_class($e).']: '.$e->getMessage().' at '.$e->getFile().':'.$e->getLine().'</error>');
+                    continue;
+                }
+
+                $job = $scheduler->scheduleUpdate($id, $updateEqualRefs, $deleteBefore, $randomTimes ? new \DateTime('+'.rand(1, 1800).'seconds') : null);
+                if ($verbose) {
+                    $output->writeln('Scheduled update job '.$job->getId().' for package '.$id);
                 }
+                $doctrine->getManager()->detach($job);
             }
 
             $doctrine->getManager()->clear();
-            unset($packages);
         }
+
+        $locker->unlockCommand($this->getName());
     }
 }

src/Packagist/WebBundle/Controller/ApiController.php

@@ -21,11 +21,13 @@ use Composer\Repository\InvalidRepositoryException;
 use Composer\Repository\VcsRepository;
 use Packagist\WebBundle\Entity\Package;
 use Packagist\WebBundle\Entity\User;
+use Packagist\WebBundle\Entity\Job;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\Method;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter;
 use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\HttpFoundation\JsonResponse;
+use Symfony\Component\HttpFoundation\NotFoundHttpException;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Security\Core\Exception\AccessDeniedException;
@@ -189,6 +191,15 @@ class ApiController extends Controller
         return new JsonResponse(array('status' => 'success'), 201);
     }
 
+    /**
+     * @Route("/jobs/{id}", name="get_job", requirements={"id"="[a-f0-9]+"}, defaults={"_format" = "json"})
+     * @Method({"GET"})
+     */
+    public function getJobAction(Request $request, string $id)
+    {
+        return new JsonResponse($this->get('scheduler')->getJobStatus($id), 200);
+    }
+
     /**
      * Expects a json like:
      *
@@ -285,50 +296,20 @@ class ApiController extends Controller
             return new Response(json_encode(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)')), 404);
         }
 
-        // don't die if this takes a while
-        set_time_limit(3600);
-
         // put both updating the database and scanning the repository in a transaction
         $em = $this->get('doctrine.orm.entity_manager');
-        $updater = $this->get('packagist.package_updater');
-        $config = Factory::createConfig();
-        $io = new BufferIO('', OutputInterface::VERBOSITY_VERY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles()));
-        $io->loadConfiguration($config);
-
-        try {
-            /** @var Package $package */
-            foreach ($packages as $package) {
-                $em->transactional(function($em) use ($package, $updater, $io, $config) {
-                    // prepare dependencies
-                    $loader = new ValidatingArrayLoader(new ArrayLoader());
-
-                    // prepare repository
-                    $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config);
-                    $repository->setLoader($loader);
-
-                    // perform the actual update (fetch and re-scan the repository's source)
-                    $updater->update($io, $config, $package, $repository);
-
-                    // update the package entity
-                    $package->setAutoUpdated(true);
-                    $em->flush($package);
-                });
-            }
-        } catch (\Exception $e) {
-            if ($e instanceof InvalidRepositoryException) {
-                $this->get('packagist.package_manager')->notifyUpdateFailure($package, $e, $io->getOutput());
-            }
+        $jobs = [];
 
-            $this->get('logger')->error('Failed update of '.$package->getName(), ['exception' => $e]);
+        /** @var Package $package */
+        foreach ($packages as $package) {
+            $package->setAutoUpdated(true);
+            $em->flush($package);
 
-            return new Response(json_encode(array(
-                'status' => 'error',
-                'message' => '['.get_class($e).'] '.$e->getMessage(),
-                'details' => '<pre>'.$io->getOutput().'</pre>'
-            )), 400);
+            $job = $this->get('scheduler')->scheduleUpdate($package, $updateEqualRefs);
+            $jobs[] = $job->getId();
         }
 
-        return new JsonResponse(array('status' => 'success'), 202);
+        return new JsonResponse(['status' => 'success', 'jobs' => $jobs], 202);
     }
 
     /**

src/Packagist/WebBundle/Controller/PackageController.php

@@ -554,55 +554,28 @@ class PackageController extends Controller
         $update = $req->request->get('update', $req->query->get('update'));
         $autoUpdated = $req->request->get('autoUpdated', $req->query->get('autoUpdated'));
         $updateEqualRefs = $req->request->get('updateAll', $req->query->get('updateAll'));
-        $showOutput = $req->request->get('showOutput', $req->query->get('showOutput', false));
 
         $user = $this->getUser() ?: $doctrine
             ->getRepository('PackagistWebBundle:User')
             ->findOneBy(array('username' => $username, 'apiToken' => $apiToken));
 
         if (!$user) {
-            return new Response(json_encode(array('status' => 'error', 'message' => 'Invalid credentials',)), 403);
+            return new JsonResponse(['status' => 'error', 'message' => 'Invalid credentials'], 403);
         }
 
         if ($package->getMaintainers()->contains($user) || $this->isGranted('ROLE_UPDATE_PACKAGES')) {
-            $req->getSession()->save();
-
             if (null !== $autoUpdated) {
-                $package->setAutoUpdated((Boolean) $autoUpdated);
+                $package->setAutoUpdated((bool) $autoUpdated);
                 $doctrine->getManager()->flush();
             }
 
             if ($update) {
-                set_time_limit(3600);
-                $updater = $this->get('packagist.package_updater');
-
-                $io = new BufferIO('', OutputInterface::VERBOSITY_VERY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles()));
-                $config = Factory::createConfig();
-                $io->loadConfiguration($config);
-                $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config);
-                $loader = new ValidatingArrayLoader(new ArrayLoader());
-                $repository->setLoader($loader);
-
-                try {
-                    $updater->update($io, $config, $package, $repository, $updateEqualRefs ? Updater::UPDATE_EQUAL_REFS : 0);
-                } catch (\Exception $e) {
-                    return new JsonResponse([
-                        'status' => 'error',
-                        'message' => '['.get_class($e).'] '.$e->getMessage(),
-                        'details' => '<pre>'.$io->getOutput().'</pre>',
-                    ], 400);
-                }
+                $job = $this->get('scheduler')->scheduleUpdate($package, $updateEqualRefs);
 
-                if ($showOutput) {
-                    return new JsonResponse([
-                        'status' => 'error',
-                        'message' => 'Update successful',
-                        'details' => '<pre>'.$io->getOutput().'</pre>',
-                    ], 400);
-                }
+                return new JsonResponse(['status' => 'success', 'job' => $job->getId()], 202);
             }
 
-            return new Response('{"status": "success"}', 202);
+            return new JsonResponse(['status' => 'success'], 202);
         }
 
         return new JsonResponse(array('status' => 'error', 'message' => 'Could not find a package that matches this request (does user maintain the package?)',), 404);

src/Packagist/WebBundle/Entity/Job.php

@@ -0,0 +1,190 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Entity;
+
+use Doctrine\ORM\Mapping as ORM;
+use DateTimeInterface;
+
+/**
+ * @ORM\Entity(repositoryClass="Packagist\WebBundle\Entity\JobRepository")
+ * @ORM\Table(
+ *     name="job",
+ *     indexes={
+ *         @ORM\Index(name="type_idx",columns={"type"}),
+ *         @ORM\Index(name="status_idx",columns={"status"}),
+ *         @ORM\Index(name="execute_dt_idx",columns={"executeAfter"}),
+ *         @ORM\Index(name="creation_idx",columns={"createdAt"}),
+ *         @ORM\Index(name="completion_idx",columns={"completedAt"}),
+ *         @ORM\Index(name="started_idx",columns={"startedAt"}),
+ *         @ORM\Index(name="package_id_idx",columns={"packageId"})
+ *     }
+ * )
+ */
+class Job
+{
+    const STATUS_QUEUED = 'queued';
+    const STATUS_STARTED = 'started';
+    const STATUS_COMPLETED = 'completed';
+    const STATUS_FAILED = 'failed'; // failed in an expected/correct way
+    const STATUS_ERRORED = 'errored'; // unexpected failure
+    const STATUS_TIMEOUT = 'timeout'; // job was marked timed out
+    const STATUS_RESCHEDULE = 'reschedule';
+
+    /**
+     * @ORM\Id
+     * @ORM\Column(type="string")
+     */
+    private $id;
+
+    /**
+     * @ORM\Column(type="string")
+     */
+    private $type;
+
+    /**
+     * @ORM\Column(type="json_array")
+     */
+    private $payload;
+
+    /**
+     * One of queued, started, completed, failed
+     *
+     * @ORM\Column(type="string")
+     */
+    private $status = self::STATUS_QUEUED;
+
+    /**
+     * @ORM\Column(type="json_array", nullable=true)
+     */
+    private $result;
+
+    /**
+     * @ORM\Column(type="datetime")
+     */
+    private $createdAt;
+
+    /**
+     * @ORM\Column(type="datetime", nullable=true)
+     */
+    private $startedAt;
+
+    /**
+     * @ORM\Column(type="datetime", nullable=true)
+     */
+    private $completedAt;
+
+    /**
+     * @ORM\Column(type="datetime", nullable=true)
+     */
+    private $executeAfter;
+
+    /**
+     * @ORM\Column(type="integer", nullable=true)
+     */
+    private $packageId;
+
+    public function start()
+    {
+        $this->startedAt = new \DateTime();
+        $this->status = self::STATUS_STARTED;
+    }
+
+    public function complete(array $result)
+    {
+        $this->result = $result;
+        $this->completedAt = new \DateTime();
+        $this->status = $result['status'];
+    }
+
+    public function reschedule(\DateTimeInterface $when)
+    {
+        $this->status = self::STATUS_QUEUED;
+        $this->startedAt = null;
+        $this->setExecuteAfter($when);
+    }
+
+    public function setId(string $id)
+    {
+        $this->id = $id;
+    }
+
+    public function getId(): string
+    {
+        return $this->id;
+    }
+
+    public function setPackageId(int $packageId)
+    {
+        $this->packageId = $packageId;
+    }
+
+    public function getPackageId()
+    {
+        return $this->packageId;
+    }
+
+    public function setType(string $type)
+    {
+        $this->type = $type;
+    }
+
+    public function getType(): string
+    {
+        return $this->type;
+    }
+
+    public function setPayload(array $payload)
+    {
+        $this->payload = $payload;
+    }
+
+    public function getPayload(): array
+    {
+        return $this->payload;
+    }
+
+    public function setStatus(string $status)
+    {
+        $this->status = $status;
+    }
+
+    public function getStatus(): string
+    {
+        return $this->status;
+    }
+
+    public function getResult()
+    {
+        return $this->result;
+    }
+
+    public function setCreatedAt(DateTimeInterface $createdAt)
+    {
+        $this->createdAt = $createdAt;
+    }
+
+    public function getCreatedAt(): DateTimeInterface
+    {
+        return $this->createdAt;
+    }
+
+    public function getStartedAt(): DateTimeInterface
+    {
+        return $this->startedAt;
+    }
+
+    public function setExecuteAfter(DateTimeInterface $executeAfter)
+    {
+        $this->executeAfter = $executeAfter;
+    }
+
+    public function getExecuteAfter()
+    {
+        return $this->executeAfter;
+    }
+
+    public function getCompletedAt()
+    {
+        return $this->completedAt;
+    }
+}

src/Packagist/WebBundle/Entity/JobRepository.php

@@ -0,0 +1,44 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Entity;
+
+use Doctrine\ORM\EntityRepository;
+
+class JobRepository extends EntityRepository
+{
+    public function start(string $jobId): bool
+    {
+        $conn = $this->getEntityManager()->getConnection();
+
+        return 1 === $conn->executeUpdate('UPDATE job SET status = :status, startedAt = :now WHERE id = :id AND startedAt IS NULL', [
+            'id' => $jobId,
+            'status' => Job::STATUS_STARTED,
+            'now' => date('Y-m-d H:i:s'),
+        ]);
+    }
+
+    public function markTimedOutJobs()
+    {
+        $conn = $this->getEntityManager()->getConnection();
+
+        $conn->executeUpdate('UPDATE job SET status = :newstatus WHERE status = :status AND startedAt < :timeout', [
+            'status' => Job::STATUS_STARTED,
+            'newstatus' => Job::STATUS_TIMEOUT,
+            'timeout' => date('Y-m-d H:i:s', strtotime('-30 minutes')),
+        ]);
+    }
+
+    public function getScheduledJobIds(): \Generator
+    {
+        $conn = $this->getEntityManager()->getConnection();
+
+        $stmt = $conn->executeQuery('SELECT id FROM job WHERE status = :status AND (executeAfter IS NULL OR executeAfter <= :now) ORDER BY createdAt ASC', [
+            'status' => Job::STATUS_QUEUED,
+            'now' => date('Y-m-d H:i:s'),
+        ]);
+
+        while ($row = $stmt->fetch(\PDO::FETCH_COLUMN)) {
+            yield $row;
+        }
+    }
+}

src/Packagist/WebBundle/Model/PackageManager.php

@@ -12,7 +12,7 @@
 
 namespace Packagist\WebBundle\Model;
 
-use Doctrine\ORM\EntityManager;
+use Symfony\Bridge\Doctrine\RegistryInterface;
 use Packagist\WebBundle\Entity\Package;
 use Psr\Log\LoggerInterface;
 
@@ -21,15 +21,15 @@ use Psr\Log\LoggerInterface;
  */
 class PackageManager
 {
-    protected $em;
+    protected $doctrine;
     protected $mailer;
     protected $twig;
     protected $logger;
     protected $options;
 
-    public function __construct(EntityManager $em, \Swift_Mailer $mailer, \Twig_Environment $twig, LoggerInterface $logger, array $options)
+    public function __construct(RegistryInterface $doctrine, \Swift_Mailer $mailer, \Twig_Environment $twig, LoggerInterface $logger, array $options)
     {
-        $this->em = $em;
+        $this->doctrine = $doctrine;
         $this->mailer = $mailer;
         $this->twig = $twig;
         $this->logger = $logger;
@@ -71,7 +71,7 @@ class PackageManager
             }
 
             $package->setUpdateFailureNotified(true);
-            $this->em->flush();
+            $this->doctrine->getEntityManager()->flush();
         }
 
         return true;

src/Packagist/WebBundle/Resources/config/services.yml

@@ -103,7 +103,7 @@ services:
     packagist.package_manager:
         class: Packagist\WebBundle\Model\PackageManager
         arguments:
-            - '@doctrine.orm.entity_manager'
+            - '@doctrine'
             - '@mailer'
             - '@twig'
             - '@logger'
@@ -135,5 +135,36 @@ services:
         class: AlgoliaSearch\Client
         arguments: ['%algolia.app_id%', '%algolia.admin_key%']
 
+    packagist.queue_worker:
+        class: Packagist\WebBundle\Service\QueueWorker
+        arguments:
+            - "@packagist.log_resetter"
+            - "@snc_redis.default_client"
+            - "@doctrine"
+            - "@logger"
+            - 'package:updates': '@updater_worker'
+
+    scheduler:
+        class: Packagist\WebBundle\Service\Scheduler
+        arguments: ["@snc_redis.default_client", "@doctrine"]
+
+    locker:
+        class: Packagist\WebBundle\Service\Locker
+        arguments: ["@doctrine"]
+
+    updater_worker:
+        class: Packagist\WebBundle\Service\UpdaterWorker
+        arguments: ["@logger", "@doctrine", "@packagist.package_updater", "@locker", "@scheduler", "@packagist.package_manager"]
+
+    packagist.log_resetter:
+        class: Packagist\WebBundle\Service\LogResetter
+        arguments: ['@service_container', '%fingers_crossed_handlers%']
+
+    packagist.console_stack_trace_line_formatter:
+        class: Symfony\Bridge\Monolog\Formatter\ConsoleFormatter
+        arguments: []
+        calls:
+          - [includeStacktraces, [true]]
+
 parameters:
     security.exception_listener.class: Packagist\WebBundle\Security\ExceptionListener

src/Packagist/WebBundle/Resources/public/js/view.js

@@ -70,6 +70,7 @@
 
     function forceUpdatePackage(e, updateAll) {
         var submit = $('input[type=submit]', '.package .force-update'), data;
+        var showOutput = e && e.shiftKey;
         if (e) {
             e.preventDefault();
         }
@@ -80,20 +81,56 @@
         if (updateAll) {
             data.push({name: 'updateAll', value: '1'});
         }
-        if (e && e.shiftKey) {
-            data.push({name: 'showOutput', value: '1'});
-        }
+
         $.ajax({
             url: $('.package .force-update').attr('action'),
             dataType: 'json',
             cache: false,
             data: data,
             type: $('.package .force-update').attr('method'),
-            success: function () {
-                document.location.reload(true);
+            success: function (data) {
+                if (data.job) {
+                    var checkJobStatus = function () {
+                        $.ajax({
+                            url: '/jobs/' + data.job,
+                            cache: false,
+                            success: function (data) {
+                                if (data.status == 'completed' || data.status == 'errored' || data.status == 'failed') {
+                                    humane.remove();
+
+                                    var message = data.message;
+                                    var details = '';
+                                    if (data.status !== 'completed') {
+                                        message += ' [' + data.exceptionClass + '] ' + data.exceptionMsg;
+                                        details = data.details;
+                                    } else if (showOutput) {
+                                        details = data.details;
+                                    }
+
+                                    if (details) {
+                                        humane.log([message, details], {timeout: 0, clickToClose: false});
+                                    } else {
+                                        humane.log(message, {timeout: 2, clickToClose: true});
+                                        setTimeout(function () {
+                                            document.location.reload(true);
+                                        }, 700);
+                                    }
+
+                                    submit.removeClass('loading');
+
+                                    return;
+                                }
+
+                                setTimeout(checkJobStatus, 1000);
+                            }
+                        });
+                    };
+
+                    setTimeout(checkJobStatus, 1000);
+                }
             },
             context: $('.package .force-update')[0]
-        }).complete(function () { submit.removeClass('loading'); });
+        });
         submit.addClass('loading');
     }
     $('.package .force-update').on('submit', forceUpdatePackage);

src/Packagist/WebBundle/Service/Locker.php

@@ -0,0 +1,54 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Symfony\Bridge\Doctrine\RegistryInterface;
+
+class Locker
+{
+    private $doctrine;
+
+    public function __construct(RegistryInterface $doctrine)
+    {
+        $this->doctrine = $doctrine;
+    }
+
+    public function lockPackageUpdate(int $packageId, int $timeout = 0)
+    {
+        $this->getConn()->connect('master');
+
+        return (bool) $this->getConn()->fetchColumn('SELECT GET_LOCK(:id, :timeout)', ['id' => 'package_update_'.$packageId, 'timeout' => $timeout]);
+    }
+
+    public function unlockPackageUpdate(int $packageId)
+    {
+        $this->getConn()->connect('master');
+
+        $this->getConn()->fetchColumn('SELECT RELEASE_LOCK(:id)', ['id' => 'package_update_'.$packageId]);
+    }
+
+    public function lockCommand(string $command, int $timeout = 0)
+    {
+        $this->getConn()->connect('master');
+
+        return (bool) $this->getConn()->fetchColumn(
+            'SELECT GET_LOCK(:id, :timeout)',
+            ['id' => $command, 'timeout' => $timeout]
+        );
+    }
+
+    public function unlockCommand(string $command)
+    {
+        $this->getConn()->connect('master');
+
+        $this->getConn()->fetchColumn(
+            'SELECT RELEASE_LOCK(:id)',
+            ['id' => $command]
+        );
+    }
+
+    private function getConn()
+    {
+        return $this->doctrine->getManager()->getConnection();
+    }
+}

src/Packagist/WebBundle/Service/LogResetter.php

@@ -0,0 +1,32 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Symfony\Component\DependencyInjection\ContainerInterface;
+use Monolog\Handler\FingersCrossedHandler;
+
+class LogResetter
+{
+    private $handlers;
+
+    public function __construct(ContainerInterface $container, array $fingersCrossedHandlerNames)
+    {
+        $this->handlers = [];
+
+        foreach ($fingersCrossedHandlerNames as $name) {
+            $handler = $container->get('monolog.handler.'.$name);
+            if (!$handler instanceof FingersCrossedHandler) {
+                throw new \RuntimeException('Misconfiguration: '.$name.' given as a fingers_crossed handler type but '.get_class($handler).' was found');
+            }
+
+            $this->handlers[] = $handler;
+        }
+    }
+
+    public function reset()
+    {
+        foreach ($this->handlers as $handler) {
+            $handler->clear();
+        }
+    }
+}

src/Packagist/WebBundle/Service/QueueWorker.php

@@ -0,0 +1,169 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Predis\Client as Redis;
+use Psr\Log\LoggerInterface;
+use Symfony\Bridge\Doctrine\RegistryInterface;
+use Packagist\WebBundle\Entity\Job;
+use Seld\Signal\SignalHandler;
+use Packagist\WebBundle\Service\LogResetter;
+
+class QueueWorker
+{
+    private $logResetter;
+    private $redis;
+    private $logger;
+    /** @var RegistryInterface */
+    private $doctrine;
+    private $jobWorkers;
+    private $processedJobs = 0;
+
+    public function __construct(LogResetter $logResetter, Redis $redis, RegistryInterface $doctrine, LoggerInterface $logger, array $jobWorkers)
+    {
+        $this->logResetter = $logResetter;
+        $this->redis = $redis;
+        $this->logger = $logger;
+        $this->doctrine = $doctrine;
+        $this->jobWorkers = $jobWorkers;
+    }
+
+    /**
+     * @param string|int $minPriority
+     */
+    public function processMessages(int $count)
+    {
+        $signal = SignalHandler::create(null, $this->logger);
+
+        $this->logger->info('Waiting for new messages');
+
+        $this->doctrine->getEntityManager()->getRepository(Job::class)->markTimedOutJobs();
+        $nextScheduledJobCheck = $this->checkForScheduledJobs($signal);
+
+        while ($this->processedJobs++ < $count) {
+            if ($signal->isTriggered()) {
+                $this->logger->debug('Signal received, aborting');
+                break;
+            }
+
+            if ($nextScheduledJobCheck <= time()) {
+                $nextScheduledJobCheck = $this->checkForScheduledJobs($signal);
+            }
+
+            $result = $this->redis->brpop('jobs', 10);
+            if (!$result) {
+                $this->logger->debug('No message in queue');
+                continue;
+            }
+
+            $jobId = $result[1];
+            $this->process($jobId, $signal);
+        }
+    }
+
+    private function checkForScheduledJobs(SignalHandler $signal): int
+    {
+        $em = $this->doctrine->getEntityManager();
+        $repo = $em->getRepository(Job::class);
+
+        foreach ($repo->getScheduledJobIds() as $jobId) {
+            if ($this->process($jobId, $signal)) {
+                $this->processedJobs++;
+            }
+        }
+
+        // check for scheduled jobs every 30 sec at least
+        return time() + 30;
+    }
+
+    /**
+     * Calls the configured processor with the job and a callback that must be called to mark the job as processed
+     */
+    private function process(string $jobId, SignalHandler $signal): bool
+    {
+        $em = $this->doctrine->getEntityManager();
+        $repo = $em->getRepository(Job::class);
+        if (!$repo->start($jobId)) {
+            // race condition, some other worker caught the job first, aborting
+            return false;
+        }
+
+        $job = $repo->findOneById($jobId);
+
+        $this->logger->pushProcessor(function ($record) use ($job) {
+            $record['extra']['job-id'] = $job->getId();
+
+            return $record;
+        });
+
+        $processor = $this->jobWorkers[$job->getType()];
+
+        // clears/resets all fingers-crossed handlers to avoid dumping info messages that happened between two job executions
+        $this->logResetter->reset();
+
+        $this->logger->debug('Processing ' . $job->getType() . ' job', ['job' => $job->getPayload()]);
+
+        try {
+            $result = $processor->process($job, $signal);
+        } catch (\Throwable $e) {
+            $result = [
+                'status' => Job::STATUS_ERRORED,
+                'message' => 'An unexpected failure occurred',
+                'exception' => $e,
+            ];
+        }
+
+        if ($result['status'] === Job::STATUS_RESCHEDULE) {
+            $job->reschedule($result['after']);
+            $em->flush($job);
+
+            // reset logger
+            $this->logResetter->reset();
+            $this->logger->popProcessor();
+
+            return true;
+        }
+
+        if (!isset($result['message']) || !isset($result['status'])) {
+            throw new \LogicException('$result must be an array with at least status and message keys');
+        }
+
+        if (!in_array($result['status'], [Job::STATUS_COMPLETED, Job::STATUS_FAILED, Job::STATUS_ERRORED], true)) {
+            throw new \LogicException('$result[\'status\'] must be one of '.Job::STATUS_COMPLETED.' or '.Job::STATUS_FAILED.', '.$result['status'].' given');
+        }
+
+        if (isset($result['exception'])) {
+            $result['exceptionMsg'] = $result['exception']->getMessage();
+            $result['exceptionClass'] = get_class($result['exception']);
+        }
+
+        // If an exception is thrown during a transaction the EntityManager is closed
+        // and we won't be able to update the job or handle future jobs
+        if (!$this->doctrine->getEntityManager()->isOpen()) {
+            $this->doctrine->resetManager();
+            $em = $this->doctrine->getEntityManager();
+            $repo = $em->getRepository(Job::class);
+        }
+
+        $job = $repo->findOneById($jobId);
+        $job->complete($result);
+
+        $this->redis->setex('job-'.$job->getId(), 600, json_encode($result));
+
+        $em->flush($job);
+        $em->clear();
+
+        if ($result['status'] === Job::STATUS_FAILED) {
+            $this->logger->warning('Job '.$job->getId().' failed', $result);
+        } elseif ($result['status'] === Job::STATUS_ERRORED) {
+            $this->logger->error('Job '.$job->getId().' errored', $result);
+        }
+
+        // clears/resets all fingers-crossed handlers so that if one triggers it doesn't dump the entire debug log for all processed
+        $this->logResetter->reset();
+
+        $this->logger->popProcessor();
+
+        return true;
+    }
+}

src/Packagist/WebBundle/Service/Scheduler.php

@@ -0,0 +1,114 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Doctrine\Common\Persistence\ManagerRegistry;
+use Predis\Client as RedisClient;
+use Packagist\WebBundle\Entity\Package;
+use Packagist\WebBundle\Entity\Organization;
+use Packagist\WebBundle\Entity\Job;
+
+class Scheduler
+{
+    /** @var ManagerRegistry */
+    private $doctrine;
+    private $redis;
+
+    public function __construct(RedisClient $redis, ManagerRegistry $doctrine)
+    {
+        $this->doctrine = $doctrine;
+        $this->redis = $redis;
+    }
+
+    public function scheduleUpdate($packageOrId, $updateEqualRefs = false, $deleteBefore = false, $executeAfter = null): Job
+    {
+        if ($packageOrId instanceof Package) {
+            $packageOrId = $packageOrId->getId();
+        } elseif (!is_int($packageOrId)) {
+            throw new \UnexpectedValueException('Expected Package instance or int package id');
+        }
+
+        return $this->createJob('package:updates', ['id' => $packageOrId, 'update_equal_refs' => $updateEqualRefs, 'delete_before' => $deleteBefore], $packageOrId, $executeAfter);
+    }
+
+    public function hasPendingUpdateJob(int $packageId, $updateEqualRefs = false, $deleteBefore = false): bool
+    {
+        $result = $this->doctrine->getManager()->getConnection()->fetchAssoc('SELECT payload FROM job WHERE packageId = :package AND status = :status', [
+            'package' => $packageId,
+            'status' => Job::STATUS_QUEUED,
+        ]);
+
+        if ($result) {
+            $payload = json_decode($result['payload'], true);
+            if ($payload['update_equal_refs'] === $updateEqualRefs && $payload['delete_before'] === $deleteBefore) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @return array [status => x, message => y]
+     */
+    public function getJobStatus(string $jobId): array
+    {
+        $data = $this->redis->get('job-'.$jobId);
+
+        if ($data) {
+            return json_decode($data, true);
+        }
+
+        return ['status' => 'running', 'message' => ''];
+    }
+
+    /**
+     * @param  Job[]   $jobs
+     * @return array[]
+     */
+    public function getJobsStatus(array $jobs): array
+    {
+        $results = [];
+
+        foreach ($jobs as $job) {
+            $jobId = $job->getId();
+            $data = $this->redis->get('job-'.$jobId);
+
+            if ($data) {
+                $results[$jobId] = json_decode($data, true);
+            } else {
+                $results[$jobId] = ['status' => $job->getStatus()];
+            }
+        }
+
+        return $results;
+    }
+
+    private function createJob(string $type, array $payload, $packageId = null, $executeAfter = null): Job
+    {
+        $jobId = bin2hex(random_bytes(20));
+
+        $job = new Job();
+        $job->setId($jobId);
+        $job->setType($type);
+        $job->setPayload($payload);
+        $job->setCreatedAt(new \DateTime());
+        if ($packageId) {
+            $job->setPackageId($packageId);
+        }
+        if ($executeAfter instanceof \DateTimeInterface) {
+            $job->setExecuteAfter($executeAfter);
+        }
+
+        $em = $this->doctrine->getManager();
+        $em->persist($job);
+        $em->flush();
+
+        // trigger immediately if not scheduled for later
+        if (!$job->getExecuteAfter()) {
+            $this->redis->lpush('jobs', $job->getId());
+        }
+
+        return $job;
+    }
+}

src/Packagist/WebBundle/Service/UpdaterWorker.php

@@ -0,0 +1,154 @@
+<?php declare(strict_types=1);
+
+namespace Packagist\WebBundle\Service;
+
+use Packagist\WebBundle\Service\Scheduler;
+use Psr\Log\LoggerInterface;
+use Composer\Package\Loader\ArrayLoader;
+use Composer\Package\Loader\ValidatingArrayLoader;
+use Symfony\Bridge\Doctrine\RegistryInterface;
+use Composer\Console\HtmlOutputFormatter;
+use Composer\Repository\InvalidRepositoryException;
+use Composer\Repository\VcsRepository;
+use Composer\IO\ConsoleIO;
+use Composer\IO\BufferIO;
+use Symfony\Component\Console\Input\StringInput;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Helper\HelperSet;
+use Monolog\Handler\StreamHandler;
+use Packagist\WebBundle\Entity\Package;
+use Packagist\WebBundle\Package\Updater;
+use Packagist\WebBundle\Entity\Job;
+use Packagist\WebBundle\Model\PackageManager;
+use Seld\Signal\SignalHandler;
+use Composer\Factory;
+
+class UpdaterWorker
+{
+    private $logger;
+    private $doctrine;
+    private $updater;
+    private $locker;
+    /** @var Scheduler */
+    private $scheduler;
+    private $packageManager;
+
+    public function __construct(
+        LoggerInterface $logger,
+        RegistryInterface $doctrine,
+        Updater $updater,
+        Locker $locker,
+        Scheduler $scheduler,
+        PackageManager $packageManager
+    ) {
+        $this->logger = $logger;
+        $this->doctrine = $doctrine;
+        $this->updater = $updater;
+        $this->locker = $locker;
+        $this->scheduler = $scheduler;
+        $this->packageManager = $packageManager;
+    }
+
+    public function process(Job $job, SignalHandler $signal): array
+    {
+        $em = $this->doctrine->getEntityManager();
+        $id = $job->getPayload()['id'];
+        $packageRepository = $em->getRepository(Package::class);
+        /** @var Package $package */
+        $package = $packageRepository->findOneById($id);
+        if (!$package) {
+            $this->logger->info('Package is gone, skipping', ['id' => $id]);
+
+            return ['status' => Job::STATUS_FAILED, 'message' => 'Package is gone, skipped'];
+        }
+
+        $lockAcquired = $this->locker->lockPackageUpdate($id);
+        if (!$lockAcquired) {
+            return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds')];
+        }
+
+        $this->logger->info('Updating '.$package->getName());
+
+        $config = Factory::createConfig();
+        $io = new BufferIO('', OutputInterface::VERBOSITY_VERY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles()));
+        $io->loadConfiguration($config);
+
+        try {
+            $flags = 0;
+            if ($job->getPayload()['update_equal_refs'] === true) {
+                $flags = Updater::UPDATE_EQUAL_REFS;
+            }
+            if ($job->getPayload()['delete_before'] === true) {
+                $flags = Updater::DELETE_BEFORE;
+            }
+
+            $em->transactional(function($em) use ($package, $io, $config, $flags) {
+                // prepare dependencies
+                $loader = new ValidatingArrayLoader(new ArrayLoader());
+
+                // prepare repository
+                $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config);
+                $repository->setLoader($loader);
+
+                // perform the actual update (fetch and re-scan the repository's source)
+                $this->updater->update($io, $config, $package, $repository, $flags);
+
+                // update the package entity
+                $package->setAutoUpdated(true);
+                $em->flush($package);
+            });
+        } catch (\Composer\Downloader\TransportException $e) {
+            // Catch request timeouts e.g. gitlab.com
+            if (strpos($e->getMessage(), 'file could not be downloaded: failed to open stream: HTTP request failed!')) {
+                return [
+                    'status' => Job::STATUS_FAILED,
+                    'message' => 'Package data could not be downloaded. Could not reach remote VCS server. Please try again later.',
+                    'exception' => $e
+                ];
+            }
+
+            return [
+                'status' => Job::STATUS_FAILED,
+                'message' => 'Package data could not be downloaded.',
+                'exception' => $e
+            ];
+        } catch (InvalidRepositoryException $e) {
+            if ($package->getRepoType() === 'package') {
+                $this->logger->warning('Composer Invalid Repository Exception.', ['exception' => $e]);
+
+                return [
+                    'status' => Job::STATUS_FAILED,
+                    'message' => explode("\n", $e->getMessage())[0],
+                    'exception' => $e
+                ];
+            }
+
+            throw $e;
+        } catch (\Throwable $e) {
+            if ($e instanceof InvalidRepositoryException) {
+                $this->packageManager->notifyUpdateFailure($package, $e, $io->getOutput());
+            } else {
+                // TODO implement this somehow to keep track of failures and maybe delete packages
+                // or at least mark abandoned if they keep failing for X times/days?
+                // $this->packageManager->logHardFailure($package);
+            }
+
+            $this->logger->error('Failed update of '.$package->getName(), ['exception' => $e]);
+
+            return [
+                'status' => Job::STATUS_FAILED,
+                'message' => 'Update failed',
+                'details' => '<pre>'.$io->getOutput().'</pre>',
+                'exception' => $e,
+            ];
+        } finally {
+            $this->locker->unlockPackageUpdate($package->getId());
+        }
+
+        return [
+            'status' => Job::STATUS_COMPLETED,
+            'message' => 'Update complete',
+            'details' => '<pre>'.$io->getOutput().'</pre>'
+        ];
+    }
+}

src/Packagist/WebBundle/Tests/Model/PackageManagerTest.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace Packagist\WebBundle\Tests\Model;
+
+use Packagist\WebBundle\Entity\Package;
+use Packagist\WebBundle\Model\PackageManager;
+
+class PackageManagerTest extends \PHPUnit_Framework_TestCase
+{
+    public function testNotifyFailure()
+    {
+        $this->markTestSkipped('Do it!');
+
+        $client = self::createClient();
+
+        $package = new Package;
+        $package->setRepository($url);
+
+        $user = new User;
+        $user->addPackages($package);
+
+        $repo = $this->getMockBuilder('Packagist\WebBundle\Entity\UserRepository')->disableOriginalConstructor()->getMock();
+        $em = $this->getMockBuilder('Doctrine\ORM\EntityManager')->disableOriginalConstructor()->getMock();
+        $updater = $this->getMockBuilder('Packagist\WebBundle\Package\Updater')->disableOriginalConstructor()->getMock();
+
+        $repo->expects($this->once())
+            ->method('findOneBy')
+            ->with($this->equalTo(array('username' => 'test', 'apiToken' => 'token')))
+            ->will($this->returnValue($user));
+
+        static::$kernel->getContainer()->set('packagist.user_repository', $repo);
+        static::$kernel->getContainer()->set('doctrine.orm.entity_manager', $em);
+        static::$kernel->getContainer()->set('packagist.package_updater', $updater);
+
+        $payload = json_encode(array('repository' => array('url' => 'git://github.com/composer/composer')));
+        $client->request('POST', '/api/github?username=test&apiToken=token', array('payload' => $payload));
+        $this->assertEquals(202, $client->getResponse()->getStatusCode());
+    }
+}