Add a new metadata change feed for mirroring v2 metadata

src/Packagist/WebBundle/Controller/PackageController.php

@@ -77,6 +77,8 @@ class PackageController extends Controller
     }
 
     /**
+     * Deprecated legacy change API for metadata v1
+     *
      * @Route("/packages/updated.json", name="updated_packages", defaults={"_format"="json"}, methods={"GET"})
      */
     public function updatedSinceAction(Request $req)
@@ -102,6 +104,46 @@ class PackageController extends Controller
         return new JsonResponse(['packageNames' => $names, 'timestamp' => $lastDumpTime]);
     }
 
+    /**
+     * @Route("/metadata/changes.json", name="metadata_changes", defaults={"_format"="json"}, methods={"GET"})
+     */
+    public function metadataChangesAction(Request $req)
+    {
+        $redis = $this->get('snc_redis.default_client');
+
+        $topDump = $redis->zrevrange('metadata-dumps', 0, 0, ['WITHSCORES' => true]) ?: ['foo' => 0];
+        $topDelete = $redis->zrevrange('metadata-deletes', 0, 0, ['WITHSCORES' => true]) ?: ['foo' => 0];
+        $oldestSyncPoint = (int) $redis->get('metadata-oldest') ?: 15850612240000;
+        $now = max((int) current($topDump), (int) current($topDelete)) + 1;
+
+        $since = $req->query->getInt('since');
+        if (!$since || $since < 15850612240000) {
+            return new JsonResponse(['error' => 'Invalid or missing "since" query parameter, make sure you store the timestamp (as `microtime(true) * 10000`) at the initial point you started mirroring, then send that to begin receiving changes, e.g. '.$this->generateUrl('metadata_changes', ['since' => $now], UrlGeneratorInterface::ABSOLUTE_URL).' for example.'], 400);
+        }
+        if ($since < $oldestSyncPoint) {
+            return new JsonResponse(['actions' => ['type' => 'resync', 'time' => $now, 'package' => '*'], 'timestamp' => $now]);
+        }
+
+        // fetch changes from $since (inclusive) up to $now (non inclusive so -1)
+        $dumps = $redis->zrevrangebyscore('metadata-dumps', $now - 1, $since, ['WITHSCORES' => true]);
+        $deletes = $redis->zrevrangebyscore('metadata-deletes', $now - 1, $since, ['WITHSCORES' => true]);
+
+        $actions = [];
+        foreach ($dumps as $package => $time) {
+            $actions[$package] = ['type' => 'update', 'package' => $package, 'time' => (int) $time];
+        }
+        foreach ($deletes as $package => $time) {
+            // if a package is dumped then deleted then dumped again because it gets re-added, we want to keep the update action
+            // but if it is deleted and marked as dumped within 10 seconds of the deletion, it probably was a race condition between
+            // dumped job and deletion, so let's replace it by a delete job anyway
+            if (!isset($actions[$package]) || $actions[$package]['time'] < $time - (10 * 10000)) {
+                $actions[$package] = ['type' => 'delete', 'package' => $package, 'time' => (int) $time];
+            }
+        }
+
+        return new JsonResponse(['actions' => array_values($actions), 'timestamp' => $now]);
+    }
+
     /**
      * @Template()
      * @Route("/packages/submit", name="submit")

src/Packagist/WebBundle/Model/PackageManager.php

@@ -113,6 +113,8 @@ class PackageManager
         } catch (\Predis\Connection\ConnectionException $e) {
         }
 
+        $this->redis->zadd('metadata-deletes', round(microtime(true)*10000), strtolower($packageName));
+
         // attempt search index cleanup
         try {
             $indexName = $this->algoliaIndexName;

src/Packagist/WebBundle/Package/SymlinkDumper.php

@@ -859,8 +859,15 @@ class SymlinkDumper
             return;
         }
 
+        // get time before file_put_contents to be sure we return a time at least as old as the filemtime, if it is older it doesn't matter
+        $timestamp = round(microtime(true)*10000);
         file_put_contents($path.'.tmp', $contents);
         rename($path.'.tmp', $path);
+
+        if (!preg_match('{/([^/]+/[^/]+)(~dev)?\.json}', $path, $match)) {
+            throw new \LogicException('Could not match package name from '.$path);
+        }
+        $this->redis->zadd('metadata-dumps', $timestamp, $match[1]);
     }
 
     private function writeFileNonAtomic($path, $contents)