logger = $logger; $this->doctrine = $doctrine; $this->updater = $updater; $this->locker = $locker; $this->scheduler = $scheduler; $this->packageManager = $packageManager; $this->downloadManager = $downloadManager; } public function process(Job $job, SignalHandler $signal): array { $em = $this->doctrine->getEntityManager(); $id = $job->getPayload()['id']; $packageRepository = $em->getRepository(Package::class); /** @var Package $package */ $package = $packageRepository->findOneById($id); if (!$package) { $this->logger->info('Package is gone, skipping', ['id' => $id]); return ['status' => Job::STATUS_PACKAGE_GONE, 'message' => 'Package was deleted, skipped']; } $lockAcquired = $this->locker->lockPackageUpdate($id); if (!$lockAcquired) { return ['status' => Job::STATUS_RESCHEDULE, 'after' => new \DateTime('+5 seconds')]; } $this->logger->info('Updating '.$package->getName()); $config = Factory::createConfig(); $io = new BufferIO('', OutputInterface::VERBOSITY_VERY_VERBOSE, new HtmlOutputFormatter(Factory::createAdditionalStyles())); $io->loadConfiguration($config); try { $flags = 0; if ($job->getPayload()['update_equal_refs'] === true) { $flags = Updater::UPDATE_EQUAL_REFS; } if ($job->getPayload()['delete_before'] === true) { $flags = Updater::DELETE_BEFORE; } // prepare dependencies $loader = new ValidatingArrayLoader(new ArrayLoader()); // prepare repository $repository = new VcsRepository(array('url' => $package->getRepository()), $io, $config); $repository->setLoader($loader); // perform the actual update (fetch and re-scan the repository's source) $package = $this->updater->update($io, $config, $package, $repository, $flags); // github update downgraded to a git clone, this should not happen, so check through API whether the package still exists if (preg_match('{[@/]github.com[:/]([^/]+/[^/]+?)(\.git)?$}i', $package->getRepository(), $match) && 0 === strpos($repository->getDriver()->getUrl(), 'git@')) { if ($result = $this->checkForDeadGitHubPackage($package, $match, $io, $io->getOutput())) { return $result; } } } catch (\Throwable $e) { $output = $io->getOutput(); if (!$this->doctrine->getEntityManager()->isOpen()) { $this->doctrine->resetManager(); $package = $this->doctrine->getEntityManager()->getRepository(Package::class)->findOneById($package->getId()); } else { // reload the package just in case as Updater tends to merge it to a new instance $package = $packageRepository->findOneById($id); } // invalid composer data somehow, notify the owner and then mark the job failed if ($e instanceof InvalidRepositoryException) { $this->packageManager->notifyUpdateFailure($package, $e, $output); return [ 'status' => Job::STATUS_FAILED, 'message' => 'Update of '.$package->getName().' failed, invalid composer.json metadata', 'details' => '
'.$output.'
', 'exception' => $e, ]; } $found404 = false; // attempt to detect a 404/dead repository // TODO check and delete those packages with crawledAt in the far future but updatedAt in the past in a second step/job if the repo is really unreachable // probably should check for download count and a few other metrics to avoid false positives and ask humans to check the others if ($e instanceof \RuntimeException && strpos($e->getMessage(), 'remote: Repository not found')) { // git clone was attempted and says the repo is not found, that's very conclusive $found404 = true; } elseif ($e instanceof \RuntimeException && strpos($e->getMessage(), 'git@gitlab.com') && strpos($e->getMessage(), 'Please make sure you have the correct access rights')) { // git clone says we have no right on gitlab for 404s $found404 = true; } elseif ($e instanceof \RuntimeException && strpos($e->getMessage(), 'git@bitbucket.org') && strpos($e->getMessage(), 'Please make sure you have the correct access rights')) { // git clone says we have no right on bitbucket for 404s $found404 = true; } elseif ($e instanceof \RuntimeException && strpos($e->getMessage(), '@github.com/') && strpos($e->getMessage(), ' Please ask the owner to check their account')) { // git clone says account is disabled on github for private repos(?) if cloning via https $found404 = true; } elseif ($e instanceof TransportException && preg_match('{https://api.bitbucket.org/2.0/repositories/[^/]+/.+?\?fields=-project}i', $e->getMessage()) && $e->getStatusCode() == 404) { // bitbucket api root returns a 404 $found404 = true; } // github 404'ed, check through API whether the package still exists and delete if not if ($found404 && preg_match('{[@/]github.com[:/]([^/]+/[^/]+?)(\.git)?$}i', $package->getRepository(), $match)) { if ($result = $this->checkForDeadGitHubPackage($package, $match, $io, $output)) { return $result; } } // detected a 404 so mark the package as gone and prevent updates for 1y if ($found404) { $package->setCrawledAt(new \DateTime('+1 year')); $this->doctrine->getEntityManager()->flush($package); return [ 'status' => Job::STATUS_PACKAGE_GONE, 'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been marked as crawled for 1year', 'details' => '
'.$output.'
', 'exception' => $e, ]; } // Catch request timeouts e.g. gitlab.com if ($e instanceof TransportException && strpos($e->getMessage(), 'file could not be downloaded: failed to open stream: HTTP request failed!')) { return [ 'status' => Job::STATUS_FAILED, 'message' => 'Package data of '.$package->getName().' could not be downloaded. Could not reach remote VCS server. Please try again later.', 'exception' => $e ]; } // generic transport exception if ($e instanceof TransportException) { return [ 'status' => Job::STATUS_FAILED, 'message' => 'Package data of '.$package->getName().' could not be downloaded.', 'exception' => $e ]; } $this->logger->error('Failed update of '.$package->getName(), ['exception' => $e]); // unexpected error so mark the job errored throw $e; } finally { $this->locker->unlockPackageUpdate($id); } return [ 'status' => Job::STATUS_COMPLETED, 'message' => 'Update of '.$package->getName().' complete', 'details' => '
'.$io->getOutput().'
' ]; } private function checkForDeadGitHubPackage(Package $package, $match, $io, $output) { $rfs = new RemoteFilesystem($io); try { $rfs->getContents('github.com', 'https://api.github.com/repos/'.$match[1], false, ['retry-auth-failure' => false]); } catch (\Throwable $e) { if ($e instanceof TransportException && $e->getStatusCode() === 404) { try { if ( // check composer repo is visible to make sure it's not github or something else glitching $rfs->getContents('github.com', 'https://api.github.com/repos/composer/composer', false, ['retry-auth-failure' => false]) // remove packages with very low downloads and that are 404 && $this->downloadManager->getTotalDownloads($package) <= 100 ) { $name = $package->getName(); $this->packageManager->deletePackage($package); return [ 'status' => Job::STATUS_PACKAGE_DELETED, 'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been deleted', 'details' => '
'.$output.'
', 'exception' => $e, ]; } } catch (\Throwable $e) { // ignore failures here, we/github must be offline } } } } }