shlinkio/shlink
1
0
Fork 0
mirror of https://github.com/shlinkio/shlink.git synced 2025-12-10 11:05:50 -06:00
Code Issues Packages Projects Releases 146 Wiki Activity

Merge pull request #2308 from acelaya-forks/feature/geolocation-updates

Browse Source 
Improve how geolocation DB files are downloaded/updated
This commit is contained in:
Alejandro Celaya 2024-12-16 20:21:35 +01:00 committed by GitHub
commit d533adf7ce
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 489 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -15,13 +15,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
This option effectively replaces the old `REDIRECT_APPEND_EXTRA_PATH` option, which is now deprecated and will be removed in Shlink 5.0.0
### Changed
* * [#2281](https://github.com/shlinkio/shlink/issues/2281) Update docker image to PHP 8.4
* [#2281](https://github.com/shlinkio/shlink/issues/2281) Update docker image to PHP 8.4
* [#2124](https://github.com/shlinkio/shlink/issues/2124) Improve how Shlink decides if a GeoLite db file needs to be downloaded, and reduces the chances for API limits to be reached.
Now Shlink tracks all download attempts, and knows which of them failed and succeeded. This lets it know when was the last error or success, how many consecutive errors have happened, etc.
It also tracks now the reason for a download to be attempted, and the error that happened when one fails.
### Deprecated
* *Nothing*
### Removed
* * [#2247](https://github.com/shlinkio/shlink/issues/2247) Drop support for PHP 8.2
* [#2247](https://github.com/shlinkio/shlink/issues/2247) Drop support for PHP 8.2
### Fixed
* *Nothing*

12
module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php
View File

@ -51,6 +51,16 @@ class DownloadGeoLiteDbCommand extends Command implements GeolocationDownloadPro
return ExitCode::EXIT_WARNING;
}
if ($result === GeolocationResult::MAX_ERRORS_REACHED) {
$this->io->warning('Max consecutive errors reached. Cannot retry for a couple of days.');
return ExitCode::EXIT_WARNING;
}
if ($result === GeolocationResult::UPDATE_IN_PROGRESS) {
$this->io->warning('A geolocation db is already being downloaded by another process.');
return ExitCode::EXIT_WARNING;
}
if ($this->progressBar === null) {
$this->io->info('GeoLite2 db file is up to date.');
} else {
@ -66,7 +76,7 @@ class DownloadGeoLiteDbCommand extends Command implements GeolocationDownloadPro
private function processGeoLiteUpdateError(GeolocationDbUpdateFailedException $e, SymfonyStyle $io): int
{
$olderDbExists = $e->olderDbExists();
$olderDbExists = $e->olderDbExists;
if ($olderDbExists) {
$io->warning(

12
module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php
View File

@ -6,6 +6,7 @@ namespace ShlinkioTest\Shlink\CLI\Command\Visit;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\Attributes\TestWith;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase;
use Shlinkio\Shlink\CLI\Command\Visit\DownloadGeoLiteDbCommand;
@ -74,17 +75,18 @@ class DownloadGeoLiteDbCommandTest extends TestCase
}
#[Test]
public function warningIsPrintedWhenLicenseIsMissing(): void
#[TestWith([GeolocationResult::LICENSE_MISSING, 'It was not possible to download GeoLite2 db'])]
#[TestWith([GeolocationResult::MAX_ERRORS_REACHED, 'Max consecutive errors reached'])]
#[TestWith([GeolocationResult::UPDATE_IN_PROGRESS, 'A geolocation db is already being downloaded'])]
public function warningIsPrintedForSomeResults(GeolocationResult $result, string $expectedWarningMessage): void
{
$this->dbUpdater->expects($this->once())->method('checkDbUpdate')->withAnyParameters()->willReturn(
GeolocationResult::LICENSE_MISSING,
);
$this->dbUpdater->expects($this->once())->method('checkDbUpdate')->withAnyParameters()->willReturn($result);
$this->commandTester->execute([]);
$output = $this->commandTester->getDisplay();
$exitCode = $this->commandTester->getStatusCode();
self::assertStringContainsString('[WARNING] It was not possible to download GeoLite2 db', $output);
self::assertStringContainsString('[WARNING] ' . $expectedWarningMessage, $output);
self::assertSame(ExitCode::EXIT_WARNING, $exitCode);
}

16
module/CLI/test/Exception/GeolocationDbUpdateFailedExceptionTest.php
View File

@ -19,7 +19,7 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase
{
$e = GeolocationDbUpdateFailedException::withOlderDb($prev);
self::assertTrue($e->olderDbExists());
self::assertTrue($e->olderDbExists);
self::assertEquals(
'An error occurred while updating geolocation database, but an older DB is already present.',
$e->getMessage(),
@ -33,7 +33,7 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase
{
$e = GeolocationDbUpdateFailedException::withoutOlderDb($prev);
self::assertFalse($e->olderDbExists());
self::assertFalse($e->olderDbExists);
self::assertEquals(
'An error occurred while updating geolocation database, and an older version could not be found.',
$e->getMessage(),
@ -48,16 +48,4 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase
yield 'RuntimeException' => [new RuntimeException('prev')];
yield 'Exception' => [new Exception('prev')];
}
#[Test]
public function withInvalidEpochInOldDbBuildsException(): void
{
$e = GeolocationDbUpdateFailedException::withInvalidEpochInOldDb('foobar');
self::assertTrue($e->olderDbExists());
self::assertEquals(
'Build epoch with value "foobar" from existing geolocation database, could not be parsed to integer.',
$e->getMessage(),
);
}
}

3
module/Core/config/dependencies.config.php
View File

@ -13,7 +13,6 @@ use Shlinkio\Shlink\Core\Geolocation\GeolocationDbUpdater;
use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier;
use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface;
use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater;
use Shlinkio\Shlink\IpGeolocation\GeoLite2\GeoLite2ReaderFactory;
use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface;
use Symfony\Component\Lock;
@ -247,9 +246,9 @@ return [
GeolocationDbUpdater::class => [
DbUpdater::class,
GeoLite2ReaderFactory::class,
LOCAL_LOCK_FACTORY,
Config\Options\TrackingOptions::class,
'em',
],
Geolocation\Middleware\IpGeolocationMiddleware::class => [
IpLocationResolverInterface::class,

63
module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Geolocation.Entity.GeolocationDbUpdate.php Normal file
View File

@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace Shlinkio\Shlink\Core;
use Doctrine\DBAL\Types\Types;
use Doctrine\ORM\Mapping\Builder\ClassMetadataBuilder;
use Doctrine\ORM\Mapping\Builder\FieldBuilder;
use Doctrine\ORM\Mapping\ClassMetadata;
use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType;
use Shlinkio\Shlink\Core\Geolocation\Entity\GeolocationDbUpdateStatus;
return static function (ClassMetadata $metadata, array $emConfig): void {
$builder = new ClassMetadataBuilder($metadata);
$builder->setTable(determineTableName('geolocation_db_updates', $emConfig));
$builder->createField('id', Types::BIGINT)
->columnName('id')
->makePrimaryKey()
->generatedValue('IDENTITY')
->option('unsigned', true)
->build();
$builder->createField('dateCreated', ChronosDateTimeType::CHRONOS_DATETIME)
->columnName('date_created')
->build();
$builder->createField('dateUpdated', ChronosDateTimeType::CHRONOS_DATETIME)
->columnName('date_updated')
->nullable()
->build();
(new FieldBuilder($builder, [
'fieldName' => 'status',
'type' => Types::STRING,
'enumType' => GeolocationDbUpdateStatus::class,
]))->columnName('status')
->length(128)
->build();
fieldWithUtf8Charset($builder->createField('error', Types::STRING), $emConfig)
->columnName('error')
->length(1024)
->nullable()
->build();
fieldWithUtf8Charset($builder->createField('reason', Types::STRING), $emConfig)
->columnName('reason')
->length(1024)
->build();
fieldWithUtf8Charset($builder->createField('filesystemId', Types::STRING), $emConfig)
->columnName('filesystem_id')
->length(512)
->build();
// Index on date_updated, as we'll usually sort the query by this field
$builder->addIndex(['date_updated'], 'IDX_geolocation_date_updated');
// Index on filesystem_id, as we'll usually filter the query by this field
$builder->addIndex(['filesystem_id'], 'IDX_geolocation_status_filesystem');
};

64
module/Core/migrations/Version20241212131058.php Normal file
View File

@ -0,0 +1,64 @@
<?php
declare(strict_types=1);
namespace ShlinkMigrations;
use Doctrine\DBAL\Platforms\MySQLPlatform;
use Doctrine\DBAL\Schema\Schema;
use Doctrine\DBAL\Types\Types;
use Doctrine\Migrations\AbstractMigration;
use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType;
/**
* Create a new table to track geolocation db updates
*/
final class Version20241212131058 extends AbstractMigration
{
private const string TABLE_NAME = 'geolocation_db_updates';
public function up(Schema $schema): void
{
$this->skipIf($schema->hasTable(self::TABLE_NAME));
$table = $schema->createTable(self::TABLE_NAME);
$table->addColumn('id', Types::BIGINT, [
'unsigned' => true,
'autoincrement' => true,
'notnull' => true,
]);
$table->setPrimaryKey(['id']);
$table->addColumn('date_created', ChronosDateTimeType::CHRONOS_DATETIME, ['default' => 'CURRENT_TIMESTAMP']);
$table->addColumn('date_updated', ChronosDateTimeType::CHRONOS_DATETIME, ['default' => 'CURRENT_TIMESTAMP']);
$table->addColumn('status', Types::STRING, [
'length' => 128,
'default' => 'in-progress', // in-progress, success, error
]);
$table->addColumn('filesystem_id', Types::STRING, ['length' => 512]);
$table->addColumn('reason', Types::STRING, ['length' => 1024]);
$table->addColumn('error', Types::STRING, [
'length' => 1024,
'default' => null,
'notnull' => false,
]);
// Index on date_updated, as we'll usually sort the query by this field
$table->addIndex(['date_updated'], 'IDX_geolocation_date_updated');
// Index on filesystem_id, as we'll usually filter the query by this field
$table->addIndex(['filesystem_id'], 'IDX_geolocation_status_filesystem');
}
public function down(Schema $schema): void
{
$this->skipIf(! $schema->hasTable(self::TABLE_NAME));
$schema->dropTable(self::TABLE_NAME);
}
public function isTransactional(): bool
{
return ! ($this->connection->getDatabasePlatform() instanceof MySQLPlatform);
}
}

1
module/Core/src/EventDispatcher/UpdateGeoLiteDb.php
View File

@ -14,6 +14,7 @@ use Throwable;
use function sprintf;
/** @todo Rename to UpdateGeolocationDb */
readonly class UpdateGeoLiteDb
{
public function __construct(

40
module/Core/src/Exception/GeolocationDbUpdateFailedException.php
View File

@ -7,52 +7,28 @@ namespace Shlinkio\Shlink\Core\Exception;
use RuntimeException;
use Throwable;
use function sprintf;
class GeolocationDbUpdateFailedException extends RuntimeException implements ExceptionInterface
{
private bool $olderDbExists;
private function __construct(string $message, Throwable|null $previous = null)
private function __construct(string $message, public readonly bool $olderDbExists, Throwable|null $prev = null)
{
parent::__construct($message, previous: $previous);
parent::__construct($message, previous: $prev);
}
public static function withOlderDb(Throwable|null $prev = null): self
{
$e = new self(
return new self(
'An error occurred while updating geolocation database, but an older DB is already present.',
$prev,
olderDbExists: true,
prev: $prev,
);
$e->olderDbExists = true;
return $e;
}
public static function withoutOlderDb(Throwable|null $prev = null): self
{
$e = new self(
return new self(
'An error occurred while updating geolocation database, and an older version could not be found.',
$prev,
olderDbExists: false,
prev: $prev,
);
$e->olderDbExists = false;
return $e;
}
public static function withInvalidEpochInOldDb(mixed $buildEpoch): self
{
$e = new self(sprintf(
'Build epoch with value "%s" from existing geolocation database, could not be parsed to integer.',
$buildEpoch,
));
$e->olderDbExists = true;
return $e;
}
public function olderDbExists(): bool
{
return $this->olderDbExists;
}
}

77
module/Core/src/Geolocation/Entity/GeolocationDbUpdate.php Normal file
View File

@ -0,0 +1,77 @@
<?php
declare(strict_types=1);
namespace Shlinkio\Shlink\Core\Geolocation\Entity;
use Cake\Chronos\Chronos;
use Shlinkio\Shlink\Common\Entity\AbstractEntity;
use Shlinkio\Shlink\Core\Exception\RuntimeException;
use function stat;
class GeolocationDbUpdate extends AbstractEntity
{
private function __construct(
public readonly string $reason,
private readonly string $filesystemId,
private GeolocationDbUpdateStatus $status = GeolocationDbUpdateStatus::IN_PROGRESS,
private readonly Chronos $dateCreated = new Chronos(),
private Chronos $dateUpdated = new Chronos(),
private string|null $error = null,
) {
}
public static function withReason(string $reason): self
{
return new self($reason, self::currentFilesystemId());
}
public static function currentFilesystemId(): string
{
$system = stat(__FILE__);
if (! $system) {
throw new RuntimeException('It was not possible to resolve filesystem ID via stat function');
}
return (string) $system['dev'];
}
public function finishSuccessfully(): self
{
$this->dateUpdated = Chronos::now();
$this->status = GeolocationDbUpdateStatus::SUCCESS;
return $this;
}
public function finishWithError(string $error): self
{
$this->error = $error;
$this->dateUpdated = Chronos::now();
$this->status = GeolocationDbUpdateStatus::ERROR;
return $this;
}
/**
* @param positive-int $days
*/
public function isOlderThan(int $days): bool
{
return Chronos::now()->greaterThan($this->dateUpdated->addDays($days));
}
public function isInProgress(): bool
{
return $this->status === GeolocationDbUpdateStatus::IN_PROGRESS;
}
public function isError(): bool
{
return $this->status === GeolocationDbUpdateStatus::ERROR;
}
public function isSuccess(): bool
{
return $this->status === GeolocationDbUpdateStatus::SUCCESS;
}
}

10
module/Core/src/Geolocation/Entity/GeolocationDbUpdateStatus.php Normal file
View File

@ -0,0 +1,10 @@
<?php
namespace Shlinkio\Shlink\Core\Geolocation\Entity;
enum GeolocationDbUpdateStatus: string
{
case IN_PROGRESS = 'in-progress';
case SUCCESS = 'success';
case ERROR = 'error';
}

131
module/Core/src/Geolocation/GeolocationDbUpdater.php
View File

@ -4,36 +4,29 @@ declare(strict_types=1);
namespace Shlinkio\Shlink\Core\Geolocation;
use Cake\Chronos\Chronos;
use Closure;
use GeoIp2\Database\Reader;
use MaxMind\Db\Reader\Metadata;
use Doctrine\ORM\EntityManagerInterface;
use Shlinkio\Shlink\Core\Config\Options\TrackingOptions;
use Shlinkio\Shlink\Core\Exception\GeolocationDbUpdateFailedException;
use Shlinkio\Shlink\Core\Geolocation\Entity\GeolocationDbUpdate;
use Shlinkio\Shlink\IpGeolocation\Exception\DbUpdateException;
use Shlinkio\Shlink\IpGeolocation\Exception\MissingLicenseException;
use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface;
use Symfony\Component\Lock\LockFactory;
use Throwable;
use function is_int;
use function sprintf;
class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
readonly class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
{
private const string LOCK_NAME = 'geolocation-db-update';
/** @var Closure(): Reader */
private readonly Closure $geoLiteDbReaderFactory;
/**
* @param callable(): Reader $geoLiteDbReaderFactory
*/
public function __construct(
private readonly DbUpdaterInterface $dbUpdater,
callable $geoLiteDbReaderFactory,
private readonly LockFactory $locker,
private readonly TrackingOptions $trackingOptions,
private DbUpdaterInterface $dbUpdater,
private LockFactory $locker,
private TrackingOptions $trackingOptions,
private EntityManagerInterface $em,
private int $maxRecentAttemptsToCheck = 15, // TODO Make this configurable
) {
$this->geoLiteDbReaderFactory = $geoLiteDbReaderFactory(...);
}
/**
@ -46,6 +39,7 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
return GeolocationResult::CHECK_SKIPPED;
}
$lock = $this->locker->createLock(self::LOCK_NAME);
$lock->acquire(blocking: true);
@ -62,43 +56,88 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
private function downloadIfNeeded(
GeolocationDownloadProgressHandlerInterface|null $downloadProgressHandler,
): GeolocationResult {
if (! $this->dbUpdater->databaseFileExists()) {
return $this->downloadNewDb($downloadProgressHandler, olderDbExists: false);
$recentDownloads = $this->em->getRepository(GeolocationDbUpdate::class)->findBy(
criteria: ['filesystemId' => GeolocationDbUpdate::currentFilesystemId()],
orderBy: ['dateUpdated' => 'DESC'],
limit: $this->maxRecentAttemptsToCheck,
);
$mostRecentDownload = $recentDownloads[0] ?? null;
// If most recent attempt is in progress, skip check.
// This is a safety check in case the lock is released before the previous download has finished.
if ($mostRecentDownload?->isInProgress()) {
return GeolocationResult::UPDATE_IN_PROGRESS;
}
$meta = ($this->geoLiteDbReaderFactory)()->metadata();
if ($this->buildIsTooOld($meta)) {
return $this->downloadNewDb($downloadProgressHandler, olderDbExists: true);
$amountOfErrorsSinceLastSuccess = 0;
foreach ($recentDownloads as $recentDownload) {
// Count attempts until a success is found
if ($recentDownload->isSuccess()) {
break;
}
$amountOfErrorsSinceLastSuccess++;
}
// If max amount of consecutive errors has been reached and the most recent one is not old enough, skip download
// for 2 days to avoid hitting potential API limits in geolocation services
$lastAttemptIsError = $mostRecentDownload !== null && $mostRecentDownload->isError();
// FIXME Once max errors are reached there will be one attempt every 2 days, but it should be 15 attempts every
// 2 days. Leaving like this for simplicity for now.
$maxConsecutiveErrorsReached = $amountOfErrorsSinceLastSuccess === $this->maxRecentAttemptsToCheck;
if ($lastAttemptIsError && $maxConsecutiveErrorsReached && ! $mostRecentDownload->isOlderThan(days: 2)) {
return GeolocationResult::MAX_ERRORS_REACHED;
}
// Try to download if:
// - There are no attempts tracked
// - The database file does not exist
// - Last update errored (and implicitly, the max amount of consecutive errors has not been reached)
// - Most recent attempt is older than 30 days (and implicitly, successful)
$reasonMatch = match (true) {
$mostRecentDownload === null => [false, 'No download attempts tracked for this instance'],
! $this->dbUpdater->databaseFileExists() => [false, 'Geolocation db file does not exist'],
$lastAttemptIsError => [true, 'Max consecutive errors not reached'],
$mostRecentDownload->isOlderThan(days: 30) => [true, 'Last successful attempt is old enough'],
default => null,
};
if ($reasonMatch !== null) {
[$olderDbExists, $reason] = $reasonMatch;
return $this->downloadAndTrackUpdate($downloadProgressHandler, $olderDbExists, $reason);
}
return GeolocationResult::DB_IS_UP_TO_DATE;
}
private function buildIsTooOld(Metadata $meta): bool
{
$buildTimestamp = $this->resolveBuildTimestamp($meta);
$buildDate = Chronos::createFromTimestamp($buildTimestamp);
/**
* @throws GeolocationDbUpdateFailedException
*/
private function downloadAndTrackUpdate(
GeolocationDownloadProgressHandlerInterface|null $downloadProgressHandler,
bool $olderDbExists,
string $reason,
): GeolocationResult {
$dbUpdate = GeolocationDbUpdate::withReason($reason);
$this->em->persist($dbUpdate);
$this->em->flush();
return Chronos::now()->greaterThan($buildDate->addDays(35));
}
private function resolveBuildTimestamp(Metadata $meta): int
{
// In theory the buildEpoch should be an int, but it has been reported to come as a string.
// See https://github.com/shlinkio/shlink/issues/1002 for context
/** @var int|string $buildEpoch */
$buildEpoch = $meta->buildEpoch;
if (is_int($buildEpoch)) {
return $buildEpoch;
try {
$result = $this->downloadNewDb($downloadProgressHandler, $olderDbExists);
$dbUpdate->finishSuccessfully();
return $result;
} catch (MissingLicenseException) {
$dbUpdate->finishWithError('Geolocation license key is missing');
return GeolocationResult::LICENSE_MISSING;
} catch (GeolocationDbUpdateFailedException $e) {
$dbUpdate->finishWithError(
sprintf('%s Prev: %s', $e->getMessage(), $e->getPrevious()?->getMessage() ?? '-'),
);
throw $e;
} catch (Throwable $e) {
$dbUpdate->finishWithError(sprintf('Unknown error: %s', $e->getMessage()));
throw $e;
} finally {
$this->em->flush();
}
$intBuildEpoch = (int) $buildEpoch;
if ($buildEpoch === (string) $intBuildEpoch) {
return $intBuildEpoch;
}
throw GeolocationDbUpdateFailedException::withInvalidEpochInOldDb($buildEpoch);
}
/**
@ -116,8 +155,6 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
=> $downloadProgressHandler?->handleProgress($total, $downloaded, $olderDbExists),
);
return $olderDbExists ? GeolocationResult::DB_UPDATED : GeolocationResult::DB_CREATED;
} catch (MissingLicenseException) {
return GeolocationResult::LICENSE_MISSING;
} catch (DbUpdateException $e) {
throw $olderDbExists
? GeolocationDbUpdateFailedException::withOlderDb($e)

9
module/Core/src/Geolocation/GeolocationResult.php
View File

@ -4,9 +4,18 @@ namespace Shlinkio\Shlink\Core\Geolocation;
enum GeolocationResult
{
/** Geolocation is not relevant, so updates are skipped */
case CHECK_SKIPPED;
/** Update is skipped because max amount of consecutive errors was reached */
case MAX_ERRORS_REACHED;
/** Update was skipped because a geolocation license key was not provided */
case LICENSE_MISSING;
/** A geolocation database didn't exist and has been created */
case DB_CREATED;
/** An outdated geolocation database existed and has been updated */
case DB_UPDATED;
/** Geolocation database does not need to be updated yet */
case DB_IS_UP_TO_DATE;
/** Geolocation db update is currently in progress */
case UPDATE_IN_PROGRESS;
}

206
module/Core/test/Geolocation/GeolocationDbUpdaterTest.php
View File

@ -6,14 +6,16 @@ namespace ShlinkioTest\Shlink\Core\Geolocation;
use Cake\Chronos\Chronos;
use Closure;
use GeoIp2\Database\Reader;
use MaxMind\Db\Reader\Metadata;
use Doctrine\ORM\EntityManagerInterface;
use Doctrine\ORM\EntityRepository;
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\Attributes\Test;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase;
use RuntimeException;
use Shlinkio\Shlink\Core\Config\Options\TrackingOptions;
use Shlinkio\Shlink\Core\Exception\GeolocationDbUpdateFailedException;
use Shlinkio\Shlink\Core\Geolocation\Entity\GeolocationDbUpdate;
use Shlinkio\Shlink\Core\Geolocation\GeolocationDbUpdater;
use Shlinkio\Shlink\Core\Geolocation\GeolocationDownloadProgressHandlerInterface;
use Shlinkio\Shlink\Core\Geolocation\GeolocationResult;
@ -29,17 +31,24 @@ use function range;
class GeolocationDbUpdaterTest extends TestCase
{
private MockObject & DbUpdaterInterface $dbUpdater;
private MockObject & Reader $geoLiteDbReader;
private MockObject & Lock\LockInterface $lock;
private MockObject & EntityManagerInterface $em;
/** @var MockObject&EntityRepository<GeolocationDbUpdate> */
private MockObject & EntityRepository $repo;
/** @var GeolocationDownloadProgressHandlerInterface&object{beforeDownloadCalled: bool, handleProgressCalled: bool} */
private GeolocationDownloadProgressHandlerInterface $progressHandler;
protected function setUp(): void
{
$this->dbUpdater = $this->createMock(DbUpdaterInterface::class);
$this->geoLiteDbReader = $this->createMock(Reader::class);
$this->lock = $this->createMock(Lock\SharedLockInterface::class);
$this->lock->method('acquire')->with($this->isTrue())->willReturn(true);
$this->em = $this->createMock(EntityManagerInterface::class);
$this->repo = $this->createMock(EntityRepository::class);
$this->em->method('getRepository')->willReturn($this->repo);
$this->progressHandler = new class implements GeolocationDownloadProgressHandlerInterface {
public function __construct(
public bool $beforeDownloadCalled = false,
@ -59,6 +68,32 @@ class GeolocationDbUpdaterTest extends TestCase
};
}
#[Test]
public function properResultIsReturnedIfMostRecentUpdateIsInProgress(): void
{
$this->repo->expects($this->once())->method('findBy')->willReturn([GeolocationDbUpdate::withReason('')]);
$this->dbUpdater->expects($this->never())->method('databaseFileExists');
$result = $this->geolocationDbUpdater()->checkDbUpdate();
self::assertEquals(GeolocationResult::UPDATE_IN_PROGRESS, $result);
}
#[Test]
public function properResultIsReturnedIfMaxConsecutiveErrorsAreReached(): void
{
$this->repo->expects($this->once())->method('findBy')->willReturn([
GeolocationDbUpdate::withReason('')->finishWithError(''),
GeolocationDbUpdate::withReason('')->finishWithError(''),
GeolocationDbUpdate::withReason('')->finishWithError(''),
]);
$this->dbUpdater->expects($this->never())->method('databaseFileExists');
$result = $this->geolocationDbUpdater()->checkDbUpdate();
self::assertEquals(GeolocationResult::MAX_ERRORS_REACHED, $result);
}
#[Test]
public function properResultIsReturnedWhenLicenseIsMissing(): void
{
@ -66,7 +101,9 @@ class GeolocationDbUpdaterTest extends TestCase
$this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->willThrowException(
new MissingLicenseException(''),
);
$this->geoLiteDbReader->expects($this->never())->method('metadata');
$this->repo->expects($this->once())->method('findBy')->willReturn([
GeolocationDbUpdate::withReason('')->finishSuccessfully(),
]);
$result = $this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler);
@ -74,16 +111,19 @@ class GeolocationDbUpdaterTest extends TestCase
self::assertEquals(GeolocationResult::LICENSE_MISSING, $result);
}
#[Test]
public function exceptionIsThrownWhenOlderDbDoesNotExistAndDownloadFails(): void
#[Test, DataProvider('provideDbDoesNotExist')]
public function exceptionIsThrownWhenOlderDbDoesNotExistAndDownloadFails(Closure $setUp): void
{
$prev = new DbUpdateException('');
$this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(false);
$expectedReason = $setUp($this);
$this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with(
$this->isInstanceOf(Closure::class),
)->willThrowException($prev);
$this->geoLiteDbReader->expects($this->never())->method('metadata');
$this->em->expects($this->once())->method('persist')->with($this->callback(
fn (GeolocationDbUpdate $newUpdate): bool => $newUpdate->reason === $expectedReason,
));
try {
$this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler);
@ -91,22 +131,36 @@ class GeolocationDbUpdaterTest extends TestCase
} catch (Throwable $e) {
self::assertInstanceOf(GeolocationDbUpdateFailedException::class, $e);
self::assertSame($prev, $e->getPrevious());
self::assertFalse($e->olderDbExists());
self::assertFalse($e->olderDbExists);
self::assertTrue($this->progressHandler->beforeDownloadCalled);
}
}
public static function provideDbDoesNotExist(): iterable
{
yield 'file does not exist' => [function (self $test): string {
$test->repo->expects($test->once())->method('findBy')->willReturn([
GeolocationDbUpdate::withReason('')->finishSuccessfully(),
]);
$test->dbUpdater->expects($test->once())->method('databaseFileExists')->willReturn(false);
return 'Geolocation db file does not exist';
}];
yield 'no attempts' => [function (self $test): string {
$test->repo->expects($test->once())->method('findBy')->willReturn([]);
$test->dbUpdater->expects($test->never())->method('databaseFileExists');
return 'No download attempts tracked for this instance';
}];
}
#[Test, DataProvider('provideBigDays')]
public function exceptionIsThrownWhenOlderDbIsTooOldAndDownloadFails(int $days): void
public function exceptionIsThrownWhenOlderDbIsOldEnoughAndDownloadFails(int $days): void
{
$prev = new DbUpdateException('');
$this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true);
$this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with(
$this->isInstanceOf(Closure::class),
)->willThrowException($prev);
$this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn(
$this->buildMetaWithBuildEpoch(Chronos::now()->subDays($days)->getTimestamp()),
);
$this->repo->expects($this->once())->method('findBy')->willReturn([self::createFinishedOldUpdate($days)]);
try {
$this->geolocationDbUpdater()->checkDbUpdate();
@ -114,80 +168,115 @@ class GeolocationDbUpdaterTest extends TestCase
} catch (Throwable $e) {
self::assertInstanceOf(GeolocationDbUpdateFailedException::class, $e);
self::assertSame($prev, $e->getPrevious());
self::assertTrue($e->olderDbExists());
self::assertTrue($e->olderDbExists);
}
}
public static function provideBigDays(): iterable
{
yield [36];
yield [31];
yield [50];
yield [75];
yield [100];
}
#[Test, DataProvider('provideSmallDays')]
public function databaseIsNotUpdatedIfItIsNewEnough(string|int $buildEpoch): void
#[Test]
public function exceptionIsThrownWhenUnknownErrorHappens(): void
{
$this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with(
$this->isInstanceOf(Closure::class),
)->willThrowException(new RuntimeException('An error occurred'));
$newUpdate = null;
$this->em->expects($this->once())->method('persist')->with($this->callback(
function (GeolocationDbUpdate $u) use (&$newUpdate): bool {
$newUpdate = $u;
return true;
},
));
try {
$this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler);
self::fail();
} catch (Throwable) {
}
self::assertTrue($this->progressHandler->beforeDownloadCalled);
self::assertNotNull($newUpdate);
self::assertTrue($newUpdate->isError());
}
#[Test, DataProvider('provideNotAldEnoughDays')]
public function databaseIsNotUpdatedIfItIsNewEnough(int $days): void
{
$this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true);
$this->dbUpdater->expects($this->never())->method('downloadFreshCopy');
$this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn(
$this->buildMetaWithBuildEpoch($buildEpoch),
);
$this->repo->expects($this->once())->method('findBy')->willReturn([self::createFinishedOldUpdate($days)]);
$result = $this->geolocationDbUpdater()->checkDbUpdate();
self::assertEquals(GeolocationResult::DB_IS_UP_TO_DATE, $result);
}
public static function provideSmallDays(): iterable
public static function provideNotAldEnoughDays(): iterable
{
$generateParamsWithTimestamp = static function (int $days) {
$timestamp = Chronos::now()->subDays($days)->getTimestamp();
return [$days % 2 === 0 ? $timestamp : (string) $timestamp];
};
return array_map($generateParamsWithTimestamp, range(0, 34));
return array_map(static fn (int $value) => [$value], range(0, 29));
}
#[Test]
public function exceptionIsThrownWhenCheckingExistingDatabaseWithInvalidBuildEpoch(): void
{
$this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true);
$this->dbUpdater->expects($this->never())->method('downloadFreshCopy');
$this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn(
$this->buildMetaWithBuildEpoch('invalid'),
);
#[Test, DataProvider('provideUpdatesThatWillDownload')]
public function properResultIsReturnedWhenDownloadSucceeds(
array $updates,
GeolocationResult $expectedResult,
string $expectedReason,
): void {
$this->repo->expects($this->once())->method('findBy')->willReturn($updates);
$this->dbUpdater->method('databaseFileExists')->willReturn(true);
$this->dbUpdater->expects($this->once())->method('downloadFreshCopy');
$this->em->expects($this->once())->method('persist')->with($this->callback(
fn (GeolocationDbUpdate $newUpdate): bool => $newUpdate->reason === $expectedReason,
));
$this->expectException(GeolocationDbUpdateFailedException::class);
$this->expectExceptionMessage(
'Build epoch with value "invalid" from existing geolocation database, could not be parsed to integer.',
);
$result = $this->geolocationDbUpdater()->checkDbUpdate();
$this->geolocationDbUpdater()->checkDbUpdate();
self::assertEquals($expectedResult, $result);
}
private function buildMetaWithBuildEpoch(string|int $buildEpoch): Metadata
public static function provideUpdatesThatWillDownload(): iterable
{
return new Metadata([
'binary_format_major_version' => '',
'binary_format_minor_version' => '',
'build_epoch' => $buildEpoch,
'database_type' => '',
'languages' => '',
'description' => '',
'ip_version' => '',
'node_count' => 1,
'record_size' => 4,
]);
yield 'no updates' => [[], GeolocationResult::DB_CREATED, 'No download attempts tracked for this instance'];
yield 'old successful update' => [
[self::createFinishedOldUpdate(days: 31)],
GeolocationResult::DB_UPDATED,
'Last successful attempt is old enough',
];
yield 'not enough errors' => [
[self::createFinishedOldUpdate(days: 3, successful: false)],
GeolocationResult::DB_UPDATED,
'Max consecutive errors not reached',
];
}
public static function createFinishedOldUpdate(int $days, bool $successful = true): GeolocationDbUpdate
{
Chronos::setTestNow(Chronos::now()->subDays($days));
$update = GeolocationDbUpdate::withReason('');
if ($successful) {
$update->finishSuccessfully();
} else {
$update->finishWithError('');
}
Chronos::setTestNow();
return $update;
}
#[Test, DataProvider('provideTrackingOptions')]
public function downloadDbIsSkippedIfTrackingIsDisabled(TrackingOptions $options): void
{
$result = $this->geolocationDbUpdater($options)->checkDbUpdate();
$this->dbUpdater->expects($this->never())->method('databaseFileExists');
$this->geoLiteDbReader->expects($this->never())->method('metadata');
$this->em->expects($this->never())->method('getRepository');
$result = $this->geolocationDbUpdater($options)->checkDbUpdate();
self::assertEquals(GeolocationResult::CHECK_SKIPPED, $result);
}
@ -204,11 +293,6 @@ class GeolocationDbUpdaterTest extends TestCase
$locker = $this->createMock(Lock\LockFactory::class);
$locker->method('createLock')->with($this->isType('string'))->willReturn($this->lock);
return new GeolocationDbUpdater(
$this->dbUpdater,
fn () => $this->geoLiteDbReader,
$locker,
$options ?? new TrackingOptions(),
);
return new GeolocationDbUpdater($this->dbUpdater, $locker, $options ?? new TrackingOptions(), $this->em, 3);
}
}