Phase 1 : pipeline d'ingestion Sentinel-2

- SentinelHubClient : auth OAuth2, Catalog API (cloud coverage),
  Process API avec evalscript AFAI binaire (UINT8, cloud-side)
- IngestionService : orchestration complète — job tracking,
  rejet nuages, téléchargement GeoTIFF, polygonize GDAL,
  simplification PostGIS, persistance SargassumObservation
- IngestSentinelCommand : 5 zones Antilles configurées,
  options --date et --zone, appelable via cron docker exec
- Dockerfile : ajout gdal-bin + python3-gdal
- Correction entity : paramètre immutable retiré de #[ORM\Column]

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gwadaking
2026-04-01 02:45:12 -04:00
parent e5e729c487
commit bf64340525
10 changed files with 696 additions and 12 deletions

View File

@@ -17,11 +17,14 @@
"phpdocumentor/reflection-docblock": "^6.0",
"phpstan/phpdoc-parser": "^2.3",
"symfony/asset": "7.4.*",
"symfony/cache": "7.4.*",
"symfony/console": "7.4.*",
"symfony/dotenv": "7.4.*",
"symfony/expression-language": "7.4.*",
"symfony/flex": "^2",
"symfony/framework-bundle": "7.4.*",
"symfony/http-client": "7.4.*",
"symfony/process": "7.4.*",
"symfony/property-access": "7.4.*",
"symfony/property-info": "7.4.*",
"symfony/runtime": "7.4.*",

246
backend/composer.lock generated
View File

@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "a0df5e917402537252c8bfc326e79d1c",
"content-hash": "3e4fe2591a536f0a12dd5307544b28f1",
"packages": [
{
"name": "api-platform/doctrine-common",
@@ -4640,6 +4640,185 @@
],
"time": "2026-03-30T12:55:43+00:00"
},
{
"name": "symfony/http-client",
"version": "v7.4.8",
"source": {
"type": "git",
"url": "https://github.com/symfony/http-client.git",
"reference": "01933e626c3de76bea1e22641e205e78f6a34342"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/http-client/zipball/01933e626c3de76bea1e22641e205e78f6a34342",
"reference": "01933e626c3de76bea1e22641e205e78f6a34342",
"shasum": ""
},
"require": {
"php": ">=8.2",
"psr/log": "^1|^2|^3",
"symfony/deprecation-contracts": "^2.5|^3",
"symfony/http-client-contracts": "~3.4.4|^3.5.2",
"symfony/polyfill-php83": "^1.29",
"symfony/service-contracts": "^2.5|^3"
},
"conflict": {
"amphp/amp": "<2.5",
"amphp/socket": "<1.1",
"php-http/discovery": "<1.15",
"symfony/http-foundation": "<6.4"
},
"provide": {
"php-http/async-client-implementation": "*",
"php-http/client-implementation": "*",
"psr/http-client-implementation": "1.0",
"symfony/http-client-implementation": "3.0"
},
"require-dev": {
"amphp/http-client": "^4.2.1|^5.0",
"amphp/http-tunnel": "^1.0|^2.0",
"guzzlehttp/promises": "^1.4|^2.0",
"nyholm/psr7": "^1.0",
"php-http/httplug": "^1.0|^2.0",
"psr/http-client": "^1.0",
"symfony/amphp-http-client-meta": "^1.0|^2.0",
"symfony/cache": "^6.4|^7.0|^8.0",
"symfony/dependency-injection": "^6.4|^7.0|^8.0",
"symfony/http-kernel": "^6.4|^7.0|^8.0",
"symfony/messenger": "^6.4|^7.0|^8.0",
"symfony/process": "^6.4|^7.0|^8.0",
"symfony/rate-limiter": "^6.4|^7.0|^8.0",
"symfony/stopwatch": "^6.4|^7.0|^8.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\HttpClient\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Provides powerful methods to fetch HTTP resources synchronously or asynchronously",
"homepage": "https://symfony.com",
"keywords": [
"http"
],
"support": {
"source": "https://github.com/symfony/http-client/tree/v7.4.8"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://github.com/nicolas-grekas",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2026-03-30T12:55:43+00:00"
},
{
"name": "symfony/http-client-contracts",
"version": "v3.6.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/http-client-contracts.git",
"reference": "75d7043853a42837e68111812f4d964b01e5101c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/75d7043853a42837e68111812f4d964b01e5101c",
"reference": "75d7043853a42837e68111812f4d964b01e5101c",
"shasum": ""
},
"require": {
"php": ">=8.1"
},
"type": "library",
"extra": {
"thanks": {
"url": "https://github.com/symfony/contracts",
"name": "symfony/contracts"
},
"branch-alias": {
"dev-main": "3.6-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Contracts\\HttpClient\\": ""
},
"exclude-from-classmap": [
"/Test/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Generic abstractions related to HTTP clients",
"homepage": "https://symfony.com",
"keywords": [
"abstractions",
"contracts",
"decoupling",
"interfaces",
"interoperability",
"standards"
],
"support": {
"source": "https://github.com/symfony/http-client-contracts/tree/v3.6.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2025-04-29T11:18:49+00:00"
},
{
"name": "symfony/http-foundation",
"version": "v7.4.8",
@@ -5492,6 +5671,71 @@
],
"time": "2024-09-09T11:45:10+00:00"
},
{
"name": "symfony/process",
"version": "v7.4.8",
"source": {
"type": "git",
"url": "https://github.com/symfony/process.git",
"reference": "60f19cd3badc8de688421e21e4305eba50f8089a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/process/zipball/60f19cd3badc8de688421e21e4305eba50f8089a",
"reference": "60f19cd3badc8de688421e21e4305eba50f8089a",
"shasum": ""
},
"require": {
"php": ">=8.2"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\Process\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Executes commands in sub-processes",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/process/tree/v7.4.8"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://github.com/nicolas-grekas",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2026-03-24T13:12:05+00:00"
},
{
"name": "symfony/property-access",
"version": "v7.4.8",

View File

@@ -472,7 +472,7 @@ use Symfony\Component\Config\Loader\ParamConfigurator as Param;
* },
* disallow_search_engine_index?: bool|Param, // Enabled by default when debug is enabled. // Default: true
* http_client?: bool|array{ // HTTP Client configuration
* enabled?: bool|Param, // Default: false
* enabled?: bool|Param, // Default: true
* max_host_connections?: int|Param, // The maximum number of connections to a single host.
* default_options?: array{
* headers?: array<string, mixed>,

View File

@@ -21,3 +21,8 @@ services:
# add more service definitions when explicit configuration is needed
# please note that last definitions always *replace* previous ones
App\Service\SentinelHub\SentinelHubClient:
arguments:
$clientId: '%env(SENTINEL_HUB_CLIENT_ID)%'
$clientSecret: '%env(SENTINEL_HUB_CLIENT_SECRET)%'

View File

@@ -0,0 +1,88 @@
<?php
namespace App\Command;
use App\Service\Ingestion\IngestionService;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
#[AsCommand(
name: 'app:ingest-sentinel',
description: 'Ingestion des données Sentinel-2 pour la détection de sargasses',
)]
class IngestSentinelCommand extends Command
{
// Zones d'intérêt Antilles : [minLon, minLat, maxLon, maxLat]
private const ZONES = [
'martinique' => [-61.30, 14.30, -60.70, 14.90],
'guadeloupe' => [-61.90, 15.80, -61.00, 16.60],
'sainte-lucie' => [-61.10, 13.70, -60.80, 14.20],
'barbade' => [-59.80, 13.00, -59.30, 13.40],
'saint-martin' => [-63.20, 17.80, -62.90, 18.20],
];
public function __construct(private IngestionService $ingestionService)
{
parent::__construct();
}
protected function configure(): void
{
$this
->addOption('date', 'd', InputOption::VALUE_OPTIONAL,
'Date à traiter (YYYY-MM-DD). Défaut : hier.', null)
->addOption('zone', 'z', InputOption::VALUE_OPTIONAL,
'Zone spécifique. Défaut : toutes les zones.', null)
->addOption('source', 's', InputOption::VALUE_OPTIONAL,
'Source satellite (Sentinel-2 ou Sentinel-3)', 'Sentinel-2');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$io = new SymfonyStyle($input, $output);
$dateStr = $input->getOption('date');
$date = $dateStr
? new \DateTimeImmutable($dateStr)
: new \DateTimeImmutable('yesterday');
$source = $input->getOption('source');
$zones = self::ZONES;
if ($zoneName = $input->getOption('zone')) {
if (!isset(self::ZONES[$zoneName])) {
$io->error(sprintf(
"Zone inconnue : \"%s\". Zones disponibles : %s",
$zoneName,
implode(', ', array_keys(self::ZONES))
));
return Command::FAILURE;
}
$zones = [$zoneName => self::ZONES[$zoneName]];
}
$io->title(sprintf('Ingestion %s — %s', $source, $date->format('Y-m-d')));
$errors = 0;
foreach ($zones as $name => $bbox) {
$io->section("Zone : {$name}");
try {
$this->ingestionService->ingest($bbox, $date, $source);
$io->success("OK — {$name}");
} catch (\Throwable $e) {
$io->error("ÉCHEC — {$name} : " . $e->getMessage());
$errors++;
}
}
if ($errors > 0) {
$io->warning("{$errors} zone(s) en erreur.");
}
return $errors === 0 ? Command::SUCCESS : Command::FAILURE;
}
}

View File

@@ -36,7 +36,7 @@ class SargassumObservation
#[ORM\Column(type: 'geometry', nullable: true, options: ['geometry_type' => 'POLYGON', 'srid' => 4326])]
private mixed $bbox = null;
#[ORM\Column(immutable: true)]
#[ORM\Column]
private ?\DateTimeImmutable $detectedAt = null;
#[ORM\Column(length: 50)]

View File

@@ -0,0 +1,189 @@
<?php
namespace App\Service\Ingestion;
use App\Entity\DataIngestionJob;
use App\Entity\SargassumObservation;
use App\Service\SentinelHub\SentinelHubClient;
use Doctrine\DBAL\Connection;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
class IngestionService
{
private const CLOUD_THRESHOLD = 60.0;
private const SARGASSUM_DN = 1;
private const SIMPLIFY_TOLERANCE = 0.001; // degrés (~100m à l'équateur)
private const PROCESSING_VERSION = '1.0.0';
public function __construct(
private SentinelHubClient $sentinelHub,
private EntityManagerInterface $em,
private LoggerInterface $logger,
#[Autowire('%kernel.project_dir%')] private string $projectDir,
) {}
public function ingest(array $bbox, \DateTimeImmutable $date, string $source = 'Sentinel-2'): void
{
$job = new DataIngestionJob();
$job->setSource($source);
$job->setTileId(sprintf('[%s]', implode(',', $bbox)));
$this->em->persist($job);
$this->em->flush();
$tmpDir = null;
try {
// Étape 1 : vérification couverture nuageuse via Catalog API
$cloudCoverage = $this->sentinelHub->getCloudCoverage($bbox, $date);
$this->logger->info('Cloud coverage check', ['bbox' => $bbox, 'coverage' => $cloudCoverage]);
if ($cloudCoverage > self::CLOUD_THRESHOLD) {
$job->setStatus('failed');
$job->setErrorMessage("Couverture nuageuse {$cloudCoverage}% > seuil " . self::CLOUD_THRESHOLD . '%');
$this->em->flush();
return;
}
// Étape 2 : téléchargement du raster AFAI binaire
$tmpDir = $this->projectDir . '/var/ingestion/' . uniqid('ingest_', true);
mkdir($tmpDir, 0755, true);
$tifPath = $tmpDir . '/afai.tif';
$geojsonPath = $tmpDir . '/polygons.geojson';
$metadata = $this->sentinelHub->downloadAfaiBinary($bbox, $date, $tifPath);
$this->logger->info('GeoTIFF downloaded', ['path' => $tifPath, 'tile' => $metadata['tileId']]);
// Étape 3 : vectorisation via GDAL
$this->polygonize($tifPath, $geojsonPath);
// Étape 4 : construction du MULTIPOLYGON depuis le GeoJSON
$geometry = $this->buildMultiPolygon($geojsonPath);
if ($geometry === null) {
$job->setStatus('success');
$job->setProcessedAt(new \DateTimeImmutable());
$job->setErrorMessage('Aucune sargasse détectée dans cette scène');
$this->em->flush();
$this->logger->info('No sargassum detected', ['bbox' => $bbox]);
return;
}
// Étape 5 : persistance
$observation = new SargassumObservation();
$observation->setGeometry($geometry);
$observation->setDetectedAt($date);
$observation->setSource($source);
$observation->setTileId($metadata['tileId']);
$observation->setCloudCoverage($cloudCoverage);
$observation->setConfidence(0.8);
$observation->setAfaiMean($metadata['afaiMean']);
$observation->setAfaiStd($metadata['afaiStd']);
$observation->setProcessingVersion(self::PROCESSING_VERSION);
$this->em->persist($observation);
$this->em->flush();
// Étape 6 : simplification + bbox + aire via PostGIS
$this->updateSpatialMeta((string) $observation->getId());
$job->setStatus('success');
$job->setProcessedAt(new \DateTimeImmutable());
$this->em->flush();
$this->logger->info('Ingestion OK', ['observation_id' => (string) $observation->getId()]);
} catch (\Throwable $e) {
$job->setStatus('failed');
$job->incrementRetry();
$job->setErrorMessage($e->getMessage());
$this->em->flush();
$this->logger->error('Ingestion failed', ['error' => $e->getMessage()]);
throw $e;
} finally {
if ($tmpDir !== null && is_dir($tmpDir)) {
$this->cleanup($tmpDir);
}
}
}
private function polygonize(string $tifPath, string $geojsonPath): void
{
$process = new Process([
'gdal_polygonize.py',
$tifPath,
'-f', 'GeoJSON',
'-q',
$geojsonPath,
]);
$process->setTimeout(120);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
}
private function buildMultiPolygon(string $geojsonPath): ?array
{
if (!file_exists($geojsonPath)) {
return null;
}
$data = json_decode(file_get_contents($geojsonPath), true);
if (empty($data['features'])) {
return null;
}
$polygons = [];
foreach ($data['features'] as $feature) {
if (($feature['properties']['DN'] ?? 0) !== self::SARGASSUM_DN) {
continue;
}
$geom = $feature['geometry'];
if ($geom['type'] === 'Polygon') {
$polygons[] = $geom['coordinates'];
} elseif ($geom['type'] === 'MultiPolygon') {
foreach ($geom['coordinates'] as $poly) {
$polygons[] = $poly;
}
}
}
if (empty($polygons)) {
return null;
}
return ['type' => 'MultiPolygon', 'coordinates' => $polygons];
}
private function updateSpatialMeta(string $id): void
{
$this->em->getConnection()->executeStatement(
'UPDATE sargassum_observation
SET
geometry = ST_SimplifyPreserveTopology(geometry, :tolerance),
bbox = ST_Envelope(geometry),
coverage_area = ROUND(CAST(ST_Area(geometry::geography) / 1000000 AS numeric), 2)
WHERE id = :id',
['id' => $id, 'tolerance' => self::SIMPLIFY_TOLERANCE]
);
}
private function cleanup(string $dir): void
{
foreach (glob($dir . '/*') ?: [] as $file) {
if (is_file($file)) {
unlink($file);
}
}
rmdir($dir);
}
}

View File

@@ -0,0 +1,153 @@
<?php
namespace App\Service\SentinelHub;
use Psr\Cache\CacheItemPoolInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class SentinelHubClient
{
private const AUTH_URL = 'https://services.sentinel-hub.com/oauth/token';
private const PROCESS_URL = 'https://services.sentinel-hub.com/api/v1/process';
private const CATALOG_URL = 'https://services.sentinel-hub.com/api/v1/catalog/1.0.0/search';
public function __construct(
private HttpClientInterface $httpClient,
private CacheItemPoolInterface $cache,
private string $clientId,
private string $clientSecret,
) {}
private function getToken(): string
{
$cacheKey = 'sentinel_hub_token';
$item = $this->cache->getItem($cacheKey);
if ($item->isHit()) {
return $item->get();
}
$response = $this->httpClient->request('POST', self::AUTH_URL, [
'body' => [
'grant_type' => 'client_credentials',
'client_id' => $this->clientId,
'client_secret' => $this->clientSecret,
],
]);
$token = $response->toArray()['access_token'];
$item->set($token)->expiresAfter(3500);
$this->cache->save($item);
return $token;
}
/**
* Retourne la couverture nuageuse (%) de la meilleure scène disponible
* pour le bbox et la date donnés.
*/
public function getCloudCoverage(array $bbox, \DateTimeImmutable $date): float
{
$token = $this->getToken();
$dateStr = $date->format('Y-m-d');
$response = $this->httpClient->request('POST', self::CATALOG_URL, [
'headers' => ['Authorization' => "Bearer {$token}"],
'json' => [
'bbox' => $bbox,
'datetime' => "{$dateStr}T00:00:00Z/{$dateStr}T23:59:59Z",
'collections' => ['sentinel-2-l2a'],
'limit' => 1,
'sortby' => [['field' => 'eo:cloud_cover', 'direction' => 'asc']],
],
]);
$data = $response->toArray();
if (empty($data['features'])) {
return 100.0; // aucune scène disponible → rejet
}
return (float) ($data['features'][0]['properties']['eo:cloud_cover'] ?? 100.0);
}
/**
* Télécharge un raster UINT8 binaire (1=sargasse, 0=non, 255=nuage/nodata)
* calculé via evalscript AFAI, et le sauvegarde dans $outputPath.
* Retourne les métadonnées de la scène.
*/
public function downloadAfaiBinary(array $bbox, \DateTimeImmutable $date, string $outputPath): array
{
$token = $this->getToken();
$dateStr = $date->format('Y-m-d');
$response = $this->httpClient->request('POST', self::PROCESS_URL, [
'headers' => ['Authorization' => "Bearer {$token}"],
'json' => [
'input' => [
'bounds' => [
'bbox' => $bbox,
'properties' => ['crs' => 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'],
],
'data' => [[
'type' => 'sentinel-2-l2a',
'dataFilter' => [
'timeRange' => [
'from' => "{$dateStr}T00:00:00Z",
'to' => "{$dateStr}T23:59:59Z",
],
],
]],
],
'output' => [
'width' => 666,
'height' => 1222,
'responses' => [[
'identifier' => 'default',
'format' => ['type' => 'image/tiff'],
]],
],
'evalscript' => $this->getAfaiBinaryEvalscript(),
],
]);
file_put_contents($outputPath, $response->getContent());
$headers = $response->getHeaders();
return [
'tileId' => $headers['x-process-request-id'][0] ?? uniqid('tile_'),
'afaiMean' => 0.0, // calculé en phase post-traitement si nécessaire
'afaiStd' => 0.0,
];
}
private function getAfaiBinaryEvalscript(): string
{
// Longueurs d'onde Sentinel-2 (nm) — constantes capteur
// B04 (Red) : 664.5 | B08 (NIR) : 832.8 | B11 (SWIR1) : 1613.7
// ratio = (λ_NIR - λ_RED) / (λ_SWIR1 - λ_RED)
return <<<'EVALSCRIPT'
//VERSION=3
function setup() {
return {
input: [{ bands: ["B04", "B08", "B11", "CLP"], units: "REFLECTANCE" }],
output: { bands: 1, sampleType: "UINT8" }
};
}
function evaluatePixel(sample) {
if (sample.CLP > 0.4) { return [255]; } // nodata : nuage
const lambdaRed = 664.5;
const lambdaNir = 832.8;
const lambdaSwir = 1613.7;
const ratio = (lambdaNir - lambdaRed) / (lambdaSwir - lambdaRed);
const afai = sample.B08 - sample.B04 - (sample.B11 - sample.B04) * ratio;
return [afai > 0.005 ? 1 : 0]; // 1 = sargasse détectée
}
EVALSCRIPT;
}
}