Release 2.9.6

.woodpecker/prod_economics.yml

@@ -26,3 +26,12 @@ steps:
         - itkdev-docker-compose-server run --rm phpfpm bin/console doctrine:migrations:migrate --no-interaction
         - itkdev-docker-compose-server run --rm phpfpm bin/console messenger:setup-transports
         - itkdev-docker-compose-server run --rm phpfpm bin/console cache:clear
+      cron:
+        # @TODO This should run with Symfony scheduler, see App\Command\QueueSyncCommand
+        leantime-sync:
+          minute: '22'
+          hour: '*'
+          day: '*'
+          month: '*'
+          weekday: '*'
+          job: 'itkdev-docker-compose-server exec phpfpm bin/console app:sync'

.woodpecker/prod_itk_economics.yml

@@ -26,3 +26,12 @@ steps:
         - itkdev-docker-compose-server run --rm phpfpm bin/console doctrine:migrations:migrate --no-interaction
         - itkdev-docker-compose-server run --rm phpfpm bin/console messenger:setup-transports
         - itkdev-docker-compose-server run --rm phpfpm bin/console cache:clear
+      cron:
+        # @TODO This should run with Symfony scheduler, see App\Command\QueueSyncCommand
+        leantime-sync:
+          minute: '22'
+          hour: '*'
+          day: '*'
+          month: '*'
+          weekday: '*'
+          job: 'itkdev-docker-compose-server exec phpfpm bin/console app:sync'

.woodpecker/stg.yml

@@ -35,3 +35,12 @@ steps:
         - docker compose run --rm node npm clean-install
         - docker compose run --rm node npm run build
         - docker compose run --rm node rm -rf node_modules
+      cron:
+        # @TODO This should run with Symfony scheduler, see App\Command\QueueSyncCommand
+        leantime-sync:
+          minute: '22'
+          hour: '*'
+          day: '*'
+          month: '*'
+          weekday: '*'
+          job: 'itkdev-docker-compose-server exec phpfpm bin/console app:sync'

CHANGELOG.md

@@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.9.6] - 2025-09-25
+
+* [PR-240](https://github.com/itk-dev/economics/pull/240)
+  Fixes and optimizations for `app:sync` command
+
+## [2.9.5] - 2025-07-10
+
+* [PR-235](https://github.com/itk-dev/economics/pull/235)
+  Various minor changes and request timeout increase
+
 ## [2.9.4] - 2025-07-10
 
 * [PR-232](https://github.com/itk-dev/economics/pull/232)

config/services.yaml

@@ -74,3 +74,7 @@ services:
     App\MessageHandler\SynchronizeHandler:
         arguments:
             $monitoringUrl: '%env(string:SYNC_MONITORING_URL)%'
+
+    App\Command\SyncCommand:
+        arguments:
+            $monitoringUrl: '%env(string:SYNC_MONITORING_URL)%'

src/Command/SyncCommand.php

@@ -2,16 +2,25 @@
 
 namespace App\Command;
 
+use App\Entity\Project;
+use App\Entity\SynchronizationJob;
+use App\Enum\SynchronizationStatusEnum;
+use App\Enum\SynchronizationStepEnum;
 use App\Exception\EconomicsException;
 use App\Exception\UnsupportedDataProviderException;
 use App\Repository\DataProviderRepository;
 use App\Repository\ProjectRepository;
+use App\Repository\SynchronizationJobRepository;
 use App\Service\DataSynchronizationService;
+use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\Persistence\ManagerRegistry;
+use Psr\Log\LoggerInterface;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
 
 #[AsCommand(
     name: 'app:sync',
@@ -23,6 +32,12 @@ public function __construct(
         private readonly ProjectRepository $projectRepository,
         private readonly DataProviderRepository $dataProviderRepository,
         private readonly DataSynchronizationService $dataSynchronizationService,
+        private readonly SynchronizationJobRepository $synchronizationJobRepository,
+        private readonly EntityManagerInterface $entityManager,
+        private readonly ManagerRegistry $managerRegistry,
+        private readonly HttpClientInterface $client,
+        private readonly LoggerInterface $logger,
+        private readonly string $monitoringUrl,
     ) {
         parent::__construct($this->getName());
     }
@@ -39,80 +54,144 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $io = new SymfonyStyle($input, $output);
 
-        $io->info('Processing projects');
-
         $dataProviders = $this->dataProviderRepository->findBy(['enabled' => true]);
 
-        foreach ($dataProviders as $dataProvider) {
-            $this->dataSynchronizationService->syncProjects(function ($i, $length) use ($io) {
-                if (0 == $i) {
-                    $io->progressStart($length);
-                } elseif ($i >= $length - 1) {
-                    $io->progressFinish();
-                } else {
-                    $io->progressAdvance();
-                }
-            }, $dataProvider);
-        }
+        $job = new SynchronizationJob();
+        $job->setStarted(new \DateTime());
+        $job->addMessage('Synchronization started');
+        $job->setProgress(0);
+        $job->setStatus(SynchronizationStatusEnum::RUNNING);
+        $this->synchronizationJobRepository->save($job, true);
+
+        try {
+            $io->info('Processing projects');
+            $job->setStep(SynchronizationStepEnum::PROJECTS);
+
+            $this->entityManager->flush();
 
-        $io->info('Processing accounts');
+            foreach ($dataProviders as $dataProvider) {
+                $job->addMessage('Processing projects from '.$dataProvider->getName());
+                $this->dataSynchronizationService->syncProjects(function ($i, $length) use ($io, $job) {
+                    $this->setProgress($i, $length, $io, $job);
+                }, $dataProvider);
+            }
+
+            $io->info('Processing accounts');
+            $this->setStep(SynchronizationStepEnum::ACCOUNTS, $job);
+
+            foreach ($dataProviders as $dataProvider) {
+                $this->dataSynchronizationService->syncAccounts(function ($i, $length) use ($io, $job) {
+                    $this->setProgress($i, $length, $io, $job);
+                }, $dataProvider);
+            }
 
-        foreach ($dataProviders as $dataProvider) {
-            $this->dataSynchronizationService->syncAccounts(function ($i, $length) use ($io) {
-                if (0 == $i) {
-                    $io->progressStart($length);
-                } elseif ($i >= $length - 1) {
-                    $io->progressFinish();
-                } else {
-                    $io->progressAdvance();
+            $io->info('Processing issues');
+            $this->setStep(SynchronizationStepEnum::ISSUES, $job);
+
+            foreach ($dataProviders as $dataProvider) {
+                $projects = $this->projectRepository->findBy(['include' => true, 'dataProvider' => $dataProvider]);
+
+                foreach ($projects as $project) {
+                    $io->writeln("Processing issues for {$project->getName()}");
+
+                    $this->dataSynchronizationService->syncIssuesForProject($project->getId(), $dataProvider, function ($i, $length) use ($io, $job) {
+                        $this->setProgress($i, $length, $io, $job);
+                    });
+
+                    $io->writeln('');
                 }
-            }, $dataProvider);
-        }
+            }
 
-        $io->info('Processing issues');
+            $io->info('Processing worklogs');
+            $this->setStep(SynchronizationStepEnum::WORKLOGS, $job);
 
-        foreach ($dataProviders as $dataProvider) {
-            $projects = $this->projectRepository->findBy(['include' => true, 'dataProvider' => $dataProvider]);
+            foreach ($dataProviders as $dataProvider) {
+                $projects = $this->projectRepository->findBy(['include' => true, 'dataProvider' => $dataProvider]);
 
-            foreach ($projects as $project) {
-                $io->writeln("Processing issues for {$project->getName()}");
+                /** @var Project $project */
+                foreach ($projects as $project) {
+                    $io->writeln("Processing worklogs for {$project->getName()}");
 
-                $this->dataSynchronizationService->syncIssuesForProject($project->getId(), $dataProvider, function ($i, $length) use ($io) {
-                    if (0 == $i) {
-                        $io->progressStart($length);
-                    } elseif ($i >= $length - 1) {
-                        $io->progressFinish();
-                    } else {
-                        $io->progressAdvance();
+                    $projectId = $project->getId();
+
+                    if (null === $projectId) {
+                        throw new \RuntimeException('Project id is null');
                     }
-                });
 
-                $io->writeln('');
+                    $this->dataSynchronizationService->syncWorklogsForProject($projectId, $dataProvider, function ($i, $length) use ($io, $job) {
+                        $this->setProgress($i, $length, $io, $job);
+                    });
+
+                    $io->writeln('');
+                }
             }
+
+            $job = $this->getJob($job);
+            $job->setStatus(SynchronizationStatusEnum::DONE);
+            $job->setEnded(new \DateTime());
+            $this->entityManager->flush();
+
+            // Call monitoring url if defined.
+            if ('' !== $this->monitoringUrl) {
+                try {
+                    $this->client->request('GET', $this->monitoringUrl);
+                } catch (\Throwable $e) {
+                    $this->logger->error('Error calling monitoringUrl: '.$e->getMessage());
+                }
+            }
+
+            return Command::SUCCESS;
+        } catch (\Exception $exception) {
+            $this->managerRegistry->resetManager();
+
+            $job = $this->getJob($job);
+            $job->setStatus(SynchronizationStatusEnum::ERROR);
+            $job->setEnded(new \DateTime());
+            $job->addMessage($exception->getMessage());
+
+            $this->entityManager->flush();
+
+            $io->error($exception->getMessage());
+
+            return Command::FAILURE;
         }
+    }
 
-        $io->info('Processing worklogs');
+    private function setProgress(int $i, int $length, SymfonyStyle $io, SynchronizationJob $job): void
+    {
+        $job = $this->getJob($job);
+
+        if (0 == $i) {
+            $io->progressStart($length);
+            $job->setProgress($i);
+        } elseif ($i >= $length - 1) {
+            $io->progressFinish();
+            $job->setProgress(100);
+        } else {
+            $length = 0 < $length ? $length : 1;
+            $io->progressAdvance();
+            $job->setProgress(intdiv($i * 100, $length));
+        }
+    }
 
-        foreach ($dataProviders as $dataProvider) {
-            $projects = $this->projectRepository->findBy(['include' => true, 'dataProvider' => $dataProvider]);
+    private function setStep(SynchronizationStepEnum $step, SynchronizationJob $job): void
+    {
+        $job = $this->getJob($job);
 
-            foreach ($projects as $project) {
-                $io->writeln("Processing worklogs for {$project->getName()}");
+        $job->setStep($step);
+        $this->entityManager->flush();
+    }
 
-                $this->dataSynchronizationService->syncWorklogsForProject($project->getId(), $dataProvider, function ($i, $length) use ($io) {
-                    if (0 == $i) {
-                        $io->progressStart($length);
-                    } elseif ($i >= $length - 1) {
-                        $io->progressFinish();
-                    } else {
-                        $io->progressAdvance();
-                    }
-                });
+    private function getJob(SynchronizationJob $job): SynchronizationJob
+    {
+        if (!$this->entityManager->contains($job)) {
+            $job = $this->synchronizationJobRepository->find($job->getId());
 
-                $io->writeln('');
+            if (null === $job) {
+                throw new \RuntimeException('Job not found');
             }
         }
 
-        return Command::SUCCESS;
+        return $job;
     }
 }

src/Controller/SynchronizationJobController.php

@@ -44,26 +44,29 @@ public function status(SynchronizationJobRepository $synchronizationJobRepositor
     #[IsGranted('ROLE_ADMIN')]
     public function sync(SynchronizationJobRepository $synchronizationJobRepository, MessageBusInterface $bus): Response
     {
-        $latestJob = $synchronizationJobRepository->getLatestJob();
-
-        if (null !== $latestJob && in_array($latestJob->getStatus(), [SynchronizationStatusEnum::NOT_STARTED, SynchronizationStatusEnum::RUNNING])) {
-            return new JsonResponse(['message' => 'existing job'], Response::HTTP_CONFLICT);
-        }
-
-        $job = new SynchronizationJob();
-        $job->setStatus(SynchronizationStatusEnum::NOT_STARTED);
-        $synchronizationJobRepository->save($job, true);
-
-        $jobId = $job->getId();
-
-        if (null === $jobId) {
-            throw new \Exception('Job id not found');
-        }
-
-        $message = new SynchronizeMessage($jobId);
-
-        $bus->dispatch($message);
+        return new JsonResponse([], Response::HTTP_SERVICE_UNAVAILABLE);
 
-        return new JsonResponse([], 200);
+        //        @TODO re-enable when sync is fully re-factored
+        //        $latestJob = $synchronizationJobRepository->getLatestJob();
+        //
+        //        if (null !== $latestJob && in_array($latestJob->getStatus(), [SynchronizationStatusEnum::NOT_STARTED, SynchronizationStatusEnum::RUNNING])) {
+        //            return new JsonResponse(['message' => 'existing job'], Response::HTTP_CONFLICT);
+        //        }
+        //
+        //        $job = new SynchronizationJob();
+        //        $job->setStatus(SynchronizationStatusEnum::NOT_STARTED);
+        //        $synchronizationJobRepository->save($job, true);
+        //
+        //        $jobId = $job->getId();
+        //
+        //        if (null === $jobId) {
+        //            throw new \Exception('Job id not found');
+        //        }
+        //
+        //        $message = new SynchronizeMessage($jobId);
+        //
+        //        $bus->dispatch($message);
+        //
+        //        return new JsonResponse([], 200);
     }
 }

src/Entity/SynchronizationJob.php

@@ -100,4 +100,11 @@ public function setMessages(?string $messages): static
 
         return $this;
     }
+
+    public function addMessage(string $message): static
+    {
+        $this->messages .= $message."\n";
+
+        return $this;
+    }
 }