Черновое приложение завершено

src/Application.php

@@ -7,6 +7,7 @@ use Avolver\CompanyEventsKitchen\Client\ClickHouseClient;
 use Avolver\CompanyEventsKitchen\Client\MongoDBClient;
 use Avolver\CompanyEventsKitchen\Command\CompaniesFillerCommand;
 use Avolver\CompanyEventsKitchen\Command\EventsRandomizerCommand;
+use Avolver\CompanyEventsKitchen\Command\EventsReportCommand;
 use Avolver\CompanyEventsKitchen\Config\Config;
 use Symfony\Component\Console\Application as ConsoleApplication;
 use Symfony\Component\Console\Input\InputInterface;
@@ -43,6 +44,7 @@ class Application extends ConsoleApplication
     {
         $this->add(new CompaniesFillerCommand());
         $this->add(new EventsRandomizerCommand());
+        $this->add(new EventsReportCommand());
 
         return parent::run($input, $output);
     }

src/Command/EventsRandomizerCommand.php

@@ -28,7 +28,7 @@ class EventsRandomizerCommand extends Command
         $this
             ->setName('generate-random-events')
             ->setDescription('Создание случайных событий для каждой компании')
-            ->addOption('count', null, InputOption::VALUE_OPTIONAL, 'количество событий', 65535);
+            ->addOption('count', null, InputOption::VALUE_OPTIONAL, 'количество событий для каждой компании', 256);
     }
 
     /**
@@ -42,10 +42,13 @@ class EventsRandomizerCommand extends Command
         $companyRepo = new CompanyRepository($this->getApplication()->getMongoClient());
         $eventsRepo  = new EventsRepository(
             $this->getApplication()->getClickhouseClient(),
-            $month,
-            (clone $month)->add(new \DateInterval('P1M'))->modify('-1 day')
+            $month
         );
 
+        if ($output->isVerbose()) {
+            $eventsRepo->setOutput($output);
+        }
+
         $eventsRepo->generateEventsForCompanies(
             $companyRepo->findAll(),
             $count

src/Command/EventsReportCommand.php

@@ -0,0 +1,79 @@
+<?php
+declare(strict_types = 1);
+
+namespace Avolver\CompanyEventsKitchen\Command;
+
+use Avolver\CompanyEventsKitchen\Application;
+use Avolver\CompanyEventsKitchen\Event\CompanyEvent;
+use Avolver\CompanyEventsKitchen\Repository\EventsRepository;
+use Avolver\CompanyEventsKitchen\Transformer\EventReportTransformer;
+use MongoDB\BSON\ObjectId;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\Table;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Команда для получения отчёта по событиям для конкретной компании
+ *
+ * @method Application getApplication
+ *
+ * @package Avolver\CompanyEventsKitchen\Command
+ */
+class EventsReportCommand extends Command
+{
+    /**
+     * {@inheritdoc}
+     */
+    protected function configure(): void
+    {
+        $this
+            ->setName('events-report')
+            ->setDescription('Создание случайных событий для каждой компании')
+            ->addOption('company-id', null, InputOption::VALUE_REQUIRED, 'ID компании')
+            ->addOption('month', null, InputOption::VALUE_OPTIONAL, 'Месяц')
+            ->addOption('year', null, InputOption::VALUE_REQUIRED, 'Год');
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $eventsRepo  = new EventsRepository(
+            $this->getApplication()->getClickhouseClient()
+        );
+
+        if (!$input->getOption('company-id')) {
+            throw new \InvalidArgumentException('Не указан идентификатор компании');
+        }
+
+        $today = new \DateTime();
+        $month = $input->getOption('month') ?: (int) $today->format('m');
+        $year  = $input->getOption('year') ?: (int) $today->format('Y');
+
+        $result = $eventsRepo->findEventsByCompanyId(
+            new ObjectId($input->getOption('company-id')),
+            $month,
+            $year
+        );
+
+        $eventTypes = CompanyEvent::keys();
+        $rows       = [];
+
+        foreach (EventReportTransformer::transform($result) as $date => $counts) {
+            $row = [$date];
+            foreach ($eventTypes as $eventType) {
+                $row[] = $counts[$eventType] ?? '0';
+            }
+            $rows[] = $row;
+        }
+
+        $table = new Table($output);
+        $table
+            ->setHeaders(array_merge(['Дата'], $eventTypes))
+            ->setRows($rows);
+        $table->render();
+    }
+}

src/Helper/RandomIterator.php

@@ -1,126 +0,0 @@
-<?php
-declare(strict_types = 1);
-
-namespace Avolver\CompanyEventsKitchen\Helper;
-
-use ArrayIterator;
-use InvalidArgumentException;
-
-/**
- * Случайный итератор
- *
- * @author hakre https://github.com/hakre
- */
-class RandomIterator implements \IteratorAggregate
-{
-    private $innerIterator;
-    private $randomIterations;
-    private $count;
-
-    /**
-     * RandomIterator constructor.
-     * @param \Traversable $iterator
-     * @param int          $randomIterations
-     */
-    public function __construct(\Traversable $iterator, $randomIterations = 1)
-    {
-
-        $this->setRandomIterations($randomIterations);
-        $this->innerIterator = $iterator;
-    }
-
-    /**
-     * @param $randomIterations
-     */
-    private function setRandomIterations($randomIterations)
-    {
-
-        $number = (int) $randomIterations;
-
-        if ($number < 1) {
-            throw new InvalidArgumentException(
-                sprintf('Number of iterations must be larger than 0, %d (%s) given.', $number, $randomIterations)
-            );
-        }
-
-        $this->randomIterations = $number;
-    }
-
-    /**
-     * @return ArrayIterator|\Traversable
-     * @throws \Exception
-     */
-    public function getIterator()
-    {
-
-        return $this->randomIterations === 1
-            ? $this->getRandomSingle()
-            : $this->getRandomMultiple();
-    }
-
-    /**
-     * @return mixed
-     */
-    public function getCount()
-    {
-
-        return $this->count;
-    }
-
-    /**
-     * SOLID This belongs into it's own type
-     *
-     * @return ArrayIterator
-     *
-     * @throws \Exception
-     */
-    private function getRandomSingle(): ArrayIterator
-    {
-
-        $result = null;
-        $count  = 0;
-        foreach ($this->innerIterator as $current) {
-            random_int(0, $count++) || $result = $current;
-        }
-
-        $this->count = $count;
-
-        return new ArrayIterator([$result]);
-    }
-
-    /**
-     * SOLID This belongs into it's own type
-     *
-     * @return ArrayIterator
-     * @throws \Exception
-     */
-    private function getRandomMultiple(): ArrayIterator
-    {
-
-        $iterator = new \IteratorIterator($this->innerIterator);
-        $iterator->rewind();
-        $it = new \NoRewindIterator($iterator);
-
-        $result    = [];
-        $pickCount = $this->randomIterations;
-        $count     = $pickCount;
-
-        while ($count-- && $it->valid()) {
-            $result[$count] = $it->current();
-            $it->next();
-        }
-
-        shuffle($result);
-
-        $count = $pickCount;
-
-        foreach ($it as $current) {
-            $random = random_int(0, $count++);
-            ($random < $pickCount) && $result[$random] = $current;
-        }
-
-        $this->count = $count;
-
-        return new ArrayIterator($result);
-    }
-}

src/Repository/CompanyRepository.php

@@ -4,7 +4,7 @@ declare(strict_types = 1);
 namespace Avolver\CompanyEventsKitchen\Repository;
 
 use Avolver\CompanyEventsKitchen\Traits\OutputTrait;
-use Symfony\Component\Console\Helper\ProgressBar;
+use Avolver\CompanyEventsKitchen\Traits\ProgressBarRepoTrait;
 use MongoDB\Driver\Exception\Exception as MongoException;
 use MongoDB\Driver\Query as MongoQuery;
 use MongoDB\Driver\Cursor as MongoCursor;
@@ -21,6 +21,7 @@ class CompanyRepository
 {
 
     use OutputTrait;
+    use ProgressBarRepoTrait;
 
     /**
      * @var MongoDBClient
@@ -114,46 +115,4 @@ class CompanyRepository
     {
         $this->threshold = $threshold;
     }
-
-    /**
-     * Создание полосы прогресса
-     *
-     * @param int $count
-     *
-     * @return ProgressBar
-     */
-    private function createProgressBar(int $count): ?ProgressBar
-    {
-        if (!$this->output) {
-            return null;
-        }
-
-        return new ProgressBar($this->output, $count);
-    }
-
-    /**
-     * Увеличение прогресса
-     *
-     * @param null|ProgressBar $progressBar
-     * @param int              $threshold
-     */
-    private function advanceProgressBar(?ProgressBar $progressBar, int $threshold): void
-    {
-        if ($progressBar) {
-            $progressBar->advance($threshold);
-        }
-    }
-
-    /**
-     * Завершение прогресса
-     *
-     * @param null|ProgressBar $progressBar
-     */
-    private function finishProgressBar(?ProgressBar $progressBar): void
-    {
-        if ($progressBar) {
-            $progressBar->finish();
-            $this->output->writeln('');
-        }
-    }
 }

src/Repository/EventsRepository.php

@@ -5,8 +5,9 @@ namespace Avolver\CompanyEventsKitchen\Repository;
 
 use Avolver\CompanyEventsKitchen\Client\ClickHouseClient;
 use Avolver\CompanyEventsKitchen\Event\CompanyEvent;
-use Avolver\CompanyEventsKitchen\Helper\RandomIterator;
 use Avolver\CompanyEventsKitchen\Traits\OutputTrait;
+use Avolver\CompanyEventsKitchen\Traits\ProgressBarRepoTrait;
+use ClickHouseDB\DatabaseException;
 use MongoDB\BSON\ObjectId;
 use MongoDB\Driver\Cursor as MongoCursor;
 
@@ -19,6 +20,7 @@ class EventsRepository
 {
 
     use OutputTrait;
+    use ProgressBarRepoTrait;
 
     /**
      * @var ClickHouseClient
@@ -28,12 +30,12 @@ class EventsRepository
     /**
      * @var \DateTime
      */
-    private $startDate;
+    private $startMonth;
 
     /**
-     * @var \DateTime
+     * @var int
      */
-    private $endDate;
+    private $daysInMonth;
 
     /**
      * Имя БД
@@ -60,14 +62,12 @@ class EventsRepository
      * EventsRepository constructor.
      *
      * @param ClickHouseClient $clickhouse
-     * @param \DateTime        $startDate
-     * @param \DateTime        $endDate
+     * @param \DateTime        $startMonth
      */
-    public function __construct(ClickHouseClient $clickhouse, \DateTime $startDate, \DateTime $endDate)
+    public function __construct(ClickHouseClient $clickhouse, \DateTime $startMonth = null)
     {
         $this->clickhouse   = $clickhouse;
-        $this->startDate    = $startDate;
-        $this->endDate      = $endDate;
+        $this->startMonth   = $startMonth;
         $this->databaseName = $this->clickhouse->getConfig()->getDatabase();
         $this->tableName    = $this->clickhouse->getConfig()->getEventsTable();
     }
@@ -79,12 +79,12 @@ class EventsRepository
     {
         // Создание БД, если это необходимо
         $this->clickhouse->write(
-            sprintf('CREATE DATABASE IF NOT EXISTS %s', $this->databaseName)
+            \sprintf('CREATE DATABASE IF NOT EXISTS %s', $this->databaseName)
         );
 
         $eventTypesMap = [];
         foreach (CompanyEvent::toArray() as $key => $value) {
-            $eventTypesMap[] = sprintf('\'%s\' = %s', $key, $value);
+            $eventTypesMap[] = \sprintf('\'%s\' = %s', $key, $value);
         }
 
         $createTableQuery = <<<QUERY
@@ -92,18 +92,16 @@ class EventsRepository
                 event_date  Date,
                 event_time  DateTime,
                 event_type  Enum8(%s),
-                id_1        Int32, 
-                id_2        Int32, 
-                id_3        Int32
+                company_id  FixedString(24) 
             )
-            ENGINE = MergeTree(event_date, (id_1, id_2, id_3), 8192)
+            ENGINE = MergeTree(event_date, (company_id), 8192)
 QUERY;
 
-        $this->clickhouse->write(sprintf(
+        $this->clickhouse->write(\sprintf(
             $createTableQuery,
             $this->databaseName,
             $this->tableName,
-            implode(', ', $eventTypesMap)
+            \implode(', ', $eventTypesMap)
         ));
     }
 
@@ -117,31 +115,40 @@ QUERY;
      */
     public function generateEventsForCompanies(MongoCursor $cursor, int $needCount = 3200): void
     {
-        //$this->createTableIfNeeded();
-        $eventTypeCount = \count(CompanyEvent::keys());
+        if (!$this->startMonth) {
+            throw new \InvalidArgumentException('Дата стартового месяца не определена');
+        }
+
+        $this->createTableIfNeeded();
         $this->clickhouse->database($this->databaseName);
+        $this->daysInMonth = (int) $this->startMonth->format('t');
+
+        $eventTypeCount = \count(CompanyEvent::keys());
+        $threshold      = 10000;
+        $eachCount      = 0;
 
-        $threshold = 100000;
-        $eachCount = 0;
+        // @todo прочитать количество из MongoDB
+        $progress    = $this->createProgressBar($needCount * 1500000);
 
         foreach ($cursor as $company) {
             $eventCount = random_int($needCount - 10, $needCount + 10);
             $eventCount = $eventCount > 0 ? $eventCount : 10;
             for ($count = 1; $count <= $eventCount; $count++) {
-                $randomDate  = $this->getRandomDate($this->startDate, $this->endDate);
+                $randomDate  = $this->getRandomDate();
                 $randomEventKey = random_int(1, $eventTypeCount);
                 $this->addEventToQueue($company->_id, $randomDate, new CompanyEvent($randomEventKey));
 
                 $eachCount++;
                 if ($eachCount % $threshold === 0) {
                     $this->insertEventsFromQueue();
-                    gc_collect_cycles();
-                    // echo " + 100k\n";
+                    $this->advanceProgressBar($progress, $threshold);
+                    \gc_collect_cycles();
                 }
             }
-
         }
 
+        $this->finishProgressBar($progress);
+
         $this->insertEventsFromQueue();
     }
 
@@ -154,15 +161,11 @@ QUERY;
      */
     public function addEventToQueue(ObjectId $companyId, \DateTime $datetime, CompanyEvent $event): void
     {
-        $idChunks = $this->convertObjectIdToInts($companyId);
-
         $this->bulkQueue[] = [
             $datetime->format('Y-m-d'),
             $datetime->getTimestamp(),
             $event->getKey(),
-            $idChunks[0],
-            $idChunks[1],
-            $idChunks[2]
+            (string) $companyId
         ];
     }
 
@@ -171,58 +174,81 @@ QUERY;
      */
     public function insertEventsFromQueue(): void
     {
-        if (!$this->bulkQueue || \count($this->bulkQueue)) {
+        if (!$this->bulkQueue || !\count($this->bulkQueue)) {
             return;
         }
 
-        $this->clickhouse->insert(
-            $this->tableName,
-            [
-                $this->bulkQueue
-            ],
-            [
-                'event_date', 'event_time', 'event_type', 'id_1', 'id_2', 'id_3'
-            ]
-        );
+        // Сортировка событий по времени, которая необходима для быстрой вставки в ClickHouse
+        \usort($this->bulkQueue, function ($left, $right) {
+            return $left[1] > $right[1];
+        });
+
+        try {
+            $this->clickhouse->insert(
+                $this->tableName,
+                $this->bulkQueue,
+                [
+                    'event_date', 'event_time', 'event_type', 'company_id'
+                ]
+            );
+        } catch (DatabaseException $e) {
+            throw $e; // new $e(substr($e->getMessage(), 0, 256) . ' — row ' . print_r($e->));
+        }
+
         $this->bulkQueue = [];
     }
 
     /**
-     * Получение случайной даты из диапозона
+     * Получение массива событий за конкретный месяц
      *
-     * @param \DateTime $startDate
-     * @param \DateTime $endDate
+     * @param ObjectId $companyId
+     * @param int      $month
+     * @param int      $year
      *
-     * @return \DateTime
+     * @return array
      *
-     * @throws \Exception
+     * @todo ...
      */
-    public function getRandomDate(\DateTime $startDate, \DateTime $endDate): \DateTime
+    public function findEventsByCompanyId(ObjectId $companyId, int $month, int $year): ?array
     {
-        $interval = new \DateInterval('P1D');
-        $period   = new \DatePeriod($startDate, $interval, $endDate);
-        $random   = new RandomIterator($period);
-
-        [$result] = iterator_to_array($random, false) ? : [null];
+        $query = <<<QUERY
+            SELECT
+              event_date AS date,
+              event_type AS type,
+              count() AS count
+            FROM kitchen.company_events
+            WHERE
+              company_id = :company_id
+              AND toMonth(event_date) = :month
+              AND toYear(event_date) = :year
+            GROUP BY event_date, event_type
+            ORDER BY event_date
+QUERY;
 
-        return $result;
+        return $this->clickhouse
+            ->select(
+                $query,
+                [
+                    'company_id' => (string) $companyId,
+                    'month'      => $month,
+                    'year'       => $year
+                ]
+            )
+            ->rows();
     }
 
     /**
-     * Преобразует идентификатор mongo в три целых числа
+     * Получение случайной даты за текущий диапозона
      *
-     * @param ObjectId $objectId
+     * @return \DateTime
      *
-     * @return array[int, int, int]
+     * @throws \Exception
      */
-    private function convertObjectIdToInts(ObjectId $objectId): array
+    public function getRandomDate(): \DateTime
     {
-        $idString = (string) $objectId;
-
-        return [
-            hexdec(mb_substr($idString, 0, 8)),
-            hexdec(mb_substr($idString, 8, 8)),
-            hexdec(mb_substr($idString, 16))
-        ];
+        return (clone $this->startMonth)
+            ->modify(
+                \sprintf('+ %d day', random_int(0, $this->daysInMonth - 2))
+            );
     }
 }

src/Traits/ProgressBarRepoTrait.php

@@ -0,0 +1,60 @@
+<?php
+declare(strict_types = 1);
+
+namespace Avolver\CompanyEventsKitchen\Traits;
+
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Подмешивание ProgressBar функционала к классу
+ *
+ * @property OutputInterface output
+ *
+ * @package Avolver\CompanyEventsKitchen\Traits
+ */
+trait ProgressBarRepoTrait
+{
+
+    /**
+     * Создание полосы прогресса
+     *
+     * @param int $count
+     *
+     * @return ProgressBar
+     */
+    protected function createProgressBar(int $count): ?ProgressBar
+    {
+        if (!$this->output) {
+            return null;
+        }
+
+        return new ProgressBar($this->output, $count);
+    }
+
+    /**
+     * Увеличение прогресса
+     *
+     * @param null|ProgressBar $progressBar
+     * @param int              $threshold
+     */
+    protected function advanceProgressBar(?ProgressBar $progressBar, int $threshold): void
+    {
+        if ($progressBar) {
+            $progressBar->advance($threshold);
+        }
+    }
+
+    /**
+     * Завершение прогресса
+     *
+     * @param null|ProgressBar $progressBar
+     */
+    protected function finishProgressBar(?ProgressBar $progressBar): void
+    {
+        if ($progressBar) {
+            $progressBar->finish();
+            $this->output->writeln('');
+        }
+    }
+}

src/Transformer/EventReportTransformer.php

@@ -0,0 +1,33 @@
+<?php
+declare(strict_types = 1);
+
+namespace Avolver\CompanyEventsKitchen\Transformer;
+
+/**
+ * Преобразователь сырых данных из EventsRepository::findEventsByCompanyId в массив, пригодный для отчёта
+ *
+ * @package Avolver\CompanyEventsKitchen\Transformer
+ */
+class EventReportTransformer
+{
+    /**
+     * Трансформация данных
+     *
+     * @param array $data
+     *
+     * @return array
+     */
+    public static function transform(array $data): array
+    {
+        $result = [];
+
+        foreach ($data as $row) {
+            if (!array_key_exists($row['date'], $result)) {
+                $result[$row['date']] = [];
+            }
+            $result[$row['date']][$row['type']] = $row['count'];
+        }
+
+        return $result;
+    }
+}