asda2432es

✏️ Editing: QueueConsumer.php

<?php
/**
 * Matomo - free/libre analytics platform
 *
 * @link https://matomo.org
 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
 *
 */

namespace Piwik\CronArchive;

use Piwik\ArchiveProcessor\Loader;
use Piwik\ArchiveProcessor\Parameters;
use Piwik\ArchiveProcessor\Rules;
use Piwik\CliMulti\RequestParser;
use Piwik\CronArchive;
use Piwik\DataAccess\ArchiveSelector;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Period;
use Piwik\Period\Factory as PeriodFactory;
use Piwik\Piwik;
use Piwik\Plugin\Manager;
use Piwik\Segment;
use Piwik\Site;
use Piwik\Timer;
use Psr\Log\LoggerInterface;

class QueueConsumer
{
    /**
     * @var LoggerInterface
     */
    private $logger;

/**
     * @var FixedSiteIds|SharedSiteIds
     */
    private $websiteIdArchiveList;

/**
     * @var int
     */
    private $countOfProcesses;

/**
     * @var int
     */
    private $pid;

/**
     * @var Model
     */
    private $model;

/**
     * @var ArchiveFilter
     */
    private $archiveFilter;

/**
     * @var SegmentArchiving
     */
    private $segmentArchiving;

/**
     * @var CronArchive
     */
    private $cronArchive;

/**
     * @var array
     */
    private $invalidationsToExclude;

/**
     * @var string[]
     */
    private $periodIdsToLabels;

/**
     * @var RequestParser
     */
    private $cliMultiRequestParser;

/**
     * @var int
     */
    private $idSite;

/**
     * @var int
     */
    private $siteRequests;

/**
     * @var Timer
     */
    private $siteTimer;

/**
     * @var string
     */
    private $currentSiteArchivingStartTime;

/**
     * @var int|null
     */
    private $maxSitesToProcess = null;

private $processedSiteCount = 0;

public function __construct(LoggerInterface $logger, $websiteIdArchiveList, $countOfProcesses, $pid, Model $model,
                                SegmentArchiving $segmentArchiving, CronArchive $cronArchive, RequestParser $cliMultiRequestParser,
                                ArchiveFilter $archiveFilter = null)
    {
        $this->logger = $logger;
        $this->websiteIdArchiveList = $websiteIdArchiveList;
        $this->countOfProcesses = $countOfProcesses;
        $this->pid = $pid;
        $this->model = $model;
        $this->segmentArchiving = $segmentArchiving;
        $this->cronArchive = $cronArchive;
        $this->cliMultiRequestParser = $cliMultiRequestParser;
        $this->archiveFilter = $archiveFilter;

// if we skip or can't process an idarchive, we want to ignore it the next time we look for an invalidated
        // archive. these IDs are stored here (using a list like this serves to keep our SQL simple).
        $this->invalidationsToExclude = [];

$this->periodIdsToLabels = array_flip(Piwik::$idPeriods);
    }

/**
     * Get next archives to process.
     *
     * Returns either an array of archives to process for the current site (may be
     * empty if there are no more archives to process for it) or null when there are
     * no more sites to process.
     *
     * @return null|array
     */
    public function getNextArchivesToProcess()
    {
        if (empty($this->idSite)) {
            if ($this->maxSitesToProcess && $this->processedSiteCount >= $this->maxSitesToProcess) {
                $this->logger->info("Maximum number of sites to process per execution has been reached.");
                return null;
            }
            $this->idSite = $this->getNextIdSiteToArchive();
            if (empty($this->idSite)) { // no sites left to archive, stop
                $this->logger->debug("No more sites left to archive, stopping.");
                return null;
            }

++$this->processedSiteCount;

/**
             * This event is triggered before the cron archiving process starts archiving data for a single
             * site.
             *
             * Note: multiple archiving processes can post this event.
             *
             * @param int $idSite The ID of the site we're archiving data for.
             * @param string $pid The PID of the process processing archives for this site.
             */
            Piwik::postEvent('CronArchive.archiveSingleSite.start', array($this->idSite, $this->pid));

$this->logger->info("Start processing archives for site {idSite}.", ['idSite' => $this->idSite]);

$this->siteTimer = new Timer();
            $this->siteRequests = 0;

// check if we need to process invalidations
            // NOTE: we do this on every site iteration so we don't end up processing say a single user entered invalidation,
            // and then stop until the next hour.
            $this->cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain($this->idSite);

$this->currentSiteArchivingStartTime = Date::now()->getDatetime();
        }

// we don't want to invalidate different periods together or segment archives w/ no-segment archives
        // together, but it's possible to end up querying these archives. if we find one, we keep track of it
        // in this array to exclude, but after we run the current batch, we reset the array so we'll still
        // process them eventually.
        $invalidationsToExcludeInBatch = [];

$siteCreationTime = Date::factory(Site::getCreationDateFor($this->idSite));

// get archives to process simultaneously
        $archivesToProcess = [];
        while (count($archivesToProcess) < $this->countOfProcesses) {
            $invalidatedArchive = $this->getNextInvalidatedArchive($this->idSite, array_keys($invalidationsToExcludeInBatch));
            if (empty($invalidatedArchive)) {
                $this->logger->debug("No next invalidated archive.");
                break;
            }

$invalidationDesc = $this->getInvalidationDescription($invalidatedArchive);

if ($invalidatedArchive['periodObj']->getDateEnd()->isEarlier($siteCreationTime)) {
                $this->logger->debug("Invalidation is for period that is older than the site's creation time, ignoring: $invalidationDesc");
                $this->model->deleteInvalidations([$invalidatedArchive]);
                continue;
            }

if (!empty($invalidatedArchive['plugin'])
                && !Manager::getInstance()->isPluginActivated($invalidatedArchive['plugin'])
            ) {
                $this->logger->debug("Plugin specific archive {$invalidatedArchive['idarchive']}'s plugin is deactivated, ignoring $invalidationDesc.");
                $this->model->deleteInvalidations([$invalidatedArchive]);
                continue;
            }

if ($invalidatedArchive['segment'] === null) {
                $this->logger->debug("Found archive for segment that is not auto archived, ignoring: $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                continue;
            }

if ($this->archiveArrayContainsArchive($archivesToProcess, $invalidatedArchive)) {
                $this->logger->debug("Found duplicate invalidated archive {$invalidatedArchive['idarchive']}, ignoring: $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                $this->model->deleteInvalidations([$invalidatedArchive]);
                continue;
            }

if ($this->model->isSimilarArchiveInProgress($invalidatedArchive)) {
                $this->logger->debug("Found duplicate invalidated archive (same archive currently in progress), ignoring: $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                $this->model->deleteInvalidations([$invalidatedArchive]);
                continue;
            }

if (self::hasIntersectingPeriod($archivesToProcess, $invalidatedArchive)) {
                $this->logger->debug("Found archive with intersecting period with others in concurrent batch, skipping until next batch: $invalidationDesc");

$idinvalidation = $invalidatedArchive['idinvalidation'];
                $invalidationsToExcludeInBatch[$idinvalidation] = true;
                continue;
            }

$reason = $this->shouldSkipArchive($invalidatedArchive);
            if ($reason) {
                $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idinvalidation']}, $reason: $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                continue;
            }

list($isUsableExists, $archivedTime) = $this->usableArchiveExists($invalidatedArchive);
            if ($isUsableExists) {
                $now = Date::now()->getDatetime();
                $this->addInvalidationToExclude($invalidatedArchive);
                if (empty($invalidatedArchive['plugin'])) {
                    $this->logger->debug("Found invalidation with usable archive (not yet outdated, ts_archived of existing = $archivedTime, now = $now) skipping until archive is out of date: $invalidationDesc");
                } else {
                    $this->logger->debug("Found invalidation with usable archive (not yet outdated, ts_archived of existing = $archivedTime, now = $now) ignoring and deleting: $invalidationDesc");
                    $this->model->deleteInvalidations([$invalidatedArchive]);
                }
                continue;
            } else {
                $now = Date::now()->getDatetime();
                $this->logger->debug("No usable archive exists (ts_archived of existing = $archivedTime, now = $now).");
            }

$alreadyInProgressId = $this->model->isArchiveAlreadyInProgress($invalidatedArchive);
            if ($alreadyInProgressId) {
                $this->addInvalidationToExclude($invalidatedArchive);
                if ($alreadyInProgressId < $invalidatedArchive['idinvalidation']) {
                    $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idinvalidation']}, invalidation already in progress. Since in progress is older, not removing invalidation.");
               } else if ($alreadyInProgressId > $invalidatedArchive['idinvalidation']) {
                    $this->logger->debug("Skipping invalidated archive {$invalidatedArchive['idinvalidation']}, invalidation already in progress. Since in progress is newer, will remove invalidation.");
                    $this->model->deleteInvalidations([$invalidatedArchive]);
                }
                continue;
            }

if ($this->canSkipArchiveBecauseNoPoint($invalidatedArchive)) {
                $this->logger->debug("Found invalidated archive we can skip (no visits): $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                $this->model->deleteInvalidations([$invalidatedArchive]);
                continue;
            }

$reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive);
            if ($reason) {
                $this->logger->debug("Skipping invalidated archive, $reason: $invalidationDesc");
                $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true;
                $this->addInvalidationToExclude($invalidatedArchive);
                continue;
            }

$started = $this->model->startArchive($invalidatedArchive);
            if (!$started) { // another process started on this archive, pull another one
                $this->logger->debug("Archive invalidation is being handled by another process: $invalidationDesc");
                $this->addInvalidationToExclude($invalidatedArchive);
                continue;
            }

$this->addInvalidationToExclude($invalidatedArchive);

$this->logger->debug("Processing invalidation: $invalidationDesc.");

$archivesToProcess[] = $invalidatedArchive;
        }

if (empty($archivesToProcess)
            && empty($invalidationsToExcludeInBatch)
        ) { // no invalidated archive left
            /**
             * This event is triggered immediately after the cron archiving process starts archiving data for a single
             * site.
             *
             * Note: multiple archiving processes can post this event.
             *
             * @param int $idSite The ID of the site we're archiving data for.
             * @param string $pid The PID of the process processing archives for this site.
             */
            Piwik::postEvent('CronArchive.archiveSingleSite.finish', array($this->idSite, $this->pid));

$this->logger->info("Finished archiving for site {idSite}, {requests} API requests, {timer} [{processed} / {totalNum} done]", [
                'idSite' => $this->idSite,
                'processed' => $this->processedSiteCount,
                'totalNum' => $this->websiteIdArchiveList->getNumSites(),
                'timer' => $this->siteTimer,
                'requests' => $this->siteRequests,
            ]);

$this->idSite = null;
        }

$this->siteRequests += count($archivesToProcess);

return $archivesToProcess;
    }

private function archiveArrayContainsArchive($archiveArray, $archive)
    {
        foreach ($archiveArray as $entry) {
            if ($entry['idsite'] == $archive['idsite']
                && $entry['period'] == $archive['period']
                && $entry['date1'] == $archive['date1']
                && $entry['date2'] == $archive['date2']
                && $entry['name'] == $archive['name']
                && $entry['plugin'] == $archive['plugin']
                && $entry['report'] == $archive['report']
            ) {
                return true;
            }
        }
        return false;
    }

private function getNextInvalidatedArchive($idSite, $extraInvalidationsToIgnore)
    {
        $iterations = 0;
        while ($iterations < 100) {
            $invalidationsToExclude = array_merge($this->invalidationsToExclude, $extraInvalidationsToIgnore);

$nextArchive = $this->model->getNextInvalidatedArchive($idSite, $this->currentSiteArchivingStartTime, $invalidationsToExclude);
            if (empty($nextArchive)) {
                break;
            }

$this->detectPluginForArchive($nextArchive);

$periodLabel = $this->periodIdsToLabels[$nextArchive['period']];
            if (!PeriodFactory::isPeriodEnabledForAPI($periodLabel)
                || PeriodFactory::isAnyLowerPeriodDisabledForAPI($periodLabel)
            ) {
                $this->logger->info("Found invalidation for period that is disabled in the API, skipping and removing: {$nextArchive['idinvalidation']}");
                $this->model->deleteInvalidations([$nextArchive]);
                continue;
            }

$periodDate = $periodLabel == 'range' ? $nextArchive['date1'] . ',' . $nextArchive['date2'] : $nextArchive['date1'];
            $nextArchive['periodObj'] = PeriodFactory::build($periodLabel, $periodDate);

$isCronArchivingEnabled = $this->findSegmentForArchive($nextArchive);
            if ($isCronArchivingEnabled) {
                return $nextArchive;
            }

$this->logger->debug("Found invalidation for segment that does not have auto archiving enabled, skipping: {$nextArchive['idinvalidation']}");
            $this->model->deleteInvalidations([$nextArchive]);

++$iterations;
        }

return null;
    }

private function shouldSkipArchive($archive)
    {
        if ($this->archiveFilter) {
            return $this->archiveFilter->filterArchive($archive);
        }

return false;
    }

// public for tests
    public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive)
    {
        $site = new Site($invalidatedArchive['idsite']);

$periodLabel = $this->periodIdsToLabels[$invalidatedArchive['period']];
        $dateStr = $periodLabel == 'range' ? ($invalidatedArchive['date1'] . ',' . $invalidatedArchive['date2']) : $invalidatedArchive['date1'];
        $period = PeriodFactory::build($periodLabel, $dateStr);

$segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]);

$params = new Parameters($site, $period, $segment);

$loader = new Loader($params);
        return $loader->canSkipThisArchive(); // if no point in archiving, skip
    }

public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess)
    {
        $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands();

foreach ($inProgressArchives as $archiveBeingProcessed) {
            if (empty($archiveBeingProcessed['period'])
                || empty($archiveBeingProcessed['date'])
            ) {
                continue;
            }

if (empty($archiveBeingProcessed['idSite'])
                || $archiveBeingProcessed['idSite'] != $archiveToProcess['idsite']
            ) {
                continue; // different site
            }

// we don't care about lower periods being concurrent if they are for different segments (that are not "all visits")
            if (!empty($archiveBeingProcessed['segment'])
                && !empty($archiveToProcess['segment'])
                && $archiveBeingProcessed['segment'] != $archiveToProcess['segment']
                && urldecode($archiveBeingProcessed['segment']) != $archiveToProcess['segment']
            ) {
                continue;
            }

$archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']);

if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) {
                return "lower or same period in progress in another local climulti process (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})";
            }

if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) {
                return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})";
            }
        }

return false;
    }

private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed)
    {
        /** @var Period $archiveToProcessPeriodObj */
        $archiveToProcessPeriodObj = $archiveToProcess['periodObj'];
        /** @var Period $archivePeriodObj */
        $archivePeriodObj = $archiveBeingProcessed['periodObj'];

if ($archiveToProcessPeriodObj->getId() >= $archivePeriodObj->getId()
            && $archiveToProcessPeriodObj->isPeriodIntersectingWith($archivePeriodObj)
        ) {
            return true;
        }

return false;
    }

private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveToProcess, array $archiveBeingProcessed)
    {
        // archive is for different site/period
        if (empty($archiveBeingProcessed['idSite'])
            || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite']
            || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId()
            || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString()
        ) {
            return false;
        }

return empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment']);
    }

private function detectPluginForArchive(&$archive)
    {
        $archive['plugin'] = $this->getPluginNameForArchiveIfAny($archive);
    }

// static so it can be unit tested
    public static function hasIntersectingPeriod(array $archivesToProcess, $invalidatedArchive)
    {
        if (empty($archivesToProcess)) {
            return false;
        }

foreach ($archivesToProcess as $archive) {
            $isSamePeriod = $archive['period'] == $invalidatedArchive['period']
                && $archive['date1'] == $invalidatedArchive['date1']
                && $archive['date2'] == $invalidatedArchive['date2'];

// don't do the check for $archvie, if we have the same period and segment as $invalidatedArchive
            // we only want to to do the intersecting periods check if there are different periods or one of the
            // invalidations is for an "all visits" archive.
            //
            // it's allowed to archive the same period concurrently for different segments, where neither is
            // "All Visits"
            if (!empty($archive['segment'])
                && !empty($invalidatedArchive['segment'])
                && $archive['segment'] != $invalidatedArchive['segment']
                && $isSamePeriod
            ) {
                continue;
            }

if ($archive['periodObj']->isPeriodIntersectingWith($invalidatedArchive['periodObj'])) {
                return true;
            }
        }

return false;
    }

private function findSegmentForArchive(&$archive)
    {
        $flag = explode('.', $archive['name'])[0];
        if ($flag == 'done') {
            $archive['segment'] = '';
            return true;
        }

$hash = substr($flag, 4);
        $storedSegment = $this->segmentArchiving->findSegmentForHash($hash, $archive['idsite']);
        if (!isset($storedSegment['definition'])) {
            $this->logger->debug("Could not find stored segment for done flag hash: $flag");
            $archive['segment'] = null;
            return false;
        }

$archive['segment'] = $storedSegment['definition'];
        return $this->segmentArchiving->isAutoArchivingEnabledFor($storedSegment);
    }

private function getPluginNameForArchiveIfAny($archive)
    {
        $name = $archive['name'];
        if (strpos($name, '.') === false) {
            return null;
        }

$parts = explode('.', $name);
        return $parts[1];
    }

public function ignoreIdInvalidation($idinvalidation)
    {
        $this->invalidationsToExclude[$idinvalidation] = $idinvalidation;
    }

public function skipToNextSite()
    {
        $this->idSite = null;
    }

private function addInvalidationToExclude(array $invalidatedArchive)
    {
        $id = $invalidatedArchive['idinvalidation'];
        if (empty($this->invalidationsToExclude[$id])) {
            $this->invalidationsToExclude[$id] = $id;
        }
    }

private function getNextIdSiteToArchive()
    {
        return $this->websiteIdArchiveList->getNextSiteId();
    }

private function getInvalidationDescription(array $invalidatedArchive)
    {
        return sprintf("[idinvalidation = %s, idsite = %s, period = %s(%s - %s), name = %s, segment = %s]",
            $invalidatedArchive['idinvalidation'],
            $invalidatedArchive['idsite'],
            $this->periodIdsToLabels[$invalidatedArchive['period']],
            $invalidatedArchive['date1'],
            $invalidatedArchive['date2'],
            $invalidatedArchive['name'],
            $invalidatedArchive['segment'] ?? ''
        );
    }

// public for test
    public function usableArchiveExists(array $invalidatedArchive)
    {
        $site = new Site($invalidatedArchive['idsite']);

$segment = new Segment($invalidatedArchive['segment'], [$invalidatedArchive['idsite']]);

$params = new Parameters($site, $period, $segment);

// if latest archive includes today and is usable (DONE_OK or DONE_INVALIDATED and recent enough), skip
        $today = Date::factoryInTimezone('today', Site::getTimezoneFor($site->getId()));
        $isArchiveIncludesToday = $period->isDateInPeriod($today);
        if (!$isArchiveIncludesToday) {
            return [false, null];
        }

// if valid archive already exists, do not re-archive
        $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel));
        $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = false);

$tsArchived = !empty($archiveIdAndVisits[4]) ? Date::factory($archiveIdAndVisits[4])->getDatetime() : null;

$idArchive = $archiveIdAndVisits[0];
        if (empty($idArchive)) {
            return [false, $tsArchived];
        }

return [true, $tsArchived];
    }

public function getIdSite()
    {
        return $this->idSite;
    }

/**
     * Set or get the maximum number of sites to process
     *
     * @param int|null $newValue New value or null to just return current value
     *
     * @return int|null New or existing value
     */
    public function setMaxSitesToProcess($newValue = null)
    {
        if (null !== $newValue) {
            $this->maxSitesToProcess = $newValue;
        }
        return $this->maxSitesToProcess;
    }
}

Ubuntu

Sid Gifari From Gifari Industries - BD Cyber Security Team