Add DB caching and make mapping independent on numToType

 lib/database.php

@@ -1,9 +1,10 @@
<?php
class Database {
    private $pdo;
    private $getByIdStatement;
    private $getByNumStatement;
    private $addStatement;
	
    private $cacheId;
    private $cacheNum;
    
    public function __construct($file) {
        $this->pdo = new PDO('sqlite:'.$file);
@@ -15,9 +16,9 @@
            weight INT
        )');
        
        $this->getByIdStatement = $this->pdo->prepare('SELECT num, weight FROM vehicles WHERE id=? LIMIT 1');
        $this->getByNumStatement = $this->pdo->prepare('SELECT id, weight FROM vehicles WHERE num=? LIMIT 1');
        $this->addStatement = $this->pdo->prepare('INSERT OR REPLACE INTO vehicles (id, num, weight) VALUES (?, ?, ?)');
        $this->addStatement = $this->pdo->prepare('INSERT OR REPLACE INTO vehicles (id, num, weight) VALUES (:id, :num, :weight)');
		
        $this->_cacheClear();
    }
    
    public function beginTransaction() {
@@ -32,29 +33,66 @@
        $this->pdo->rollback();
    }
    
    protected function _cachePopulate() {
        if($this->cacheId === NULL) {
            $st = $this->pdo->prepare('SELECT * FROM vehicles');
            $st->execute();
            $result = $st->fetchAll(PDO::FETCH_ASSOC);
			
            $this->cacheId = [];
            $this->cacheNum = [];
            foreach($result as $vehicle) {
                $this->_cacheAdd($vehicle);
            }
        }
    }
	
    protected function _cacheAdd($vehicle) {
        $this->_cachePopulate();
        $this->cacheId[$vehicle['id']] = $vehicle;
        $this->cacheNum[$vehicle['num']] = $vehicle;
    }
	
    protected function _cacheClear() {
        $this->cacheId = NULL;
        $this->cacheNum = NULL;
    }
	
    public function getAll() {
        $this->_cachePopulate();
        return $this->cacheId;
    }
	
    public function getById($id) {
        $this->getByIdStatement->execute([$id]);
        return $this->getByIdStatement->fetch();
        $this->_cachePopulate();
        return $this->cacheId[$id] ?? NULL;
    }
    
    public function getByNum($num) {
        $st = $this->getByNumStatement->execute([(int)substr($num, 2)]);
        return $this->getByNumStatement->fetch();
        $this->_cachePopulate();
        return $this->cacheNum[$num] ?? NULL;
    }
    
    public function clear() {
        $this->pdo->query('DELETE FROM vehicles');
        $this->_cacheClear();
    }
    
    public function add($id, $num, $weight) {
        $this->addStatement->execute([$id, $num, $weight]);
        $vehicle = [
            'id' => (string)$id,
            'num' => (string)$num,
            'weight' => (string)$weight
        ];
        $this->addStatement->execute($vehicle);
        $this->_cacheAdd($vehicle);
    }
    
    public function addMapping($mapping) {
        $this->beginTransaction();
        $weight = count($mapping);
        foreach($mapping as $id => $vehicle) {
            $this->add($id, (int)substr($vehicle['num'], 2), $weight);
        foreach($mapping as $id => $num) {
            $this->add($id, $num, $weight);
        }
        $this->commit();
    }

 lib/mapper.php

@@ -1,6 +1,5 @@
<?php
require_once(__DIR__.'/../vendor/autoload.php');
require_once(__DIR__.'/vehicle_types.php');

use transit_realtime\FeedMessage;

@@ -42,8 +41,8 @@
                    continue;
                }
            }
            $this->ttssTrips[(int)$vehicle->tripId] = [
                'id' => $vehicle->id,
            $this->ttssTrips[(string)$vehicle->tripId] = [
                'id' => (string)$vehicle->id,
                'latitude' => (float)$vehicle->latitude / 3600000.0,
                'longitude' => (float)$vehicle->longitude / 3600000.0,
            ];
@@ -67,7 +66,7 @@
            $trip = $vehiclePosition->getTrip();
            $tripId = $trip->getTripId();
            $this->gtfsrtTrips[self::convertTripId($tripId)] = [
                'id' => $entity->getId(),
                'id' => (string)$entity->getId(),
                'num' => $vehicle->getLicensePlate(),
                'tripId' => $tripId,
                'latitude' => $position->getLatitude(),
@@ -126,23 +125,12 @@
        return $bestOffset;
    }
    
    public function mapUsingOffset($offset, $mapper) {
    public function mapUsingOffset($offset) {
        $result = [];
        foreach($this->gtfsrtTrips as $gtfsTripId => $gtfsTrip) {
            $ttssTripId = $gtfsTripId + $offset;
            if(isset($this->ttssTrips[$ttssTripId])) {
                $data = $mapper($gtfsTrip['id']);
                $num = $gtfsTrip['num'];
                if(!is_array($data) || !isset($data['num'])) {
                    $data = [
                        'num' => $num ?: '??'.$gtfsTrip['id'],
                        'low' => NULL,
                    ];
                } elseif($data['num'] != $num) {
                    // Ignore due to incorrect depot markings in the data
                    //$this->logger->warn('Got '.$num.', database has '.$data['num']);
                }
                $result[$this->ttssTrips[$ttssTripId]['id']] = $data;
                $result[$this->ttssTrips[$ttssTripId]['id']] = $gtfsTrip['id'];
            }
        }
        return $result;

 parse.php

@@ -2,8 +2,7 @@
require_once(__DIR__.'/lib/database.php');
require_once(__DIR__.'/lib/fetch.php');
require_once(__DIR__.'/lib/mapper.php');

$logger = new Monolog\Logger('Parse changes');
require_once(__DIR__.'/lib/vehicle_types.php');

$sources = [
    'bus' => [
@@ -77,7 +76,7 @@
        }
        
        $logger->info('Got offset '.$offset.', creating mapping...');
        $mapping = $mapper->mapUsingOffset($offset, $source['mapper']);
        $mapping = $mapper->mapUsingOffset($offset);
        
        $logger->info('Checking the data for correctness...');
        $weight = count($mapping);
@@ -86,11 +85,11 @@
        $incorrect = 0;
        $old = 0;
        $maxWeight = 0;
        foreach($mapping as $id => $vehicle) {
        foreach($mapping as $id => $num) {
            $dbVehicle = $db->getById($id);
            if($dbVehicle) {
                $maxWeight = max($maxWeight, $dbVehicle['weight']);
                if((int)substr($vehicle['num'], 2) == (int)$dbVehicle['num']) {
                $maxWeight = max($maxWeight, (int)$dbVehicle['weight']);
                if($num === $dbVehicle['num']) {
                    $correct += 1;
                } else {
                    $incorrect += 1;
@@ -98,32 +97,26 @@
                continue;
            }
            
            $dbVehicle = $db->getByNum($vehicle['num']);
            if($dbVehicle && $dbVehicle['id'] != $id) {
            $dbVehicle = $db->getByNum($num);
            if($dbVehicle && $dbVehicle['id'] !== $id) {
                $old += 1;
            }
        }

        $logger->info('Weight: '.$weight.', correct: '.$correct.', incorrect: '.$incorrect.', old: '.$old);
        
        $previousMapping = NULL;
        if($incorrect > $correct && $maxWeight > $weight) {
            throw new Exception('Ignoring result due to better data already present');
        } elseif($old > $correct) {
            $logger->warn('Replacing DB data with the new mapping');
            $db->clear();
        } else {
            $previousMapping = @json_decode(@file_get_contents($source['result']), TRUE);
        }
        
        $db->addMapping($mapping);
        
        if(is_array($previousMapping)) {
            $logger->info('Merging previous data with current mapping');
            $mapping = $mapping + $previousMapping;
            ksort($mapping);
        $jsonContent = [];
        foreach($db->getAll() as $vehicle) {
            $jsonContent[$vehicle['id']] = $source['mapper']($vehicle['num']);
        }
        
        $json = json_encode($mapping);
        $json = json_encode($jsonContent);
        if(!file_put_contents($source['result_temp'], $json)) {
            throw new Exception('Result save failed');
        }