From 7b702660ef52338b3c14c1310f53d43cd346ccac Mon Sep 17 00:00:00 2001
From: Jacek Kowalski <Jacek@jacekk.info>
Date: Fri, 21 Jun 2019 11:27:37 +0000
Subject: [PATCH] Add DB caching and make mapping independent on numToType
---
lib/mapper.php | 22 ++--------
parse.php | 31 ++++++---------
lib/database.php | 62 +++++++++++++++++++++++++------
3 files changed, 67 insertions(+), 48 deletions(-)
diff --git a/lib/database.php b/lib/database.php
index 2c86a7b..22cc51c 100644
--- a/lib/database.php
+++ b/lib/database.php
@@ -1,9 +1,10 @@
<?php
class Database {
private $pdo;
- private $getByIdStatement;
- private $getByNumStatement;
private $addStatement;
+
+ private $cacheId;
+ private $cacheNum;
public function __construct($file) {
$this->pdo = new PDO('sqlite:'.$file);
@@ -15,9 +16,9 @@
weight INT
)');
- $this->getByIdStatement = $this->pdo->prepare('SELECT num, weight FROM vehicles WHERE id=? LIMIT 1');
- $this->getByNumStatement = $this->pdo->prepare('SELECT id, weight FROM vehicles WHERE num=? LIMIT 1');
- $this->addStatement = $this->pdo->prepare('INSERT OR REPLACE INTO vehicles (id, num, weight) VALUES (?, ?, ?)');
+ $this->addStatement = $this->pdo->prepare('INSERT OR REPLACE INTO vehicles (id, num, weight) VALUES (:id, :num, :weight)');
+
+ $this->_cacheClear();
}
public function beginTransaction() {
@@ -32,29 +33,66 @@
$this->pdo->rollback();
}
+ protected function _cachePopulate() {
+ if($this->cacheId === NULL) {
+ $st = $this->pdo->prepare('SELECT * FROM vehicles');
+ $st->execute();
+ $result = $st->fetchAll(PDO::FETCH_ASSOC);
+
+ $this->cacheId = [];
+ $this->cacheNum = [];
+ foreach($result as $vehicle) {
+ $this->_cacheAdd($vehicle);
+ }
+ }
+ }
+
+ protected function _cacheAdd($vehicle) {
+ $this->_cachePopulate();
+ $this->cacheId[$vehicle['id']] = $vehicle;
+ $this->cacheNum[$vehicle['num']] = $vehicle;
+ }
+
+ protected function _cacheClear() {
+ $this->cacheId = NULL;
+ $this->cacheNum = NULL;
+ }
+
+ public function getAll() {
+ $this->_cachePopulate();
+ return $this->cacheId;
+ }
+
public function getById($id) {
- $this->getByIdStatement->execute([$id]);
- return $this->getByIdStatement->fetch();
+ $this->_cachePopulate();
+ return $this->cacheId[$id] ?? NULL;
}
public function getByNum($num) {
- $st = $this->getByNumStatement->execute([(int)substr($num, 2)]);
- return $this->getByNumStatement->fetch();
+ $this->_cachePopulate();
+ return $this->cacheNum[$num] ?? NULL;
}
public function clear() {
$this->pdo->query('DELETE FROM vehicles');
+ $this->_cacheClear();
}
public function add($id, $num, $weight) {
- $this->addStatement->execute([$id, $num, $weight]);
+ $vehicle = [
+ 'id' => (string)$id,
+ 'num' => (string)$num,
+ 'weight' => (string)$weight
+ ];
+ $this->addStatement->execute($vehicle);
+ $this->_cacheAdd($vehicle);
}
public function addMapping($mapping) {
$this->beginTransaction();
$weight = count($mapping);
- foreach($mapping as $id => $vehicle) {
- $this->add($id, (int)substr($vehicle['num'], 2), $weight);
+ foreach($mapping as $id => $num) {
+ $this->add($id, $num, $weight);
}
$this->commit();
}
diff --git a/lib/mapper.php b/lib/mapper.php
index 1eeb1d2..02033ae 100644
--- a/lib/mapper.php
+++ b/lib/mapper.php
@@ -1,6 +1,5 @@
<?php
require_once(__DIR__.'/../vendor/autoload.php');
-require_once(__DIR__.'/vehicle_types.php');
use transit_realtime\FeedMessage;
@@ -42,8 +41,8 @@
continue;
}
}
- $this->ttssTrips[(int)$vehicle->tripId] = [
- 'id' => $vehicle->id,
+ $this->ttssTrips[(string)$vehicle->tripId] = [
+ 'id' => (string)$vehicle->id,
'latitude' => (float)$vehicle->latitude / 3600000.0,
'longitude' => (float)$vehicle->longitude / 3600000.0,
];
@@ -67,7 +66,7 @@
$trip = $vehiclePosition->getTrip();
$tripId = $trip->getTripId();
$this->gtfsrtTrips[self::convertTripId($tripId)] = [
- 'id' => $entity->getId(),
+ 'id' => (string)$entity->getId(),
'num' => $vehicle->getLicensePlate(),
'tripId' => $tripId,
'latitude' => $position->getLatitude(),
@@ -126,23 +125,12 @@
return $bestOffset;
}
- public function mapUsingOffset($offset, $mapper) {
+ public function mapUsingOffset($offset) {
$result = [];
foreach($this->gtfsrtTrips as $gtfsTripId => $gtfsTrip) {
$ttssTripId = $gtfsTripId + $offset;
if(isset($this->ttssTrips[$ttssTripId])) {
- $data = $mapper($gtfsTrip['id']);
- $num = $gtfsTrip['num'];
- if(!is_array($data) || !isset($data['num'])) {
- $data = [
- 'num' => $num ?: '??'.$gtfsTrip['id'],
- 'low' => NULL,
- ];
- } elseif($data['num'] != $num) {
- // Ignore due to incorrect depot markings in the data
- //$this->logger->warn('Got '.$num.', database has '.$data['num']);
- }
- $result[$this->ttssTrips[$ttssTripId]['id']] = $data;
+ $result[$this->ttssTrips[$ttssTripId]['id']] = $gtfsTrip['id'];
}
}
return $result;
diff --git a/parse.php b/parse.php
index b517ae2..9e66121 100644
--- a/parse.php
+++ b/parse.php
@@ -2,8 +2,7 @@
require_once(__DIR__.'/lib/database.php');
require_once(__DIR__.'/lib/fetch.php');
require_once(__DIR__.'/lib/mapper.php');
-
-$logger = new Monolog\Logger('Parse changes');
+require_once(__DIR__.'/lib/vehicle_types.php');
$sources = [
'bus' => [
@@ -77,7 +76,7 @@
}
$logger->info('Got offset '.$offset.', creating mapping...');
- $mapping = $mapper->mapUsingOffset($offset, $source['mapper']);
+ $mapping = $mapper->mapUsingOffset($offset);
$logger->info('Checking the data for correctness...');
$weight = count($mapping);
@@ -86,11 +85,11 @@
$incorrect = 0;
$old = 0;
$maxWeight = 0;
- foreach($mapping as $id => $vehicle) {
+ foreach($mapping as $id => $num) {
$dbVehicle = $db->getById($id);
if($dbVehicle) {
- $maxWeight = max($maxWeight, $dbVehicle['weight']);
- if((int)substr($vehicle['num'], 2) == (int)$dbVehicle['num']) {
+ $maxWeight = max($maxWeight, (int)$dbVehicle['weight']);
+ if($num === $dbVehicle['num']) {
$correct += 1;
} else {
$incorrect += 1;
@@ -98,32 +97,26 @@
continue;
}
- $dbVehicle = $db->getByNum($vehicle['num']);
- if($dbVehicle && $dbVehicle['id'] != $id) {
+ $dbVehicle = $db->getByNum($num);
+ if($dbVehicle && $dbVehicle['id'] !== $id) {
$old += 1;
}
}
+
$logger->info('Weight: '.$weight.', correct: '.$correct.', incorrect: '.$incorrect.', old: '.$old);
- $previousMapping = NULL;
if($incorrect > $correct && $maxWeight > $weight) {
throw new Exception('Ignoring result due to better data already present');
- } elseif($old > $correct) {
- $logger->warn('Replacing DB data with the new mapping');
- $db->clear();
- } else {
- $previousMapping = @json_decode(@file_get_contents($source['result']), TRUE);
}
$db->addMapping($mapping);
- if(is_array($previousMapping)) {
- $logger->info('Merging previous data with current mapping');
- $mapping = $mapping + $previousMapping;
- ksort($mapping);
+ $jsonContent = [];
+ foreach($db->getAll() as $vehicle) {
+ $jsonContent[$vehicle['id']] = $source['mapper']($vehicle['num']);
}
- $json = json_encode($mapping);
+ $json = json_encode($jsonContent);
if(!file_put_contents($source['result_temp'], $json)) {
throw new Exception('Result save failed');
}
--
Gitblit v1.9.1