From 1a98b6480fb4e8beb9091326112d8ebc6ca1e7cd Mon Sep 17 00:00:00 2001
From: Jacek Kowalski <Jacek@jacekk.info>
Date: Mon, 18 Feb 2019 23:46:55 +0000
Subject: [PATCH] Don't overwrite non-conflicting vehicle mappings from previous runs
---
parse.php | 78 ++++++++++++++++++++++----
lib/database.php | 47 +++++++++++++++
2 files changed, 113 insertions(+), 12 deletions(-)
diff --git a/lib/database.php b/lib/database.php
new file mode 100644
index 0000000..48ad047
--- /dev/null
+++ b/lib/database.php
@@ -0,0 +1,47 @@
+<?php
+class Database {
+ private $pdo;
+ private $getByIdStatement;
+ private $getByNumStatement;
+ private $addStatement;
+
+ public function __construct($file) {
+ $this->pdo = new PDO('sqlite:'.$file);
+ $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+
+ $this->pdo->query('CREATE TABLE IF NOT EXISTS vehicles (
+ id INT PRIMARY KEY,
+ num INT UNIQUE,
+ weight INT
+ )');
+
+ $this->getByIdStatement = $this->pdo->prepare('SELECT num, weight FROM vehicles WHERE id=? LIMIT 1');
+ $this->getByNumStatement = $this->pdo->prepare('SELECT id, weight FROM vehicles WHERE num=? LIMIT 1');
+ $this->addStatement = $this->pdo->prepare('INSERT OR REPLACE INTO vehicles (id, num, weight) VALUES (?, ?, ?)');
+ }
+
+ public function getById($id) {
+ $this->getByIdStatement->execute([$id]);
+ return $this->getByIdStatement->fetch();
+ }
+
+ public function getByNum($num) {
+ $st = $this->getByNumStatement->execute([(int)substr($num, 2)]);
+ return $this->getByNumStatement->fetch();
+ }
+
+ public function clear() {
+ $this->pdo->query('DELETE FROM vehicles');
+ }
+
+ public function add($id, $num, $weight) {
+ $this->addStatement->execute([$id, $num, $weight]);
+ }
+
+ public function addMapping($mapping) {
+ $weight = count($mapping);
+ foreach($mapping as $id => $vehicle) {
+ $this->add($id, (int)substr($vehicle['num'], 2), $weight);
+ }
+ }
+}
diff --git a/parse.php b/parse.php
index 02e5f28..432b3ef 100644
--- a/parse.php
+++ b/parse.php
@@ -1,4 +1,5 @@
<?php
+require_once(__DIR__.'/lib/database.php');
require_once(__DIR__.'/lib/fetch.php');
require_once(__DIR__.'/lib/mapper.php');
@@ -10,6 +11,7 @@
'gtfs_file' => 'VehiclePositions_A.pb',
'ttss' => 'http://91.223.13.70/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
'ttss_file' => 'vehicles_A.json',
+ 'database' => 'mapping_A.sqlite3',
'result' => 'mapping_A.json',
],
];
@@ -17,32 +19,84 @@
foreach($sources as $name => $source) {
$logger = new Monolog\Logger('fetch_'.$name);
try {
+ foreach(['gtfs_file', 'ttss_file', 'database', 'result'] as $field) {
+ $source[$field] = __DIR__.'/data/'.$source[$field];
+ }
+ $source['result_temp'] = $source['result'].'.tmp';
+
$logger->info('Fetching '.$name.' position data from FTP...');
- $updated = ftp_fetch_if_newer($source['gtfs'], __DIR__.'/data/'.$source['gtfs_file']);
+ $updated = ftp_fetch_if_newer($source['gtfs'], $source['gtfs_file']);
if(!$updated) {
$logger->info('Nothing to do, remote file not newer than local one');
continue;
}
$logger->info('Fetching '.$name.' positions from TTSS...');
- fetch($source['ttss'], __DIR__.'/data/'.$source['ttss_file']);
+ fetch($source['ttss'],$source['ttss_file']);
$logger->info('Loading data...');
$mapper = new Mapper();
- $mapper->loadTTSS(__DIR__.'/data/'.$source['ttss_file']);
- $mapper->loadGTFS(__DIR__.'/data/'.$source['gtfs_file']);
+ $mapper->loadTTSS($source['ttss_file']);
+ $mapper->loadGTFS($source['gtfs_file']);
+
+ $db = new Database($source['database']);
$logger->info('Finding correct offset...');
$offset = $mapper->findOffset();
- if($offset) {
- $logger->info('Got offset '.$offset.', creating mapping...');
- $mapping = $mapper->getMapping($offset);
- $json = json_encode($mapping);
- if(!file_put_contents(__DIR__.'/data/'.$source['result'].'.tmp', $json)) {
- throw new Exception('Result save failed');
- }
- rename(__DIR__.'/data/'.$source['result'].'.tmp', __DIR__.'/data/'.$source['result']);
+ if(!$offset) {
+ throw new Exception('Offset not found');
}
+
+ $logger->info('Got offset '.$offset.', creating mapping...');
+ $mapping = $mapper->getMapping($offset);
+
+ $logger->info('Checking the data for correctness...');
+ $weight = count($mapping);
+ $replace = 0;
+ $ignore = 0;
+ foreach($mapping as $id => $vehicle) {
+ $dbVehicle = $db->getById($id);
+ if($dbVehicle) {
+ if((int)substr($vehicle['num'], 2) != (int)$dbVehicle['num']) {
+ if($weight > $dbVehicle['weight']) {
+ $replace += 1;
+ } else {
+ $ignore += 1;
+ }
+ }
+ continue;
+ }
+
+ $dbVehicle = $db->getByNum($vehicle['num']);
+ if($dbVehicle && $dbVehicle['id'] != $id) {
+ $replace += 1;
+ }
+ }
+ $logger->info('Weight: '.$weight.', ignore: '.$ignore.', replace: '.$replace);
+
+ $previousMapping = NULL;
+ if($ignore > 0 && $ignore >= $replace) {
+ throw new Exception('Ignoring result due to better data already present');
+ } elseif($replace > 0) {
+ $logger->warn('Replacing DB data with the mapping');
+ $db->clear();
+ } else {
+ $previousMapping = @json_decode(@file_get_contents($source['result']), TRUE);
+ }
+
+ $db->addMapping($mapping);
+
+ if(is_array($previousMapping)) {
+ $logger->info('Merging previous data with current mapping');
+ $mapping = $previousMapping + $mapping;
+ ksort($mapping);
+ }
+
+ $json = json_encode($mapping);
+ if(!file_put_contents($source['result_temp'], $json)) {
+ throw new Exception('Result save failed');
+ }
+ rename($source['result_temp'], $source['result']);
$logger->info('Finished');
} catch(Throwable $e) {
$logger->error($e->getMessage(), ['exception' => $e, 'exception_string' => (string)$e]);
--
Gitblit v1.9.1