From 4673cc38e7b5b1b87d8e009137fc2d5eae688fd7 Mon Sep 17 00:00:00 2001 From: Jacek Kowalski <Jacek@jacekk.info> Date: Thu, 07 Feb 2019 23:27:25 +0000 Subject: [PATCH] Get rid of bash helper that downloads files from FTP/HTTP --- lib/vehicle_types.php | 109 +++++++++ lib/mapper.php | 138 ++++++++++++ parse.php | 177 +++------------ /dev/null | 15 - lib/fetch.php | 86 +++++++ composer.json | 3 common.php | 109 --------- 7 files changed, 376 insertions(+), 261 deletions(-) diff --git a/common.php b/common.php index 70f2eef..1fb3bdd 100644 --- a/common.php +++ b/common.php @@ -1,109 +1,2 @@ <?php -function numToType($id, $data, $defaultLow=NULL) { - $data = explode("\n", trim($data)); - foreach($data as $line) { - $line = explode("\t", trim($line)); - if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) { - return [ - 'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT), - 'type' => $line[3], - 'low' => isset($line[4]) ? $line[4] : $defaultLow, - ]; - } - } - return []; -} -function numToTypeT($id) { -$data = <<<'END' -101 107 HW E1 0 -108 113 RW E1 0 -114 126 HW E1 0 -127 127 RW E1 0 -128 130 HW E1 0 -131 132 RW E1 0 -133 133 HW E1 0 -134 134 RW E1 0 -135 136 HW E1 0 -137 139 RW E1 0 -140 147 HW E1 0 -148 150 RW E1 0 -151 152 HW E1 0 -153 153 RW E1 0 -154 154 HW E1 0 -155 155 RW E1 0 -156 158 HW E1 0 -159 159 RW E1 0 -160 174 HW E1 0 -201 245 RZ 105N 0 -246 299 HZ 105N 0 -301 312 RF GT8S 0 -313 313 RF GT8C 1 -314 322 RF GT8S 0 -323 323 RF GT8N 1 -324 324 RF GT8S 0 -325 328 RF GT8N 1 -401 440 HL EU8N 1 -451 456 HK N8C-NF 0 -457 461 HK N8S-NF 1 -462 462 HK N8C-NF 0 -601 614 RP NGT6 (1) 2 -615 626 RP NGT6 (2) 2 -627 650 RP NGT6 (3) 2 -801 824 RY NGT8 2 -899 899 RY 126N 2 -901 914 RG 2014N 2 -915 936 HG 2014N 2 -999 999 HG 405N 1 -END; - return numToType($id, $data); -} -function numToTypeB($id) { -$data = <<<'END' -2 4 DN Solaris Urbino 18 IV Electric -71 83 BH Solaris Urbino 18 III Hybrid -84 96 BH Volvo 7900A Hybrid -103 105 PA Mercedes-Benz 516 -106 112 DA Autosan M09LE -113 121 BA Autosan M09LE -122 128 DA Autosan M09LE -129 139 BA Autosan M09LE -141 146 PM MAN NL283 Lion's City -200 200 DO Mercedes Conecto -206 210 PO Mercedes O530 C2 Hybrid -211 218 DO Mercedes O530 -219 243 PO Mercedes O530 C2 Hybrid -244 269 DO Mercedes O530 C2 -270 299 BO Mercedes O530 C2 -301 338 DU Solaris Urbino 12 IV -339 340 BU Solaris Urbino 12 IV -341 345 DU Solaris Urbino 12 III -400 403 BH Solaris Urbino 12,9 III Hybrid -404 408 DH Solaris Urbino 12,9 III Hybrid -501 510 BR Solaris Urbino 18 IV -511 568 DR Solaris Urbino 18 IV -569 579 BR Solaris Urbino 18 IV -580 595 DR Solaris Urbino 18 IV -601 601 DE Solaris Urbino 12 III Electric -602 605 DE Solaris Urbino 8,9LE Electric -606 606 DE Solaris Urbino 12 III Electric -607 623 DE Solaris Urbino 12 IV Electric -700 700 DC Mercedes Conecto G -701 731 DC Mercedes O530G -732 732 DC Mercedes Conecto G -737 741 BR Solaris Urbino 18 III -742 745 DR Solaris Urbino 18 III -746 764 PR Solaris Urbino 18 III -765 768 DR Solaris Urbino 18 III -769 776 PR Solaris Urbino 18 MetroStyle -777 777 DR Solaris Urbino 18 III -778 797 PR Solaris Urbino 18 IV -851 903 BU Solaris Urbino 12 III -904 905 DU Solaris Urbino 12 III -906 926 BU Solaris Urbino 12 III -927 976 PU Solaris Urbino 12 III -977 977 DU Solaris Urbino 12 III -978 991 PU Solaris Urbino 12 IV -992 997 BU Solaris Urbino 12 IV -END; - return numToType($id, $data, 2); -} +require_once(__DIR__.'/lib/vehicle_types.php'); diff --git a/composer.json b/composer.json index f001973..277e92a 100644 --- a/composer.json +++ b/composer.json @@ -1,5 +1,6 @@ { "require": { - "google/gtfs-realtime-bindings": "^0.0.2" + "google/gtfs-realtime-bindings": "^0.0.2", + "monolog/monolog": "^1.24" } } diff --git a/lib/fetch.php b/lib/fetch.php new file mode 100644 index 0000000..9c54486 --- /dev/null +++ b/lib/fetch.php @@ -0,0 +1,86 @@ +<?php +function ftp_fetch_if_newer($url, $file = NULL) { + $url = parse_url($url); + if(!isset($url['scheme']) || $url['scheme'] != 'ftp') { + throw new Exception('Only FTP URLs are supported'); + } + if(!isset($url['host'])) { + throw new Exception('Hostname not present in the URL'); + } + if(!isset($url['path'])) { + throw new Exception('Path component not present in the URL'); + } + if(!isset($url['port'])) { + $url['port'] = 21; + } + if(!isset($url['user'])) { + $url['user'] = 'anonymous'; + } + if(!isset($url['pass'])) { + $url['pass'] = 'anonymous@mpk.jacekk.net'; + } + if($file == NULL) { + $file = basename($url['path']); + } + + $localTime = -1; + $localSize = -1; + if(is_file($file)) { + $localTime = filemtime($file); + $localSize = filesize($file); + } + + $ftp = ftp_connect($url['host'], $url['port'], 10); + if($ftp === FALSE) { + throw new Exception('FTP connection failed'); + } + if(!ftp_login($ftp, $url['user'], $url['pass'])) { + throw new Exception('FTP login failed'); + } + if(!ftp_pasv($ftp, TRUE)) { + throw new Exception('Passive FTP request failed'); + } + $remoteSize = ftp_size($ftp, $url['path']); + if($remoteSize < 0) { + throw new Exception('FTP file size fetch failed'); + } + $remoteTime = ftp_mdtm($ftp, $url['path']); + if($remoteTime < 0) { + throw new Exception('FTP modification time fetch failed'); + } + + $updated = FALSE; + + if($localSize != $remoteSize || $localTime < $remoteTime) { + if(file_exists($file.'.tmp')) { + unlink($file.'.tmp'); + } + if(ftp_get($ftp, $file.'.tmp', $url['path'], FTP_BINARY)) { + touch($file.'.tmp', $remoteTime); + if(!rename($file.'.tmp', $file)) { + throw new Exception('Temporary file rename failed'); + } + $updated = TRUE; + } + } + + ftp_close($ftp); + + return $updated; +} + +function fetch($url, $file = NULL) { + if($file == NULL) { + $file = basename($url['url']); + } + $data = file_get_contents($url); + if($data === FALSE) { + throw new Exception('URL fetch failed'); + } + if(file_put_contents($file.'.tmp', $data) === FALSE) { + throw new Exception('Temporary file creation failed'); + } + if(!rename($file.'.tmp', $file)) { + throw new Exception('Temporary file rename failed'); + } +} diff --git a/lib/mapper.php b/lib/mapper.php new file mode 100644 index 0000000..a993f4c --- /dev/null +++ b/lib/mapper.php @@ -0,0 +1,138 @@ +<?php +require_once(__DIR__.'/../vendor/autoload.php'); +require_once(__DIR__.'/vehicle_types.php'); + +use transit_realtime\FeedMessage; + +class Mapper { + private $jsonTrips = []; + private $gtfsTrips = []; + private $logger = NULL; + + private $specialNames = [ + 'Zjazd do zajezdni', + 'Przejazd techniczny', + 'Wyjazd na trasÄ™', + ]; + + public function __construct() { + $this->logger = new Monolog\Logger(__CLASS__); + } + + public static function convertTripId($tripId) { + $tripId = explode('_', $tripId); + if($tripId[0] != 'block') return; + if($tripId[2] != 'trip') return; + return 4096 * (int)$tripId[1] + (int)$tripId[3]; + } + + public function loadTTSS($file) { + $json = json_decode(file_get_contents($file)); + foreach($json->vehicles as $vehicle) { + if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue; + if(!isset($vehicle->tripId) || !$vehicle->tripId) continue; + if(!isset($vehicle->name) || !$vehicle->name) continue; + if(!isset($vehicle->latitude) || !$vehicle->latitude) continue; + if(!isset($vehicle->longitude) || !$vehicle->longitude) continue; + foreach($this->specialNames as $name) { + if(substr($vehicle->name, -strlen($name)) == $name) { + continue; + } + } + $this->jsonTrips[(int)$vehicle->tripId] = [ + 'id' => $vehicle->id, + 'latitude' => (float)$vehicle->latitude / 3600000.0, + 'longitude' => (float)$vehicle->longitude / 3600000.0, + ]; + } + ksort($this->jsonTrips); + } + + public function loadGTFS($file) { + $data = file_get_contents($file); + $feed = new FeedMessage(); + $feed->parse($data); + foreach ($feed->getEntityList() as $entity) { + $vehiclePosition = $entity->getVehicle(); + $position = $vehiclePosition->getPosition(); + $vehicle = $vehiclePosition->getVehicle(); + $trip = $vehiclePosition->getTrip(); + $tripId = $trip->getTripId(); + $this->gtfsTrips[self::convertTripId($tripId)] = [ + 'id' => $entity->getId(), + 'num' => $vehicle->getLicensePlate(), + 'tripId' => $tripId, + 'latitude' => $position->getLatitude(), + 'longitude' => $position->getLongitude(), + ]; + } + ksort($this->gtfsTrips); + } + + public function findOffset() { + if(count($this->jsonTrips) == 0 || count($this->gtfsTrips) == 0) { + return NULL; + } + + $jsonTripIds = array_keys($this->jsonTrips); + $gtfsTripIds = array_keys($this->gtfsTrips); + + $possibleOffsets = []; + for($i = 0; $i < count($this->jsonTrips); $i++) { + for($j = 0; $j < count($this->gtfsTrips); $j++) { + $possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE; + } + } + $possibleOffsets = array_keys($possibleOffsets); + + $bestOffset = 0; + $maxMatched = 0; + $options = 0; + + foreach($possibleOffsets as $offset) { + $matched = 0; + + foreach($gtfsTripIds as $tripId) { + $tripId += $offset; + if(isset($this->jsonTrips[$tripId])) { + $matched++; + } + } + + if($matched > $maxMatched) { + $bestOffset = $offset; + $maxMatched = $matched; + $options = 1; + } elseif($matched == $maxMatched) { + $options++; + } + } + + if($options != 1) { + throw new Exception('Found '.$options.' possible mappings!'); + } + return $bestOffset; + } + + public function getMapping($offset) { + $result = []; + foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) { + $jsonTripId = $gtfsTripId + $offset; + if(isset($this->jsonTrips[$jsonTripId])) { + $data = numToTypeB($gtfsTrip['id']); + $num = $gtfsTrip['num']; + if(!is_array($data) || !isset($data['num'])) { + $data = [ + 'num' => $num, + 'low' => 2, + ]; + } elseif($data['num'] != $num) { + // Ignore due to incorrect depot markings in the data + //$this->logger->warn('Got '.$num.', database has '.$data['num']); + } + $result[$this->jsonTrips[$jsonTripId]['id']] = $data; + } + } + return $result; + } +} diff --git a/lib/vehicle_types.php b/lib/vehicle_types.php new file mode 100644 index 0000000..70f2eef --- /dev/null +++ b/lib/vehicle_types.php @@ -0,0 +1,109 @@ +<?php +function numToType($id, $data, $defaultLow=NULL) { + $data = explode("\n", trim($data)); + foreach($data as $line) { + $line = explode("\t", trim($line)); + if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) { + return [ + 'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT), + 'type' => $line[3], + 'low' => isset($line[4]) ? $line[4] : $defaultLow, + ]; + } + } + return []; +} +function numToTypeT($id) { +$data = <<<'END' +101 107 HW E1 0 +108 113 RW E1 0 +114 126 HW E1 0 +127 127 RW E1 0 +128 130 HW E1 0 +131 132 RW E1 0 +133 133 HW E1 0 +134 134 RW E1 0 +135 136 HW E1 0 +137 139 RW E1 0 +140 147 HW E1 0 +148 150 RW E1 0 +151 152 HW E1 0 +153 153 RW E1 0 +154 154 HW E1 0 +155 155 RW E1 0 +156 158 HW E1 0 +159 159 RW E1 0 +160 174 HW E1 0 +201 245 RZ 105N 0 +246 299 HZ 105N 0 +301 312 RF GT8S 0 +313 313 RF GT8C 1 +314 322 RF GT8S 0 +323 323 RF GT8N 1 +324 324 RF GT8S 0 +325 328 RF GT8N 1 +401 440 HL EU8N 1 +451 456 HK N8C-NF 0 +457 461 HK N8S-NF 1 +462 462 HK N8C-NF 0 +601 614 RP NGT6 (1) 2 +615 626 RP NGT6 (2) 2 +627 650 RP NGT6 (3) 2 +801 824 RY NGT8 2 +899 899 RY 126N 2 +901 914 RG 2014N 2 +915 936 HG 2014N 2 +999 999 HG 405N 1 +END; + return numToType($id, $data); +} +function numToTypeB($id) { +$data = <<<'END' +2 4 DN Solaris Urbino 18 IV Electric +71 83 BH Solaris Urbino 18 III Hybrid +84 96 BH Volvo 7900A Hybrid +103 105 PA Mercedes-Benz 516 +106 112 DA Autosan M09LE +113 121 BA Autosan M09LE +122 128 DA Autosan M09LE +129 139 BA Autosan M09LE +141 146 PM MAN NL283 Lion's City +200 200 DO Mercedes Conecto +206 210 PO Mercedes O530 C2 Hybrid +211 218 DO Mercedes O530 +219 243 PO Mercedes O530 C2 Hybrid +244 269 DO Mercedes O530 C2 +270 299 BO Mercedes O530 C2 +301 338 DU Solaris Urbino 12 IV +339 340 BU Solaris Urbino 12 IV +341 345 DU Solaris Urbino 12 III +400 403 BH Solaris Urbino 12,9 III Hybrid +404 408 DH Solaris Urbino 12,9 III Hybrid +501 510 BR Solaris Urbino 18 IV +511 568 DR Solaris Urbino 18 IV +569 579 BR Solaris Urbino 18 IV +580 595 DR Solaris Urbino 18 IV +601 601 DE Solaris Urbino 12 III Electric +602 605 DE Solaris Urbino 8,9LE Electric +606 606 DE Solaris Urbino 12 III Electric +607 623 DE Solaris Urbino 12 IV Electric +700 700 DC Mercedes Conecto G +701 731 DC Mercedes O530G +732 732 DC Mercedes Conecto G +737 741 BR Solaris Urbino 18 III +742 745 DR Solaris Urbino 18 III +746 764 PR Solaris Urbino 18 III +765 768 DR Solaris Urbino 18 III +769 776 PR Solaris Urbino 18 MetroStyle +777 777 DR Solaris Urbino 18 III +778 797 PR Solaris Urbino 18 IV +851 903 BU Solaris Urbino 12 III +904 905 DU Solaris Urbino 12 III +906 926 BU Solaris Urbino 12 III +927 976 PU Solaris Urbino 12 III +977 977 DU Solaris Urbino 12 III +978 991 PU Solaris Urbino 12 IV +992 997 BU Solaris Urbino 12 IV +END; + return numToType($id, $data, 2); +} diff --git a/parse.php b/parse.php index df4e100..c1232f8 100644 --- a/parse.php +++ b/parse.php @@ -1,142 +1,45 @@ <?php -require('vendor/autoload.php'); -require('common.php'); +require_once(__DIR__.'/lib/fetch.php'); +require_once(__DIR__.'/lib/mapper.php'); -use transit_realtime\FeedMessage; +$logger = new Monolog\Logger('Parse changes'); -class IdMapper { - private $jsonTrips = []; - private $gtfsTrips = []; - - private $specialNames = [ - 'Zjazd do zajezdni', - 'Przejazd techniczny', - 'Wyjazd na trasÄ™', - ]; - - public static function convertTripId($tripId) { - $tripId = explode('_', $tripId); - if($tripId[0] != 'block') return; - if($tripId[2] != 'trip') return; - return 4096 * (int)$tripId[1] + (int)$tripId[3]; - } - - public function loadJson($file) { - $json = json_decode(file_get_contents($file)); - foreach($json->vehicles as $vehicle) { - if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue; - if(!isset($vehicle->tripId) || !$vehicle->tripId) continue; - if(!isset($vehicle->name) || !$vehicle->name) continue; - if(!isset($vehicle->latitude) || !$vehicle->latitude) continue; - if(!isset($vehicle->longitude) || !$vehicle->longitude) continue; - foreach($this->specialNames as $name) { - if(substr($vehicle->name, -strlen($name)) == $name) { - continue; - } - } - $this->jsonTrips[(int)$vehicle->tripId] = [ - 'id' => $vehicle->id, - 'latitude' => (float)$vehicle->latitude / 3600000.0, - 'longitude' => (float)$vehicle->longitude / 3600000.0, - ]; - } - ksort($this->jsonTrips); - } - - public function loadGtfs($file) { - $data = file_get_contents($file); - $feed = new FeedMessage(); - $feed->parse($data); - foreach ($feed->getEntityList() as $entity) { - $vehiclePosition = $entity->getVehicle(); - $position = $vehiclePosition->getPosition(); - $vehicle = $vehiclePosition->getVehicle(); - $trip = $vehiclePosition->getTrip(); - $tripId = $trip->getTripId(); - $this->gtfsTrips[self::convertTripId($tripId)] = [ - 'id' => $entity->getId(), - 'num' => $vehicle->getLicensePlate(), - 'tripId' => $tripId, - 'latitude' => $position->getLatitude(), - 'longitude' => $position->getLongitude(), - ]; - } - ksort($this->gtfsTrips); - } - - public function findOffset() { - if(count($this->jsonTrips) == 0 || count($this->gtfsTrips) == 0) { - return NULL; - } - - $jsonTripIds = array_keys($this->jsonTrips); - $gtfsTripIds = array_keys($this->gtfsTrips); - - $possibleOffsets = []; - for($i = 0; $i < count($this->jsonTrips); $i++) { - for($j = 0; $j < count($this->gtfsTrips); $j++) { - $possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE; - } - } - $possibleOffsets = array_keys($possibleOffsets); - - $bestOffset = 0; - $maxMatched = 0; - $options = 0; - - foreach($possibleOffsets as $offset) { - $matched = 0; - - foreach($gtfsTripIds as $tripId) { - $tripId += $offset; - if(isset($this->jsonTrips[$tripId])) { - $matched++; - } - } - - if($matched > $maxMatched) { - $bestOffset = $offset; - $maxMatched = $matched; - $options = 1; - } elseif($matched == $maxMatched) { - $options++; - } - } - - if($options != 1) { - fwrite(STDERR, 'Found '.$options.' possible mappings!'."\n"); - return FALSE; - } - return $bestOffset; - } - - public function getMapping($offset) { - $result = []; - foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) { - $jsonTripId = $gtfsTripId + $offset; - if(isset($this->jsonTrips[$jsonTripId])) { - $data = numToTypeB($gtfsTrip['id']); - $num = $gtfsTrip['num']; - if(!is_array($data) || !isset($data['num'])) { - $data = [ - 'num' => $num, - 'low' => 2, - ]; - } elseif($data['num'] != $num) { - // Ignore due to incorrect depot markings in the data - //fwrite(STDERR, 'Got '.$num.', database has '.$data['num']."\n"); - } - $result[$this->jsonTrips[$jsonTripId]['id']] = $data; - } - } - return $result; - } -} +$sources = [ + 'buses' => [ + 'gtfs' => 'ftp://ztp.krakow.pl/VehiclePositions_A.pb', + 'gtfs_file' => 'VehiclePositions_A.pb', + 'ttss' => 'http://91.223.13.70/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles', + 'ttss_file' => 'vehicles_A.json', + ], +]; -$mapper = new IdMapper(); -$mapper->loadJson('./data/vehicles_A.json'); -$mapper->loadGtfs('./data/VehiclePositions_A.pb'); -$offset = $mapper->findOffset(); -if($offset) { - echo json_encode($mapper->getMapping($offset)); +foreach($sources as $name => $source) { + $logger = new Monolog\Logger('fetch_'.$name); + try { + $logger->info('Fetching '.$name.' position data from FTP...'); + $updated = ftp_fetch_if_newer($source['gtfs'], __DIR__.'/data/'.$source['gtfs_file']); + if(!$updated) { + $logger->info('Nothing to do, remote file not newer than local one'); + continue; + } + + $logger->info('Fetching '.$name.' positions from TTSS...'); + fetch($source['ttss'], __DIR__.'/data/'.$source['ttss_file']); + + $logger->info('Loading data...'); + $mapper = new Mapper(); + $mapper->loadTTSS(__DIR__.'/data/'.$source['ttss_file']); + $mapper->loadGTFS(__DIR__.'/data/'.$source['gtfs_file']); + + $logger->info('Finding correct offset...'); + $offset = $mapper->findOffset(); + if($offset) { + $logger->info('Got offset '.$offset.', creating mapping...'); + $mapping = $mapper->getMapping($offset); + echo json_encode($mapping); + } + $logger->info('Finished'); + } catch(Throwable $e) { + $logger->error($e->getMessage(), ['exception' => $e, 'exception_string' => (string)$e]); + } } diff --git a/parse.sh b/parse.sh deleted file mode 100755 index af8bf4a..0000000 --- a/parse.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -DIR=`dirname "$0"` -cd "$DIR" -DIR=`pwd` - -cd "$DIR/data" -wget -O vehicles_A.json "http://91.223.13.70/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles" -wget -N "ftp://ztp.krakow.pl/VehiclePositions_A.pb" - -cd "$DIR" -php parse.php > data/mapping_A.tmp -if [ -s data/mapping_A.tmp ]; then - mv -f data/mapping_A.tmp data/mapping_A.json -fi -- Gitblit v1.9.1