ttss/mpk-ttss-mapping - git.Jacekk.net

parent: 1073190b | patch | commit | show whitespace

Get rid of bash helper that downloads files from FTP/HTTP

Jacek Kowalski

2019-02-07 4673cc38e7b5b1b87d8e009137fc2d5eae688fd7

Get rid of bash helper that downloads files from FTP/HTTP

Additionally only parse data when GTFS file has changed
since last download. Some refactoring was also done.

1 files deleted

3 files modified

3 files added

	common.php	109 ●●●●● patch \| view \| raw \| blame \| history
	composer.json	3 ●●●●● patch \| view \| raw \| blame \| history
	lib/fetch.php	86 ●●●●● patch \| view \| raw \| blame \| history
	lib/mapper.php	138 ●●●●● patch \| view \| raw \| blame \| history
	lib/vehicle_types.php	109 ●●●●● patch \| view \| raw \| blame \| history
	parse.php	161 ●●●●● patch \| view \| raw \| blame \| history
	parse.sh	15 ●●●●● patch \| view \| raw \| blame \| history

 common.php

@@ -1,109 +1,2 @@
<?php
function numToType($id, $data, $defaultLow=NULL) {
    $data = explode("\n", trim($data));
    foreach($data as $line) {
        $line = explode("\t", trim($line));
        if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) {
            return [
                'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT),
                'type' => $line[3],
                'low' => isset($line[4]) ? $line[4] : $defaultLow,
            ];
        }
    }
    return [];
}
function numToTypeT($id) {
$data = <<<'END'
101    107    HW    E1    0
108    113    RW    E1    0
114    126    HW    E1    0
127    127    RW    E1    0
128    130    HW    E1    0
131    132    RW    E1    0
133    133    HW    E1    0
134    134    RW    E1    0
135    136    HW    E1    0
137    139    RW    E1    0
140    147    HW    E1    0
148    150    RW    E1    0
151    152    HW    E1    0
153    153    RW    E1    0
154    154    HW    E1    0
155    155    RW    E1    0
156    158    HW    E1    0
159    159    RW    E1    0
160    174    HW    E1    0
201    245    RZ    105N    0
246    299    HZ    105N    0
301    312    RF    GT8S    0
313    313    RF    GT8C    1
314    322    RF    GT8S    0
323    323    RF    GT8N    1
324    324    RF    GT8S    0
325    328    RF    GT8N    1
401    440    HL    EU8N    1
451    456    HK    N8C-NF    0
457    461    HK    N8S-NF    1
462    462    HK    N8C-NF    0
601    614    RP    NGT6 (1)    2
615    626    RP    NGT6 (2)    2
627    650    RP    NGT6 (3)    2
801    824    RY    NGT8    2
899    899    RY    126N    2
901    914    RG    2014N    2
915    936    HG    2014N    2
999    999    HG    405N    1
END;
    return numToType($id, $data);
}
function numToTypeB($id) {
$data = <<<'END'
2    4    DN    Solaris Urbino 18 IV Electric
71    83    BH    Solaris Urbino 18 III Hybrid
84    96    BH    Volvo 7900A Hybrid
103    105    PA    Mercedes-Benz 516
106    112    DA    Autosan M09LE
113    121    BA    Autosan M09LE
122    128    DA    Autosan M09LE
129    139    BA    Autosan M09LE
141    146    PM    MAN NL283 Lion's City
200    200    DO    Mercedes Conecto
206    210    PO    Mercedes O530 C2 Hybrid
211    218    DO    Mercedes O530
219    243    PO    Mercedes O530 C2 Hybrid
244    269    DO    Mercedes O530 C2
270    299    BO    Mercedes O530 C2
301    338    DU    Solaris Urbino 12 IV
339    340    BU    Solaris Urbino 12 IV
341    345    DU    Solaris Urbino 12 III
400    403    BH    Solaris Urbino 12,9 III Hybrid
404    408    DH    Solaris Urbino 12,9 III Hybrid
501    510    BR    Solaris Urbino 18 IV
511    568    DR    Solaris Urbino 18 IV
569    579    BR    Solaris Urbino 18 IV
580    595    DR    Solaris Urbino 18 IV
601    601    DE    Solaris Urbino 12 III Electric
602    605    DE    Solaris Urbino 8,9LE Electric
606    606    DE    Solaris Urbino 12 III Electric
607    623    DE    Solaris Urbino 12 IV Electric
700    700    DC    Mercedes Conecto G
701    731    DC    Mercedes O530G
732    732    DC    Mercedes Conecto G
737    741    BR    Solaris Urbino 18 III
742    745    DR    Solaris Urbino 18 III
746    764    PR    Solaris Urbino 18 III
765    768    DR    Solaris Urbino 18 III
769    776    PR    Solaris Urbino 18 MetroStyle
777    777    DR    Solaris Urbino 18 III
778    797    PR    Solaris Urbino 18 IV
851    903    BU    Solaris Urbino 12 III
904    905    DU    Solaris Urbino 12 III
906    926    BU    Solaris Urbino 12 III
927    976    PU    Solaris Urbino 12 III
977    977    DU    Solaris Urbino 12 III
978    991    PU    Solaris Urbino 12 IV
992    997    BU    Solaris Urbino 12 IV
END;
    return numToType($id, $data, 2);
}
require_once(__DIR__.'/lib/vehicle_types.php');

 composer.json

@@ -1,5 +1,6 @@
{
    "require": {
        "google/gtfs-realtime-bindings": "^0.0.2"
        "google/gtfs-realtime-bindings": "^0.0.2",
        "monolog/monolog": "^1.24"
    }
}

 lib/fetch.php

New file
@@ -0,0 +1,86 @@
<?php
function ftp_fetch_if_newer($url, $file = NULL) {
    $url = parse_url($url);
    if(!isset($url['scheme']) || $url['scheme'] != 'ftp') {
        throw new Exception('Only FTP URLs are supported');
    }
    if(!isset($url['host'])) {
        throw new Exception('Hostname not present in the URL');
    }
    if(!isset($url['path'])) {
        throw new Exception('Path component not present in the URL');
    }
    if(!isset($url['port'])) {
        $url['port'] = 21;
    }
    if(!isset($url['user'])) {
        $url['user'] = 'anonymous';
    }
    if(!isset($url['pass'])) {
        $url['pass'] = 'anonymous@mpk.jacekk.net';
    }
    if($file == NULL) {
        $file = basename($url['path']);
    }
	
    $localTime = -1;
    $localSize = -1;
    if(is_file($file)) {
        $localTime = filemtime($file);
        $localSize = filesize($file);
    }
	
    $ftp = ftp_connect($url['host'], $url['port'], 10);
    if($ftp === FALSE) {
        throw new Exception('FTP connection failed');
    }
    if(!ftp_login($ftp, $url['user'], $url['pass'])) {
        throw new Exception('FTP login failed');
    }
    if(!ftp_pasv($ftp, TRUE)) {
        throw new Exception('Passive FTP request failed');
    }
    $remoteSize = ftp_size($ftp, $url['path']);
    if($remoteSize < 0) {
        throw new Exception('FTP file size fetch failed');
    }
    $remoteTime = ftp_mdtm($ftp, $url['path']);
    if($remoteTime < 0) {
        throw new Exception('FTP modification time fetch failed');
    }
	
    $updated = FALSE;
	
    if($localSize != $remoteSize || $localTime < $remoteTime) {
        if(file_exists($file.'.tmp')) {
            unlink($file.'.tmp');
        }
        if(ftp_get($ftp, $file.'.tmp', $url['path'], FTP_BINARY)) {
            touch($file.'.tmp', $remoteTime);
            if(!rename($file.'.tmp', $file)) {
                throw new Exception('Temporary file rename failed');
            }
            $updated = TRUE;
        }
    }
	
    ftp_close($ftp);
	
    return $updated;
}

function fetch($url, $file = NULL) {
    if($file == NULL) {
        $file = basename($url['url']);
    }
    $data = file_get_contents($url);
    if($data === FALSE) {
        throw new Exception('URL fetch failed');
    }
    if(file_put_contents($file.'.tmp', $data) === FALSE) {
        throw new Exception('Temporary file creation failed');
    }
    if(!rename($file.'.tmp', $file)) {
        throw new Exception('Temporary file rename failed');
    }
}

 lib/mapper.php

New file
@@ -0,0 +1,138 @@
<?php
require_once(__DIR__.'/../vendor/autoload.php');
require_once(__DIR__.'/vehicle_types.php');

use transit_realtime\FeedMessage;

class Mapper {
    private $jsonTrips = [];
    private $gtfsTrips = [];
    private $logger = NULL;
	
    private $specialNames = [
        'Zjazd do zajezdni',
        'Przejazd techniczny',
        'Wyjazd na trasę',
    ];
	
    public function __construct() {
        $this->logger = new Monolog\Logger(__CLASS__);
    }
	
    public static function convertTripId($tripId) {
        $tripId = explode('_', $tripId);
        if($tripId[0] != 'block') return;
        if($tripId[2] != 'trip') return;
        return 4096 * (int)$tripId[1] + (int)$tripId[3];
    }
	
    public function loadTTSS($file) {
        $json = json_decode(file_get_contents($file));
        foreach($json->vehicles as $vehicle) {
            if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue;
            if(!isset($vehicle->tripId) || !$vehicle->tripId) continue;
            if(!isset($vehicle->name) || !$vehicle->name) continue;
            if(!isset($vehicle->latitude) || !$vehicle->latitude) continue;
            if(!isset($vehicle->longitude) || !$vehicle->longitude) continue;
            foreach($this->specialNames as $name) {
                if(substr($vehicle->name, -strlen($name)) == $name) {
                    continue;
                }
            }
            $this->jsonTrips[(int)$vehicle->tripId] = [
                'id' => $vehicle->id,
                'latitude' => (float)$vehicle->latitude / 3600000.0,
                'longitude' => (float)$vehicle->longitude / 3600000.0,
            ];
        }
        ksort($this->jsonTrips);
    }
	
    public function loadGTFS($file) {
        $data = file_get_contents($file);
        $feed = new FeedMessage();
        $feed->parse($data);
        foreach ($feed->getEntityList() as $entity) {
            $vehiclePosition = $entity->getVehicle();
            $position = $vehiclePosition->getPosition();
            $vehicle = $vehiclePosition->getVehicle();
            $trip = $vehiclePosition->getTrip();
            $tripId = $trip->getTripId();
            $this->gtfsTrips[self::convertTripId($tripId)] = [
                'id' => $entity->getId(),
                'num' => $vehicle->getLicensePlate(),
                'tripId' => $tripId,
                'latitude' => $position->getLatitude(),
                'longitude' => $position->getLongitude(),
            ];
        }
        ksort($this->gtfsTrips);
    }
	
    public function findOffset() {
        if(count($this->jsonTrips) == 0 || count($this->gtfsTrips) == 0) {
            return NULL;
        }
		
        $jsonTripIds = array_keys($this->jsonTrips);
        $gtfsTripIds = array_keys($this->gtfsTrips);
		
        $possibleOffsets = [];
        for($i = 0; $i < count($this->jsonTrips); $i++) {
            for($j = 0; $j < count($this->gtfsTrips); $j++) {
                $possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE;
            }
        }
        $possibleOffsets = array_keys($possibleOffsets);
		
        $bestOffset = 0;
        $maxMatched = 0;
        $options = 0;
		
        foreach($possibleOffsets as $offset) {
            $matched = 0;
			
            foreach($gtfsTripIds as $tripId) {
                $tripId += $offset;
                if(isset($this->jsonTrips[$tripId])) {
                    $matched++;
                }
            }
			
            if($matched > $maxMatched) {
                $bestOffset = $offset;
                $maxMatched = $matched;
                $options = 1;
            } elseif($matched == $maxMatched) {
                $options++;
            }
        }
		
        if($options != 1) {
            throw new Exception('Found '.$options.' possible mappings!');
        }
        return $bestOffset;
    }
	
    public function getMapping($offset) {
        $result = [];
        foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) {
            $jsonTripId = $gtfsTripId + $offset;
            if(isset($this->jsonTrips[$jsonTripId])) {
                $data = numToTypeB($gtfsTrip['id']);
                $num = $gtfsTrip['num'];
                if(!is_array($data) || !isset($data['num'])) {
                    $data = [
                        'num' => $num,
                        'low' => 2,
                    ];
                } elseif($data['num'] != $num) {
                    // Ignore due to incorrect depot markings in the data
                    //$this->logger->warn('Got '.$num.', database has '.$data['num']);
                }
                $result[$this->jsonTrips[$jsonTripId]['id']] = $data;
            }
        }
        return $result;
    }
}

 lib/vehicle_types.php

New file
@@ -0,0 +1,109 @@
<?php
function numToType($id, $data, $defaultLow=NULL) {
    $data = explode("\n", trim($data));
    foreach($data as $line) {
        $line = explode("\t", trim($line));
        if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) {
            return [
                'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT),
                'type' => $line[3],
                'low' => isset($line[4]) ? $line[4] : $defaultLow,
            ];
        }
    }
    return [];
}
function numToTypeT($id) {
$data = <<<'END'
101    107    HW    E1    0
108    113    RW    E1    0
114    126    HW    E1    0
127    127    RW    E1    0
128    130    HW    E1    0
131    132    RW    E1    0
133    133    HW    E1    0
134    134    RW    E1    0
135    136    HW    E1    0
137    139    RW    E1    0
140    147    HW    E1    0
148    150    RW    E1    0
151    152    HW    E1    0
153    153    RW    E1    0
154    154    HW    E1    0
155    155    RW    E1    0
156    158    HW    E1    0
159    159    RW    E1    0
160    174    HW    E1    0
201    245    RZ    105N    0
246    299    HZ    105N    0
301    312    RF    GT8S    0
313    313    RF    GT8C    1
314    322    RF    GT8S    0
323    323    RF    GT8N    1
324    324    RF    GT8S    0
325    328    RF    GT8N    1
401    440    HL    EU8N    1
451    456    HK    N8C-NF    0
457    461    HK    N8S-NF    1
462    462    HK    N8C-NF    0
601    614    RP    NGT6 (1)    2
615    626    RP    NGT6 (2)    2
627    650    RP    NGT6 (3)    2
801    824    RY    NGT8    2
899    899    RY    126N    2
901    914    RG    2014N    2
915    936    HG    2014N    2
999    999    HG    405N    1
END;
    return numToType($id, $data);
}
function numToTypeB($id) {
$data = <<<'END'
2    4    DN    Solaris Urbino 18 IV Electric
71    83    BH    Solaris Urbino 18 III Hybrid
84    96    BH    Volvo 7900A Hybrid
103    105    PA    Mercedes-Benz 516
106    112    DA    Autosan M09LE
113    121    BA    Autosan M09LE
122    128    DA    Autosan M09LE
129    139    BA    Autosan M09LE
141    146    PM    MAN NL283 Lion's City
200    200    DO    Mercedes Conecto
206    210    PO    Mercedes O530 C2 Hybrid
211    218    DO    Mercedes O530
219    243    PO    Mercedes O530 C2 Hybrid
244    269    DO    Mercedes O530 C2
270    299    BO    Mercedes O530 C2
301    338    DU    Solaris Urbino 12 IV
339    340    BU    Solaris Urbino 12 IV
341    345    DU    Solaris Urbino 12 III
400    403    BH    Solaris Urbino 12,9 III Hybrid
404    408    DH    Solaris Urbino 12,9 III Hybrid
501    510    BR    Solaris Urbino 18 IV
511    568    DR    Solaris Urbino 18 IV
569    579    BR    Solaris Urbino 18 IV
580    595    DR    Solaris Urbino 18 IV
601    601    DE    Solaris Urbino 12 III Electric
602    605    DE    Solaris Urbino 8,9LE Electric
606    606    DE    Solaris Urbino 12 III Electric
607    623    DE    Solaris Urbino 12 IV Electric
700    700    DC    Mercedes Conecto G
701    731    DC    Mercedes O530G
732    732    DC    Mercedes Conecto G
737    741    BR    Solaris Urbino 18 III
742    745    DR    Solaris Urbino 18 III
746    764    PR    Solaris Urbino 18 III
765    768    DR    Solaris Urbino 18 III
769    776    PR    Solaris Urbino 18 MetroStyle
777    777    DR    Solaris Urbino 18 III
778    797    PR    Solaris Urbino 18 IV
851    903    BU    Solaris Urbino 12 III
904    905    DU    Solaris Urbino 12 III
906    926    BU    Solaris Urbino 12 III
927    976    PU    Solaris Urbino 12 III
977    977    DU    Solaris Urbino 12 III
978    991    PU    Solaris Urbino 12 IV
992    997    BU    Solaris Urbino 12 IV
END;
    return numToType($id, $data, 2);
}

 parse.php

@@ -1,142 +1,45 @@
<?php
require('vendor/autoload.php');
require('common.php');
require_once(__DIR__.'/lib/fetch.php');
require_once(__DIR__.'/lib/mapper.php');

use transit_realtime\FeedMessage;
$logger = new Monolog\Logger('Parse changes');

class IdMapper {
    private $jsonTrips = [];
    private $gtfsTrips = [];
	
    private $specialNames = [
        'Zjazd do zajezdni',
        'Przejazd techniczny',
        'Wyjazd na trasę',
$sources = [
    'buses' => [
        'gtfs' => 'ftp://ztp.krakow.pl/VehiclePositions_A.pb',
        'gtfs_file' => 'VehiclePositions_A.pb',
        'ttss' => 'http://91.223.13.70/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
        'ttss_file' => 'vehicles_A.json',
    ],
    ];
    
    public static function convertTripId($tripId) {
        $tripId = explode('_', $tripId);
        if($tripId[0] != 'block') return;
        if($tripId[2] != 'trip') return;
        return 4096 * (int)$tripId[1] + (int)$tripId[3];
    }
	
    public function loadJson($file) {
        $json = json_decode(file_get_contents($file));
        foreach($json->vehicles as $vehicle) {
            if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue;
            if(!isset($vehicle->tripId) || !$vehicle->tripId) continue;
            if(!isset($vehicle->name) || !$vehicle->name) continue;
            if(!isset($vehicle->latitude) || !$vehicle->latitude) continue;
            if(!isset($vehicle->longitude) || !$vehicle->longitude) continue;
            foreach($this->specialNames as $name) {
                if(substr($vehicle->name, -strlen($name)) == $name) {
foreach($sources as $name => $source) {
    $logger = new Monolog\Logger('fetch_'.$name);
    try {
        $logger->info('Fetching '.$name.' position data from FTP...');
        $updated = ftp_fetch_if_newer($source['gtfs'], __DIR__.'/data/'.$source['gtfs_file']);
        if(!$updated) {
            $logger->info('Nothing to do, remote file not newer than local one');
                    continue;
                }
            }
            $this->jsonTrips[(int)$vehicle->tripId] = [
                'id' => $vehicle->id,
                'latitude' => (float)$vehicle->latitude / 3600000.0,
                'longitude' => (float)$vehicle->longitude / 3600000.0,
            ];
        }
        ksort($this->jsonTrips);
    }
    
    public function loadGtfs($file) {
        $data = file_get_contents($file);
        $feed = new FeedMessage();
        $feed->parse($data);
        foreach ($feed->getEntityList() as $entity) {
            $vehiclePosition = $entity->getVehicle();
            $position = $vehiclePosition->getPosition();
            $vehicle = $vehiclePosition->getVehicle();
            $trip = $vehiclePosition->getTrip();
            $tripId = $trip->getTripId();
            $this->gtfsTrips[self::convertTripId($tripId)] = [
                'id' => $entity->getId(),
                'num' => $vehicle->getLicensePlate(),
                'tripId' => $tripId,
                'latitude' => $position->getLatitude(),
                'longitude' => $position->getLongitude(),
            ];
        }
        ksort($this->gtfsTrips);
    }
        $logger->info('Fetching '.$name.' positions from TTSS...');
        fetch($source['ttss'], __DIR__.'/data/'.$source['ttss_file']);
    
    public function findOffset() {
        if(count($this->jsonTrips) == 0 || count($this->gtfsTrips) == 0) {
            return NULL;
        }
        $logger->info('Loading data...');
        $mapper = new Mapper();
        $mapper->loadTTSS(__DIR__.'/data/'.$source['ttss_file']);
        $mapper->loadGTFS(__DIR__.'/data/'.$source['gtfs_file']);
        
        $jsonTripIds = array_keys($this->jsonTrips);
        $gtfsTripIds = array_keys($this->gtfsTrips);
		
        $possibleOffsets = [];
        for($i = 0; $i < count($this->jsonTrips); $i++) {
            for($j = 0; $j < count($this->gtfsTrips); $j++) {
                $possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE;
            }
        }
        $possibleOffsets = array_keys($possibleOffsets);
		
        $bestOffset = 0;
        $maxMatched = 0;
        $options = 0;
		
        foreach($possibleOffsets as $offset) {
            $matched = 0;
			
            foreach($gtfsTripIds as $tripId) {
                $tripId += $offset;
                if(isset($this->jsonTrips[$tripId])) {
                    $matched++;
                }
            }
			
            if($matched > $maxMatched) {
                $bestOffset = $offset;
                $maxMatched = $matched;
                $options = 1;
            } elseif($matched == $maxMatched) {
                $options++;
            }
        }
		
        if($options != 1) {
            fwrite(STDERR, 'Found '.$options.' possible mappings!'."\n");
            return FALSE;
        }
        return $bestOffset;
    }
	
    public function getMapping($offset) {
        $result = [];
        foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) {
            $jsonTripId = $gtfsTripId + $offset;
            if(isset($this->jsonTrips[$jsonTripId])) {
                $data = numToTypeB($gtfsTrip['id']);
                $num = $gtfsTrip['num'];
                if(!is_array($data) || !isset($data['num'])) {
                    $data = [
                        'num' => $num,
                        'low' => 2,
                    ];
                } elseif($data['num'] != $num) {
                    // Ignore due to incorrect depot markings in the data
                    //fwrite(STDERR, 'Got '.$num.', database has '.$data['num']."\n");
                }
                $result[$this->jsonTrips[$jsonTripId]['id']] = $data;
            }
        }
        return $result;
    }
}

$mapper = new IdMapper();
$mapper->loadJson('./data/vehicles_A.json');
$mapper->loadGtfs('./data/VehiclePositions_A.pb');
        $logger->info('Finding correct offset...');
$offset = $mapper->findOffset();
if($offset) {
    echo json_encode($mapper->getMapping($offset));
            $logger->info('Got offset '.$offset.', creating mapping...');
            $mapping = $mapper->getMapping($offset);
            echo json_encode($mapping);
        }
        $logger->info('Finished');
    } catch(Throwable $e) {
        $logger->error($e->getMessage(), ['exception' => $e, 'exception_string' => (string)$e]);
    }
}

 parse.sh

File was deleted

			@@ -1,109 +1,2 @@
			<?php
			function numToType($id, $data, $defaultLow=NULL) {
			$data = explode("\n", trim($data));
			foreach($data as $line) {
			$line = explode("\t", trim($line));
			if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) {
			return [
			'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT),
			'type' => $line[3],
			'low' => isset($line[4]) ? $line[4] : $defaultLow,
			];
			}
			}
			return [];
			}
			function numToTypeT($id) {
			$data = <<<'END'
			101 107 HW E1 0
			108 113 RW E1 0
			114 126 HW E1 0
			127 127 RW E1 0
			128 130 HW E1 0
			131 132 RW E1 0
			133 133 HW E1 0
			134 134 RW E1 0
			135 136 HW E1 0
			137 139 RW E1 0
			140 147 HW E1 0
			148 150 RW E1 0
			151 152 HW E1 0
			153 153 RW E1 0
			154 154 HW E1 0
			155 155 RW E1 0
			156 158 HW E1 0
			159 159 RW E1 0
			160 174 HW E1 0
			201 245 RZ 105N 0
			246 299 HZ 105N 0
			301 312 RF GT8S 0
			313 313 RF GT8C 1
			314 322 RF GT8S 0
			323 323 RF GT8N 1
			324 324 RF GT8S 0
			325 328 RF GT8N 1
			401 440 HL EU8N 1
			451 456 HK N8C-NF 0
			457 461 HK N8S-NF 1
			462 462 HK N8C-NF 0
			601 614 RP NGT6 (1) 2
			615 626 RP NGT6 (2) 2
			627 650 RP NGT6 (3) 2
			801 824 RY NGT8 2
			899 899 RY 126N 2
			901 914 RG 2014N 2
			915 936 HG 2014N 2
			999 999 HG 405N 1
			END;
			return numToType($id, $data);
			}
			function numToTypeB($id) {
			$data = <<<'END'
			2 4 DN Solaris Urbino 18 IV Electric
			71 83 BH Solaris Urbino 18 III Hybrid
			84 96 BH Volvo 7900A Hybrid
			103 105 PA Mercedes-Benz 516
			106 112 DA Autosan M09LE
			113 121 BA Autosan M09LE
			122 128 DA Autosan M09LE
			129 139 BA Autosan M09LE
			141 146 PM MAN NL283 Lion's City
			200 200 DO Mercedes Conecto
			206 210 PO Mercedes O530 C2 Hybrid
			211 218 DO Mercedes O530
			219 243 PO Mercedes O530 C2 Hybrid
			244 269 DO Mercedes O530 C2
			270 299 BO Mercedes O530 C2
			301 338 DU Solaris Urbino 12 IV
			339 340 BU Solaris Urbino 12 IV
			341 345 DU Solaris Urbino 12 III
			400 403 BH Solaris Urbino 12,9 III Hybrid
			404 408 DH Solaris Urbino 12,9 III Hybrid
			501 510 BR Solaris Urbino 18 IV
			511 568 DR Solaris Urbino 18 IV
			569 579 BR Solaris Urbino 18 IV
			580 595 DR Solaris Urbino 18 IV
			601 601 DE Solaris Urbino 12 III Electric
			602 605 DE Solaris Urbino 8,9LE Electric
			606 606 DE Solaris Urbino 12 III Electric
			607 623 DE Solaris Urbino 12 IV Electric
			700 700 DC Mercedes Conecto G
			701 731 DC Mercedes O530G
			732 732 DC Mercedes Conecto G
			737 741 BR Solaris Urbino 18 III
			742 745 DR Solaris Urbino 18 III
			746 764 PR Solaris Urbino 18 III
			765 768 DR Solaris Urbino 18 III
			769 776 PR Solaris Urbino 18 MetroStyle
			777 777 DR Solaris Urbino 18 III
			778 797 PR Solaris Urbino 18 IV
			851 903 BU Solaris Urbino 12 III
			904 905 DU Solaris Urbino 12 III
			906 926 BU Solaris Urbino 12 III
			927 976 PU Solaris Urbino 12 III
			977 977 DU Solaris Urbino 12 III
			978 991 PU Solaris Urbino 12 IV
			992 997 BU Solaris Urbino 12 IV
			END;
			return numToType($id, $data, 2);
			}
			require_once(__DIR__.'/lib/vehicle_types.php');

			@@ -1,5 +1,6 @@
			{
			"require": {
			"google/gtfs-realtime-bindings": "^0.0.2"
			"google/gtfs-realtime-bindings": "^0.0.2",
			"monolog/monolog": "^1.24"
			}
			}

New file
			@@ -0,0 +1,86 @@
			<?php
			function ftp_fetch_if_newer($url, $file = NULL) {
			$url = parse_url($url);
			if(!isset($url['scheme']) \|\| $url['scheme'] != 'ftp') {
			throw new Exception('Only FTP URLs are supported');
			}
			if(!isset($url['host'])) {
			throw new Exception('Hostname not present in the URL');
			}
			if(!isset($url['path'])) {
			throw new Exception('Path component not present in the URL');
			}
			if(!isset($url['port'])) {
			$url['port'] = 21;
			}
			if(!isset($url['user'])) {
			$url['user'] = 'anonymous';
			}
			if(!isset($url['pass'])) {
			$url['pass'] = 'anonymous@mpk.jacekk.net';
			}
			if($file == NULL) {
			$file = basename($url['path']);
			}

			$localTime = -1;
			$localSize = -1;
			if(is_file($file)) {
			$localTime = filemtime($file);
			$localSize = filesize($file);
			}

			$ftp = ftp_connect($url['host'], $url['port'], 10);
			if($ftp === FALSE) {
			throw new Exception('FTP connection failed');
			}
			if(!ftp_login($ftp, $url['user'], $url['pass'])) {
			throw new Exception('FTP login failed');
			}
			if(!ftp_pasv($ftp, TRUE)) {
			throw new Exception('Passive FTP request failed');
			}
			$remoteSize = ftp_size($ftp, $url['path']);
			if($remoteSize < 0) {
			throw new Exception('FTP file size fetch failed');
			}
			$remoteTime = ftp_mdtm($ftp, $url['path']);
			if($remoteTime < 0) {
			throw new Exception('FTP modification time fetch failed');
			}

			$updated = FALSE;

			if($localSize != $remoteSize \|\| $localTime < $remoteTime) {
			if(file_exists($file.'.tmp')) {
			unlink($file.'.tmp');
			}
			if(ftp_get($ftp, $file.'.tmp', $url['path'], FTP_BINARY)) {
			touch($file.'.tmp', $remoteTime);
			if(!rename($file.'.tmp', $file)) {
			throw new Exception('Temporary file rename failed');
			}
			$updated = TRUE;
			}
			}

			ftp_close($ftp);

			return $updated;
			}

			function fetch($url, $file = NULL) {
			if($file == NULL) {
			$file = basename($url['url']);
			}
			$data = file_get_contents($url);
			if($data === FALSE) {
			throw new Exception('URL fetch failed');
			}
			if(file_put_contents($file.'.tmp', $data) === FALSE) {
			throw new Exception('Temporary file creation failed');
			}
			if(!rename($file.'.tmp', $file)) {
			throw new Exception('Temporary file rename failed');
			}
			}

New file
			@@ -0,0 +1,138 @@
			<?php
			require_once(__DIR__.'/../vendor/autoload.php');
			require_once(__DIR__.'/vehicle_types.php');

			use transit_realtime\FeedMessage;

			class Mapper {
			private $jsonTrips = [];
			private $gtfsTrips = [];
			private $logger = NULL;

			private $specialNames = [
			'Zjazd do zajezdni',
			'Przejazd techniczny',
			'Wyjazd na trasę',
			];

			public function __construct() {
			$this->logger = new Monolog\Logger(__CLASS__);
			}

			public static function convertTripId($tripId) {
			$tripId = explode('_', $tripId);
			if($tripId[0] != 'block') return;
			if($tripId[2] != 'trip') return;
			return 4096 * (int)$tripId[1] + (int)$tripId[3];
			}

			public function loadTTSS($file) {
			$json = json_decode(file_get_contents($file));
			foreach($json->vehicles as $vehicle) {
			if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue;
			if(!isset($vehicle->tripId) \|\| !$vehicle->tripId) continue;
			if(!isset($vehicle->name) \|\| !$vehicle->name) continue;
			if(!isset($vehicle->latitude) \|\| !$vehicle->latitude) continue;
			if(!isset($vehicle->longitude) \|\| !$vehicle->longitude) continue;
			foreach($this->specialNames as $name) {
			if(substr($vehicle->name, -strlen($name)) == $name) {
			continue;
			}
			}
			$this->jsonTrips[(int)$vehicle->tripId] = [
			'id' => $vehicle->id,
			'latitude' => (float)$vehicle->latitude / 3600000.0,
			'longitude' => (float)$vehicle->longitude / 3600000.0,
			];
			}
			ksort($this->jsonTrips);
			}

			public function loadGTFS($file) {
			$data = file_get_contents($file);
			$feed = new FeedMessage();
			$feed->parse($data);
			foreach ($feed->getEntityList() as $entity) {
			$vehiclePosition = $entity->getVehicle();
			$position = $vehiclePosition->getPosition();
			$vehicle = $vehiclePosition->getVehicle();
			$trip = $vehiclePosition->getTrip();
			$tripId = $trip->getTripId();
			$this->gtfsTrips[self::convertTripId($tripId)] = [
			'id' => $entity->getId(),
			'num' => $vehicle->getLicensePlate(),
			'tripId' => $tripId,
			'latitude' => $position->getLatitude(),
			'longitude' => $position->getLongitude(),
			];
			}
			ksort($this->gtfsTrips);
			}

			public function findOffset() {
			if(count($this->jsonTrips) == 0 \|\| count($this->gtfsTrips) == 0) {
			return NULL;
			}

			$jsonTripIds = array_keys($this->jsonTrips);
			$gtfsTripIds = array_keys($this->gtfsTrips);

			$possibleOffsets = [];
			for($i = 0; $i < count($this->jsonTrips); $i++) {
			for($j = 0; $j < count($this->gtfsTrips); $j++) {
			$possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE;
			}
			}
			$possibleOffsets = array_keys($possibleOffsets);

			$bestOffset = 0;
			$maxMatched = 0;
			$options = 0;

			foreach($possibleOffsets as $offset) {
			$matched = 0;

			foreach($gtfsTripIds as $tripId) {
			$tripId += $offset;
			if(isset($this->jsonTrips[$tripId])) {
			$matched++;
			}
			}

			if($matched > $maxMatched) {
			$bestOffset = $offset;
			$maxMatched = $matched;
			$options = 1;
			} elseif($matched == $maxMatched) {
			$options++;
			}
			}

			if($options != 1) {
			throw new Exception('Found '.$options.' possible mappings!');
			}
			return $bestOffset;
			}

			public function getMapping($offset) {
			$result = [];
			foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) {
			$jsonTripId = $gtfsTripId + $offset;
			if(isset($this->jsonTrips[$jsonTripId])) {
			$data = numToTypeB($gtfsTrip['id']);
			$num = $gtfsTrip['num'];
			if(!is_array($data) \|\| !isset($data['num'])) {
			$data = [
			'num' => $num,
			'low' => 2,
			];
			} elseif($data['num'] != $num) {
			// Ignore due to incorrect depot markings in the data
			//$this->logger->warn('Got '.$num.', database has '.$data['num']);
			}
			$result[$this->jsonTrips[$jsonTripId]['id']] = $data;
			}
			}
			return $result;
			}
			}

New file
			@@ -0,0 +1,109 @@
			<?php
			function numToType($id, $data, $defaultLow=NULL) {
			$data = explode("\n", trim($data));
			foreach($data as $line) {
			$line = explode("\t", trim($line));
			if((int)$line[0] <= (int)$id && (int)$id <= (int)$line[1]) {
			return [
			'num' => $line[2] . str_pad($id, 3, '0', STR_PAD_LEFT),
			'type' => $line[3],
			'low' => isset($line[4]) ? $line[4] : $defaultLow,
			];
			}
			}
			return [];
			}
			function numToTypeT($id) {
			$data = <<<'END'
			101 107 HW E1 0
			108 113 RW E1 0
			114 126 HW E1 0
			127 127 RW E1 0
			128 130 HW E1 0
			131 132 RW E1 0
			133 133 HW E1 0
			134 134 RW E1 0
			135 136 HW E1 0
			137 139 RW E1 0
			140 147 HW E1 0
			148 150 RW E1 0
			151 152 HW E1 0
			153 153 RW E1 0
			154 154 HW E1 0
			155 155 RW E1 0
			156 158 HW E1 0
			159 159 RW E1 0
			160 174 HW E1 0
			201 245 RZ 105N 0
			246 299 HZ 105N 0
			301 312 RF GT8S 0
			313 313 RF GT8C 1
			314 322 RF GT8S 0
			323 323 RF GT8N 1
			324 324 RF GT8S 0
			325 328 RF GT8N 1
			401 440 HL EU8N 1
			451 456 HK N8C-NF 0
			457 461 HK N8S-NF 1
			462 462 HK N8C-NF 0
			601 614 RP NGT6 (1) 2
			615 626 RP NGT6 (2) 2
			627 650 RP NGT6 (3) 2
			801 824 RY NGT8 2
			899 899 RY 126N 2
			901 914 RG 2014N 2
			915 936 HG 2014N 2
			999 999 HG 405N 1
			END;
			return numToType($id, $data);
			}
			function numToTypeB($id) {
			$data = <<<'END'
			2 4 DN Solaris Urbino 18 IV Electric
			71 83 BH Solaris Urbino 18 III Hybrid
			84 96 BH Volvo 7900A Hybrid
			103 105 PA Mercedes-Benz 516
			106 112 DA Autosan M09LE
			113 121 BA Autosan M09LE
			122 128 DA Autosan M09LE
			129 139 BA Autosan M09LE
			141 146 PM MAN NL283 Lion's City
			200 200 DO Mercedes Conecto
			206 210 PO Mercedes O530 C2 Hybrid
			211 218 DO Mercedes O530
			219 243 PO Mercedes O530 C2 Hybrid
			244 269 DO Mercedes O530 C2
			270 299 BO Mercedes O530 C2
			301 338 DU Solaris Urbino 12 IV
			339 340 BU Solaris Urbino 12 IV
			341 345 DU Solaris Urbino 12 III
			400 403 BH Solaris Urbino 12,9 III Hybrid
			404 408 DH Solaris Urbino 12,9 III Hybrid
			501 510 BR Solaris Urbino 18 IV
			511 568 DR Solaris Urbino 18 IV
			569 579 BR Solaris Urbino 18 IV
			580 595 DR Solaris Urbino 18 IV
			601 601 DE Solaris Urbino 12 III Electric
			602 605 DE Solaris Urbino 8,9LE Electric
			606 606 DE Solaris Urbino 12 III Electric
			607 623 DE Solaris Urbino 12 IV Electric
			700 700 DC Mercedes Conecto G
			701 731 DC Mercedes O530G
			732 732 DC Mercedes Conecto G
			737 741 BR Solaris Urbino 18 III
			742 745 DR Solaris Urbino 18 III
			746 764 PR Solaris Urbino 18 III
			765 768 DR Solaris Urbino 18 III
			769 776 PR Solaris Urbino 18 MetroStyle
			777 777 DR Solaris Urbino 18 III
			778 797 PR Solaris Urbino 18 IV
			851 903 BU Solaris Urbino 12 III
			904 905 DU Solaris Urbino 12 III
			906 926 BU Solaris Urbino 12 III
			927 976 PU Solaris Urbino 12 III
			977 977 DU Solaris Urbino 12 III
			978 991 PU Solaris Urbino 12 IV
			992 997 BU Solaris Urbino 12 IV
			END;
			return numToType($id, $data, 2);
			}

			@@ -1,142 +1,45 @@
			<?php
			require('vendor/autoload.php');
			require('common.php');
			require_once(__DIR__.'/lib/fetch.php');
			require_once(__DIR__.'/lib/mapper.php');

			use transit_realtime\FeedMessage;
			$logger = new Monolog\Logger('Parse changes');

			class IdMapper {
			private $jsonTrips = [];
			private $gtfsTrips = [];

			private $specialNames = [
			'Zjazd do zajezdni',
			'Przejazd techniczny',
			'Wyjazd na trasę',
			$sources = [
			'buses' => [
			'gtfs' => 'ftp://ztp.krakow.pl/VehiclePositions_A.pb',
			'gtfs_file' => 'VehiclePositions_A.pb',
			'ttss' => 'http://91.223.13.70/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
			'ttss_file' => 'vehicles_A.json',
			],
			];

			public static function convertTripId($tripId) {
			$tripId = explode('_', $tripId);
			if($tripId[0] != 'block') return;
			if($tripId[2] != 'trip') return;
			return 4096 * (int)$tripId[1] + (int)$tripId[3];
			}

			public function loadJson($file) {
			$json = json_decode(file_get_contents($file));
			foreach($json->vehicles as $vehicle) {
			if(isset($vehicle->isDeleted) && $vehicle->isDeleted) continue;
			if(!isset($vehicle->tripId) \|\| !$vehicle->tripId) continue;
			if(!isset($vehicle->name) \|\| !$vehicle->name) continue;
			if(!isset($vehicle->latitude) \|\| !$vehicle->latitude) continue;
			if(!isset($vehicle->longitude) \|\| !$vehicle->longitude) continue;
			foreach($this->specialNames as $name) {
			if(substr($vehicle->name, -strlen($name)) == $name) {
			foreach($sources as $name => $source) {
			$logger = new Monolog\Logger('fetch_'.$name);
			try {
			$logger->info('Fetching '.$name.' position data from FTP...');
			$updated = ftp_fetch_if_newer($source['gtfs'], __DIR__.'/data/'.$source['gtfs_file']);
			if(!$updated) {
			$logger->info('Nothing to do, remote file not newer than local one');
			continue;
			}
			}
			$this->jsonTrips[(int)$vehicle->tripId] = [
			'id' => $vehicle->id,
			'latitude' => (float)$vehicle->latitude / 3600000.0,
			'longitude' => (float)$vehicle->longitude / 3600000.0,
			];
			}
			ksort($this->jsonTrips);
			}

			public function loadGtfs($file) {
			$data = file_get_contents($file);
			$feed = new FeedMessage();
			$feed->parse($data);
			foreach ($feed->getEntityList() as $entity) {
			$vehiclePosition = $entity->getVehicle();
			$position = $vehiclePosition->getPosition();
			$vehicle = $vehiclePosition->getVehicle();
			$trip = $vehiclePosition->getTrip();
			$tripId = $trip->getTripId();
			$this->gtfsTrips[self::convertTripId($tripId)] = [
			'id' => $entity->getId(),
			'num' => $vehicle->getLicensePlate(),
			'tripId' => $tripId,
			'latitude' => $position->getLatitude(),
			'longitude' => $position->getLongitude(),
			];
			}
			ksort($this->gtfsTrips);
			}
			$logger->info('Fetching '.$name.' positions from TTSS...');
			fetch($source['ttss'], __DIR__.'/data/'.$source['ttss_file']);

			public function findOffset() {
			if(count($this->jsonTrips) == 0 \|\| count($this->gtfsTrips) == 0) {
			return NULL;
			}
			$logger->info('Loading data...');
			$mapper = new Mapper();
			$mapper->loadTTSS(__DIR__.'/data/'.$source['ttss_file']);
			$mapper->loadGTFS(__DIR__.'/data/'.$source['gtfs_file']);

			$jsonTripIds = array_keys($this->jsonTrips);
			$gtfsTripIds = array_keys($this->gtfsTrips);

			$possibleOffsets = [];
			for($i = 0; $i < count($this->jsonTrips); $i++) {
			for($j = 0; $j < count($this->gtfsTrips); $j++) {
			$possibleOffsets[$jsonTripIds[$i] - $gtfsTripIds[$j]] = TRUE;
			}
			}
			$possibleOffsets = array_keys($possibleOffsets);

			$bestOffset = 0;
			$maxMatched = 0;
			$options = 0;

			foreach($possibleOffsets as $offset) {
			$matched = 0;

			foreach($gtfsTripIds as $tripId) {
			$tripId += $offset;
			if(isset($this->jsonTrips[$tripId])) {
			$matched++;
			}
			}

			if($matched > $maxMatched) {
			$bestOffset = $offset;
			$maxMatched = $matched;
			$options = 1;
			} elseif($matched == $maxMatched) {
			$options++;
			}
			}

			if($options != 1) {
			fwrite(STDERR, 'Found '.$options.' possible mappings!'."\n");
			return FALSE;
			}
			return $bestOffset;
			}

			public function getMapping($offset) {
			$result = [];
			foreach($this->gtfsTrips as $gtfsTripId => $gtfsTrip) {
			$jsonTripId = $gtfsTripId + $offset;
			if(isset($this->jsonTrips[$jsonTripId])) {
			$data = numToTypeB($gtfsTrip['id']);
			$num = $gtfsTrip['num'];
			if(!is_array($data) \|\| !isset($data['num'])) {
			$data = [
			'num' => $num,
			'low' => 2,
			];
			} elseif($data['num'] != $num) {
			// Ignore due to incorrect depot markings in the data
			//fwrite(STDERR, 'Got '.$num.', database has '.$data['num']."\n");
			}
			$result[$this->jsonTrips[$jsonTripId]['id']] = $data;
			}
			}
			return $result;
			}
			}

			$mapper = new IdMapper();
			$mapper->loadJson('./data/vehicles_A.json');
			$mapper->loadGtfs('./data/VehiclePositions_A.pb');
			$logger->info('Finding correct offset...');
			$offset = $mapper->findOffset();
			if($offset) {
			echo json_encode($mapper->getMapping($offset));
			$logger->info('Got offset '.$offset.', creating mapping...');
			$mapping = $mapper->getMapping($offset);
			echo json_encode($mapping);
			}
			$logger->info('Finished');
			} catch(Throwable $e) {
			$logger->error($e->getMessage(), ['exception' => $e, 'exception_string' => (string)$e]);
			}
			}