Jacek Kowalski
2019-06-20 978f77a82328250542c2baa73bc96a383501a269
commit | author | age
138f39 1 <?php
1a98b6 2 require_once(__DIR__.'/lib/database.php');
4673cc 3 require_once(__DIR__.'/lib/fetch.php');
JK 4 require_once(__DIR__.'/lib/mapper.php');
138f39 5
4673cc 6 $logger = new Monolog\Logger('Parse changes');
138f39 7
4673cc 8 $sources = [
0c85a7 9     'bus' => [
JK 10         'gtfsrt' => 'ftp://ztp.krakow.pl/VehiclePositions_A.pb',
11         'gtfsrt_file' => 'VehiclePositions_A.pb',
a2010c 12         'ttss' => 'http://ttss.mpk.krakow.pl/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
4673cc 13         'ttss_file' => 'vehicles_A.json',
1a98b6 14         'database' => 'mapping_A.sqlite3',
914dc8 15         'result' => 'mapping_A.json',
9970fb 16         'mapper' => 'numToTypeB',
4673cc 17     ],
f557e6 18     'tram' => [
18fdd5 19         'gtfsrt' => 'ftp://ztp.krakow.pl/VehiclePositions_T.pb',
f557e6 20         'gtfsrt_file' => 'VehiclePositions_T.pb',
JK 21         'ttss' => 'http://www.ttss.krakow.pl/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
22         'ttss_file' => 'vehicles_T.json',
23         'database' => 'mapping_T.sqlite3',
24         'result' => 'mapping_T.json',
25         'mapper' => 'numToTypeT',
26     ],
96b13b 27     'tram2' => [
JK 28         'gtfsrt' => 'ftp://ztp.krakow.pl/VehiclePositions.pb',
29         'gtfsrt_file' => 'VehiclePositions_T.pb',
30         'ttss' => 'http://www.ttss.krakow.pl/internetservice/geoserviceDispatcher/services/vehicleinfo/vehicles',
31         'ttss_file' => 'vehicles_T.json',
32         'database' => 'mapping_T.sqlite3',
33         'result' => 'mapping_T.json',
34         'mapper' => 'numToTypeT',
35     ],
4673cc 36 ];
138f39 37
4673cc 38 foreach($sources as $name => $source) {
JK 39     $logger = new Monolog\Logger('fetch_'.$name);
40     try {
0c85a7 41         foreach(['gtfsrt_file', 'ttss_file', 'database', 'result'] as $field) {
1a98b6 42             $source[$field] = __DIR__.'/data/'.$source[$field];
JK 43         }
44         $source['result_temp'] = $source['result'].'.tmp';
45         
4673cc 46         $logger->info('Fetching '.$name.' position data from FTP...');
0c85a7 47         $updated = ftp_fetch_if_newer($source['gtfsrt'], $source['gtfsrt_file']);
4673cc 48         if(!$updated) {
JK 49             $logger->info('Nothing to do, remote file not newer than local one');
50             continue;
51         }
52         
0c85a7 53         $logger->info('Fetching '.$name.' position data from TTSS...');
JK 54         fetch($source['ttss'], $source['ttss_file']);
4673cc 55         
JK 56         $logger->info('Loading data...');
57         $mapper = new Mapper();
9afb6b 58         
1a98b6 59         $mapper->loadTTSS($source['ttss_file']);
9afb6b 60         $timeDifference = time() - $mapper->getTTSSDate();
5b197e 61         if(abs($timeDifference) > 120) {
9afb6b 62             throw new Exception('TTSS timestamp difference ('.$timeDifference.'s) is too high, aborting!');
JK 63         }
64         
0c85a7 65         $mapper->loadGTFSRT($source['gtfsrt_file']);
9afb6b 66         $timeDifference = time() - $mapper->getGTFSRTDate();
5b197e 67         if(abs($timeDifference) > 120) {
9afb6b 68             throw new Exception('GTFSRT timestamp difference ('.$timeDifference.'s) is too high, aborting!');
JK 69         }
1a98b6 70         
JK 71         $db = new Database($source['database']);
4673cc 72         
JK 73         $logger->info('Finding correct offset...');
74         $offset = $mapper->findOffset();
1a98b6 75         if(!$offset) {
JK 76             throw new Exception('Offset not found');
4673cc 77         }
1a98b6 78         
JK 79         $logger->info('Got offset '.$offset.', creating mapping...');
9970fb 80         $mapping = $mapper->mapUsingOffset($offset, $source['mapper']);
1a98b6 81         
JK 82         $logger->info('Checking the data for correctness...');
83         $weight = count($mapping);
33182e 84         
JK 85         $correct = 0;
86         $incorrect = 0;
87         $old = 0;
88         $maxWeight = 0;
1a98b6 89         foreach($mapping as $id => $vehicle) {
JK 90             $dbVehicle = $db->getById($id);
91             if($dbVehicle) {
33182e 92                 $maxWeight = max($maxWeight, $dbVehicle['weight']);
JK 93                 if((int)substr($vehicle['num'], 2) == (int)$dbVehicle['num']) {
94                     $correct += 1;
95                 } else {
96                     $incorrect += 1;
1a98b6 97                 }
JK 98                 continue;
99             }
100             
101             $dbVehicle = $db->getByNum($vehicle['num']);
102             if($dbVehicle && $dbVehicle['id'] != $id) {
33182e 103                 $old += 1;
1a98b6 104             }
JK 105         }
33182e 106         $logger->info('Weight: '.$weight.', correct: '.$correct.', incorrect: '.$incorrect.', old: '.$old);
1a98b6 107         
JK 108         $previousMapping = NULL;
33182e 109         if($incorrect > $correct && $maxWeight > $weight) {
1a98b6 110             throw new Exception('Ignoring result due to better data already present');
5af1a7 111         } elseif($old > $correct) {
33182e 112             $logger->warn('Replacing DB data with the new mapping');
1a98b6 113             $db->clear();
JK 114         } else {
115             $previousMapping = @json_decode(@file_get_contents($source['result']), TRUE);
116         }
117         
118         $db->addMapping($mapping);
119         
120         if(is_array($previousMapping)) {
121             $logger->info('Merging previous data with current mapping');
423dc9 122             $mapping = $mapping + $previousMapping;
1a98b6 123             ksort($mapping);
JK 124         }
125         
126         $json = json_encode($mapping);
127         if(!file_put_contents($source['result_temp'], $json)) {
128             throw new Exception('Result save failed');
129         }
130         rename($source['result_temp'], $source['result']);
4673cc 131         $logger->info('Finished');
JK 132     } catch(Throwable $e) {
133         $logger->error($e->getMessage(), ['exception' => $e, 'exception_string' => (string)$e]);
134     }
138f39 135 }