From 9419f76eae6a64f329c7fe4df2bf9c5a2ef94f93 Mon Sep 17 00:00:00 2001
From: Jacek Kowalski <Jacek@jacekk.info>
Date: Tue, 23 Jul 2019 21:54:40 +0000
Subject: [PATCH] Allow to fetch GTFS data from sources other than FTP servers
---
lib/Fetch.php | 80 +++++++++++++++++++++++++++++++++-------
parse.php | 8 ++--
2 files changed, 70 insertions(+), 18 deletions(-)
diff --git a/lib/Fetch.php b/lib/Fetch.php
index 78e5bfe..3b8d540 100644
--- a/lib/Fetch.php
+++ b/lib/Fetch.php
@@ -20,7 +20,7 @@
if(!isset($url['pass'])) {
$url['pass'] = 'anonymous@mpk.jacekk.net';
}
- if($file == NULL) {
+ if($file === NULL) {
$file = basename($url['path']);
}
@@ -37,34 +37,86 @@
$updated = FALSE;
- if($localTime < $remoteTime || ($localTime == $remoteTime && $localSize != $remoteSize)) {
- if(file_exists($file.'.tmp')) {
- unlink($file.'.tmp');
- }
- $ftp->get($file.'.tmp', $url['path'], FTP_BINARY);
- touch($file.'.tmp', $remoteTime);
- if(!rename($file.'.tmp', $file)) {
- throw new Exception('Temporary file rename failed');
- }
- $updated = TRUE;
+ if($localTime >= $remoteTime && $localSize == $remoteSize) {
+ return FALSE;
}
- return $updated;
+ if(file_exists($file.'.tmp')) {
+ unlink($file.'.tmp');
+ }
+ $ftp->get($file.'.tmp', $url['path'], FTP_BINARY);
+ touch($file.'.tmp', $remoteTime);
+ if(!rename($file.'.tmp', $file)) {
+ throw new Exception('Temporary file rename failed');
+ }
+
+ return TRUE;
+ }
+
+ static function parse_http_headers($headers) {
+ $hasHeader = FALSE;
+ foreach($headers as $header) {
+ if(substr($header, 0, 5) === 'HTTP/') {
+ $code = substr($header, 9, 3);
+ if($code === '304') {
+ return NULL;
+ } elseif(substr($code, 0, 1) == '2') {
+ $hasHeader = TRUE;
+ }
+ } elseif($hasHeader && strtolower(substr($header, 0, 15)) === 'last-modified: ') {
+ return strptime(substr($header, 15), 'D, d M Y H:i:s T');
+ }
+ }
+ return FALSE;
}
static function generic($url, $file = NULL) {
- if($file == NULL) {
+ if($file === NULL) {
$file = basename($url['url']);
}
- $data = file_get_contents($url);
+
+ $context = [];
+ if(is_file($file)) {
+ $file_date = filemtime($file);
+ $context['http'] = [
+ 'header' => [
+ 'If-Modified-Since: '.gmdate('D, d M Y H:i:s T', $file_date),
+ ],
+ ];
+ }
+
+ $data = file_get_contents($url, FALSE, stream_context_create($context));
+ $remoteTime = FALSE;
+ if(isset($http_response_header) && is_array($http_response_header)) {
+ $remoteTime = self::parse_http_headers($http_response_header);
+ if($remoteTime === NULL) {
+ return FALSE;
+ }
+ }
+
if($data === FALSE) {
throw new Exception('URL fetch failed');
}
if(file_put_contents($file.'.tmp', $data) === FALSE) {
throw new Exception('Temporary file creation failed');
}
+ if($remoteTime !== FALSE) {
+ touch($file.'.tmp', $remoteTime);
+ }
if(!rename($file.'.tmp', $file)) {
throw new Exception('Temporary file rename failed');
}
+
+ return TRUE;
+ }
+
+ static function auto($url, $file = NULL) {
+ if($file === NULL) {
+ $file = basename($url['url']);
+ }
+ if(substr($url, 0, 4) == 'ftp:') {
+ return self::ftp($url, $file);
+ }
+ return self::generic($url, $file);
}
}
diff --git a/parse.php b/parse.php
index b26114c..3252150 100644
--- a/parse.php
+++ b/parse.php
@@ -5,15 +5,15 @@
foreach($sources as $name => $source) {
$logger = new Monolog\Logger('fetch_'.$name);
try {
- $logger->info('Fetching '.$name.' position data from FTP...');
- $updated = Fetch::ftp($source['gtfsrt'], $source['gtfsrt_file']);
+ $logger->info('Fetching '.$name.' GTFS position data ...');
+ $updated = Fetch::auto($source['gtfsrt'], $source['gtfsrt_file']);
if(!$updated) {
$logger->info('Nothing to do, remote file not newer than local one');
continue;
}
- $logger->info('Fetching '.$name.' position data from TTSS...');
- Fetch::generic($source['ttss'], $source['ttss_file']);
+ $logger->info('Fetching '.$name.' TTSS position data...');
+ Fetch::auto($source['ttss'], $source['ttss_file']);
$logger->info('Loading data...');
$mapper = new Mapper();
--
Gitblit v1.9.1