001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.URL; 009import java.nio.charset.StandardCharsets; 010import java.util.HashSet; 011import java.util.List; 012import java.util.Map; 013import java.util.Map.Entry; 014import java.util.Optional; 015import java.util.Set; 016import java.util.concurrent.ConcurrentHashMap; 017import java.util.concurrent.ConcurrentMap; 018import java.util.concurrent.ThreadPoolExecutor; 019import java.util.concurrent.TimeUnit; 020import java.util.regex.Matcher; 021import java.util.regex.Pattern; 022 023import org.apache.commons.jcs.access.behavior.ICacheAccess; 024import org.openstreetmap.gui.jmapviewer.Tile; 025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 030import org.openstreetmap.josm.data.cache.CacheEntry; 031import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 032import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 034import org.openstreetmap.josm.data.preferences.LongProperty; 035import org.openstreetmap.josm.tools.HttpClient; 036import org.openstreetmap.josm.tools.Logging; 037 038/** 039 * Class bridging TMS requests to JCS cache requests 040 * 041 * @author Wiktor Niesiobędzki 042 * @since 8168 043 */ 044public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 045 private static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 046 private static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 047 private static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException>(.+)</ServiceException>.+"); 048 protected final Tile tile; 049 private volatile URL url; 050 051 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 052 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 053 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 054 055 /** 056 * Constructor for creating a job, to get a specific tile from cache 057 * @param listener Tile loader listener 058 * @param tile to be fetched from cache 059 * @param cache object 060 * @param connectTimeout when connecting to remote resource 061 * @param readTimeout when connecting to remote resource 062 * @param headers HTTP headers to be sent together with request 063 * @param downloadExecutor that will be executing the jobs 064 */ 065 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 066 ICacheAccess<String, BufferedImageCacheEntry> cache, 067 int connectTimeout, int readTimeout, Map<String, String> headers, 068 ThreadPoolExecutor downloadExecutor) { 069 super(cache, connectTimeout, readTimeout, headers, downloadExecutor); 070 this.tile = tile; 071 if (listener != null) { 072 String deduplicationKey = getCacheKey(); 073 synchronized (inProgress) { 074 inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener); 075 } 076 } 077 } 078 079 @Override 080 public String getCacheKey() { 081 if (tile != null) { 082 TileSource tileSource = tile.getTileSource(); 083 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':' 084 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 085 } 086 return null; 087 } 088 089 /* 090 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 091 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 092 * data from cache, that's why URL creation is postponed until it's needed 093 * 094 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 095 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 096 * 097 */ 098 @Override 099 public URL getUrl() throws IOException { 100 if (url == null) { 101 synchronized (this) { 102 if (url == null) { 103 String sUrl = tile.getUrl(); 104 if (!"".equals(sUrl)) { 105 url = new URL(sUrl); 106 } 107 } 108 } 109 } 110 return url; 111 } 112 113 @Override 114 public boolean isObjectLoadable() { 115 if (cacheData != null) { 116 byte[] content = cacheData.getContent(); 117 try { 118 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 119 } catch (IOException e) { 120 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 121 new Object[] {tile.getKey(), e.getMessage()} 122 ); 123 } 124 } 125 return false; 126 } 127 128 @Override 129 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 130 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 131 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 132 attributes.setNoTileAtZoom(true); 133 return false; // do no try to load data from no-tile at zoom, cache empty object instead 134 } 135 return super.isResponseLoadable(headers, statusCode, content); 136 } 137 138 @Override 139 protected boolean cacheAsEmpty() { 140 return isNoTileAtZoom() || super.cacheAsEmpty(); 141 } 142 143 @Override 144 public void submit(boolean force) { 145 tile.initLoading(); 146 try { 147 super.submit(this, force); 148 } catch (IOException | IllegalArgumentException e) { 149 // if we fail to submit the job, mark tile as loaded and set error message 150 Logging.log(Logging.LEVEL_WARN, e); 151 tile.finishLoading(); 152 tile.setError(e.getMessage()); 153 } 154 } 155 156 @Override 157 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 158 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 159 Set<TileLoaderListener> listeners; 160 synchronized (inProgress) { 161 listeners = inProgress.remove(getCacheKey()); 162 } 163 boolean status = result.equals(LoadResult.SUCCESS); 164 165 try { 166 tile.finishLoading(); // whatever happened set that loading has finished 167 // set tile metadata 168 if (this.attributes != null) { 169 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 170 tile.putValue(e.getKey(), e.getValue()); 171 } 172 } 173 174 switch(result) { 175 case SUCCESS: 176 handleNoTileAtZoom(); 177 if (attributes != null) { 178 int httpStatusCode = attributes.getResponseCode(); 179 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 180 if (attributes.getErrorMessage() == null) { 181 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 182 } else { 183 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 184 } 185 status = false; 186 } 187 } 188 status &= tryLoadTileImage(object); //try to keep returned image as background 189 break; 190 case FAILURE: 191 tile.setError("Problem loading tile"); 192 tryLoadTileImage(object); 193 break; 194 case CANCELED: 195 tile.loadingCanceled(); 196 // do nothing 197 } 198 199 // always check, if there is some listener interested in fact, that tile has finished loading 200 if (listeners != null) { // listeners might be null, if some other thread notified already about success 201 for (TileLoaderListener l: listeners) { 202 l.tileLoadingFinished(tile, status); 203 } 204 } 205 } catch (IOException e) { 206 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); 207 tile.setError(e); 208 tile.setLoaded(false); 209 if (listeners != null) { // listeners might be null, if some other thread notified already about success 210 for (TileLoaderListener l: listeners) { 211 l.tileLoadingFinished(tile, false); 212 } 213 } 214 } 215 } 216 217 /** 218 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 219 * 220 * @return base URL of TMS or server url as defined in super class 221 */ 222 @Override 223 protected String getServerKey() { 224 TileSource ts = tile.getSource(); 225 if (ts instanceof AbstractTMSTileSource) { 226 return ((AbstractTMSTileSource) ts).getBaseUrl(); 227 } 228 return super.getServerKey(); 229 } 230 231 @Override 232 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 233 return new BufferedImageCacheEntry(content); 234 } 235 236 @Override 237 public void submit() { 238 submit(false); 239 } 240 241 @Override 242 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 243 CacheEntryAttributes ret = super.parseHeaders(urlConn); 244 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 245 // at least for some short period of time, but not too long 246 if (ret.getExpirationTime() < now + MINIMUM_EXPIRES.get()) { 247 ret.setExpirationTime(now + MINIMUM_EXPIRES.get()); 248 } 249 if (ret.getExpirationTime() > now + MAXIMUM_EXPIRES.get()) { 250 ret.setExpirationTime(now + MAXIMUM_EXPIRES.get()); 251 } 252 return ret; 253 } 254 255 private boolean handleNoTileAtZoom() { 256 if (isNoTileAtZoom()) { 257 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 258 tile.setError("No tile at this zoom level"); 259 tile.putValue("tile-info", "no-tile"); 260 return true; 261 } 262 return false; 263 } 264 265 private boolean isNoTileAtZoom() { 266 if (attributes == null) { 267 Logging.warn("Cache attributes are null"); 268 } 269 return attributes != null && attributes.isNoTileAtZoom(); 270 } 271 272 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 273 if (object != null) { 274 byte[] content = object.getContent(); 275 if (content.length > 0) { 276 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 277 tile.loadImage(in); 278 if (tile.getImage() == null) { 279 String s = new String(content, StandardCharsets.UTF_8); 280 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 281 if (m.matches()) { 282 tile.setError(m.group(1)); 283 Logging.error(m.group(1)); 284 Logging.debug(s); 285 } else { 286 tile.setError(tr("Could not load image from tile server")); 287 } 288 return false; 289 } 290 } catch (UnsatisfiedLinkError e) { 291 throw new IOException(e); 292 } 293 } 294 } 295 return true; 296 } 297}