// ClassicalSharp copyright 2014-2016 UnknownShadow200 | Licensed under MIT using System; using System.Collections.Generic; using System.Drawing; using System.IO; using System.Net; using System.Text; using System.Threading; #if ANDROID using Android.Graphics; #endif namespace ClassicalSharp.Network { /// Specialised producer and consumer queue for downloading data asynchronously. public class AsyncDownloader : IGameComponent { EventWaitHandle handle = new EventWaitHandle(false, EventResetMode.AutoReset); Thread worker; readonly object requestLocker = new object(); List requests = new List(); readonly object downloadedLocker = new object(); Dictionary downloaded = new Dictionary(); string skinServer = null; public Request CurrentItem; public int CurrentItemProgress = -3; public AsyncDownloader() { } public AsyncDownloader(string skinServer) { Init(skinServer); } public void Init(Game game) { Init(game.skinServer); } void Init(string skinServer) { this.skinServer = skinServer; WebRequest.DefaultWebProxy = null; worker = new Thread(DownloadThreadWorker, 256 * 1024); worker.Name = "ClassicalSharp.AsyncDownloader"; worker.IsBackground = true; worker.Start(); } public void Ready(Game game) { } public void Reset(Game game) { lock(requestLocker) requests.Clear(); handle.Set(); } public void OnNewMap(Game game) { } public void OnNewMapLoaded(Game game) { } /// Asynchronously downloads a skin. If 'skinName' points to the url then the skin is /// downloaded from that url, otherwise it is downloaded from the url 'defaultSkinServer'/'skinName'.png /// Identifier is skin_'skinName'. public void DownloadSkin(string identifier, string skinName) { string strippedSkinName = Utils.StripColours(skinName); string url = Utils.IsUrlPrefix(skinName, 0) ? skinName : skinServer + strippedSkinName + ".png"; AddRequest(url, true, identifier, RequestType.Bitmap, DateTime.MinValue , null); } /// Asynchronously downloads a bitmap image from the specified url. public void DownloadImage(string url, bool priority, string identifier) { AddRequest(url, priority, identifier, RequestType.Bitmap, DateTime.MinValue, null); } /// Asynchronously downloads a string from the specified url. public void DownloadPage(string url, bool priority, string identifier) { AddRequest(url, priority, identifier, RequestType.String, DateTime.MinValue, null); } /// Asynchronously downloads a byte array. public void DownloadData(string url, bool priority, string identifier) { AddRequest(url, priority, identifier, RequestType.ByteArray, DateTime.MinValue, null); } /// Asynchronously downloads a bitmap image. public void DownloadImage(string url, bool priority, string identifier, DateTime lastModified, string etag) { AddRequest(url, priority, identifier, RequestType.Bitmap, lastModified, etag); } /// Asynchronously downloads a byte array. public void DownloadData(string url, bool priority, string identifier, DateTime lastModified, string etag) { AddRequest(url, priority, identifier, RequestType.ByteArray, lastModified, etag); } /// Asynchronously retrieves the content length of the body response. public void RetrieveContentLength(string url, bool priority, string identifier) { AddRequest(url, priority, identifier, RequestType.ContentLength, DateTime.MinValue, null); } void AddRequest(string url, bool priority, string identifier, RequestType type, DateTime lastModified, string etag) { lock(requestLocker) { Request request = new Request(url, identifier, type, lastModified, etag); if (priority) { requests.Insert(0, request); } else { requests.Add(request); } } handle.Set(); } /// Informs the asynchronous thread that it should stop processing further requests /// and can consequentially exit the for loop.
/// Note that this will *block** the calling thread as the method waits until the asynchronous /// thread has exited the for loop.
public void Dispose() { lock(requestLocker) { requests.Insert(0, null); } handle.Set(); worker.Join(); ((IDisposable)handle).Dispose(); } /// Removes older entries that were downloaded a certain time ago /// but were never removed from the downloaded queue. public void PurgeOldEntriesTask(ScheduledTask task) { const int seconds = 10; lock(downloadedLocker) { DateTime now = DateTime.UtcNow; List itemsToRemove = new List(downloaded.Count); foreach (var item in downloaded) { DateTime timestamp = item.Value.TimeDownloaded; if ((now - timestamp).TotalSeconds > seconds) { itemsToRemove.Add(item.Key); } } for (int i = 0; i < itemsToRemove.Count; i++) { string key = itemsToRemove[i]; DownloadedItem item; downloaded.TryGetValue(key, out item); downloaded.Remove(key); Bitmap bmp = item.Data as Bitmap; if (bmp != null) bmp.Dispose(); } } } /// Returns whether the requested item exists in the downloaded queue. /// If it does, it removes the item from the queue and outputs it. /// If the asynchronous thread failed to download the item, this method /// will return 'true' and 'item' will be set. However, the contents of the 'item' object will be null. public bool TryGetItem(string identifier, out DownloadedItem item) { bool success = false; lock(downloadedLocker) { success = downloaded.TryGetValue(identifier, out item); if (success) { downloaded.Remove(identifier); } } return success; } void DownloadThreadWorker() { while (true) { Request request = null; lock(requestLocker) { if (requests.Count > 0) { request = requests[0]; requests.RemoveAt(0); if (request == null) return; } } if (request != null) { CurrentItem = request; CurrentItemProgress = -2; ProcessRequest(request); CurrentItem = null; CurrentItemProgress = -3; } else { handle.WaitOne(); } } } void ProcessRequest(Request request) { string url = request.Url; Utils.LogDebug("Downloading {0} from: {1}", request.Type, url); object value = null; HttpStatusCode status = HttpStatusCode.OK; string etag = null; DateTime lastModified = DateTime.MinValue; try { HttpWebRequest req = MakeRequest(request); using (HttpWebResponse response = (HttpWebResponse)req.GetResponse()) { etag = response.Headers[HttpResponseHeader.ETag]; if (response.Headers[HttpResponseHeader.LastModified] != null) lastModified = response.LastModified; value = DownloadContent(request, response); } } catch (Exception ex) { if (!(ex is WebException || ex is ArgumentException || ex is UriFormatException || ex is IOException)) throw; if (ex is WebException) { WebException webEx = (WebException)ex; if (webEx.Response != null) { status = ((HttpWebResponse)webEx.Response).StatusCode; webEx.Response.Close(); } } if (status != HttpStatusCode.OK) { Utils.LogDebug("Failed to download (" + (int)status + ") from: " + url); } else { Utils.LogDebug("Failed to download from: " + url); } } value = CheckIsValidImage(value, url); lock(downloadedLocker) { DownloadedItem oldItem; DownloadedItem newItem = new DownloadedItem(value, request.TimeAdded, url, status, etag, lastModified); if (downloaded.TryGetValue(request.Identifier, out oldItem)) { if (oldItem.TimeAdded > newItem.TimeAdded) { DownloadedItem old = oldItem; oldItem = newItem; newItem = old; } Bitmap oldBmp = oldItem.Data as Bitmap; if (oldBmp != null) oldBmp.Dispose(); } downloaded[request.Identifier] = newItem; } } object DownloadContent(Request request, HttpWebResponse response) { if (request.Type == RequestType.Bitmap) { MemoryStream data = DownloadBytes(response); return Platform.ReadBmp(data); } else if (request.Type == RequestType.String) { MemoryStream data = DownloadBytes(response); byte[] rawBuffer = data.GetBuffer(); return Encoding.UTF8.GetString(rawBuffer, 0, (int)data.Length); } else if (request.Type == RequestType.ByteArray) { MemoryStream data = DownloadBytes(response); return data.ToArray(); } else if (request.Type == RequestType.ContentLength) { return response.ContentLength; } return null; } object CheckIsValidImage(object value, string url) { // Mono seems to be returning a bitmap with a native pointer of zero in some weird cases. // We can detect this as every single property access raises an ArgumentException. try { Bitmap bmp = value as Bitmap; if (bmp != null) { int height = bmp.Height; } return value; } catch (ArgumentException) { Utils.LogDebug("Failed to download from: " + url); return null; } } HttpWebRequest MakeRequest(Request request) { HttpWebRequest req = (HttpWebRequest)WebRequest.Create(request.Url); req.AutomaticDecompression = DecompressionMethods.GZip; req.ReadWriteTimeout = 90 * 1000; req.Timeout = 90 * 1000; req.Proxy = null; req.UserAgent = Program.AppName; if (request.LastModified != DateTime.MinValue) req.IfModifiedSince = request.LastModified; if (request.ETag != null) req.Headers["If-None-Match"] = request.ETag; return req; } static byte[] buffer = new byte[4096 * 8]; MemoryStream DownloadBytes(HttpWebResponse response) { int length = (int)response.ContentLength; MemoryStream dst = length > 0 ? new MemoryStream(length) : new MemoryStream(); CurrentItemProgress = length > 0 ? 0 : -1; using (Stream src = response.GetResponseStream()) { int read = 0; while ((read = src.Read(buffer, 0, buffer.Length)) > 0) { dst.Write(buffer, 0, read); if (length <= 0) continue; CurrentItemProgress = (int)(100 * (float)dst.Length / length); } } return dst; } } public enum RequestType { Bitmap, String, ByteArray, ContentLength } public sealed class Request { /// Full url to GET from. public string Url; /// Unique identifier for this request. public string Identifier; /// Type of data to return for this request. public RequestType Type; /// Point in time this request was added to the fetch queue. public DateTime TimeAdded; /// Point in time the item most recently cached. (if at all) public DateTime LastModified; /// ETag of the item most recently cached. (if any) public string ETag; public Request(string url, string identifier, RequestType type, DateTime lastModified, string etag) { Url = url; Identifier = identifier; Type = type; TimeAdded = DateTime.UtcNow; LastModified = lastModified; ETag = etag; } } /// Represents an item that was asynchronously downloaded. public class DownloadedItem { /// Contents that were downloaded. public object Data; /// Point in time the item was originally added to the download queue. public DateTime TimeAdded; /// Point in time the item was fully downloaded. public DateTime TimeDownloaded; /// Full URL this item was downloaded from. public string Url; /// HTTP status code returned by the server for this request. public HttpStatusCode ResponseCode; /// Unique identifier assigned by the server to this item. public string ETag; /// Time the server indicates this item was last modified. public DateTime LastModified; public DownloadedItem(object data, DateTime timeAdded, string url, HttpStatusCode code, string etag, DateTime lastModified) { Data = data; TimeAdded = timeAdded; TimeDownloaded = DateTime.UtcNow; Url = url; ResponseCode = code; ETag = etag; LastModified = lastModified; } } }