using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using UnityEngine; using UnityEngine.Networking; using UnityEngine.SceneManagement; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace QuestAppLauncher { /// /// Downloads assets (app icons packs and names files) from configured repos. /// public class AssetsDownloader { // Download cache folder that contains downloaded files const string DownloadCacheFolder = "download_cache"; // Manifest file to track what we've downloaded const string DownloadManifestFile = "download_manifest.json"; // Temporary filename for download const string TempDownloadFileExtention = ".tmp_download"; // GitHub API url const string GithubApiUrl = @"https://api.github.com/repos/"; // Rate limit in minutes const int RateLimitInMins = 5; // Used for mutual exclusion when loading assets static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1); // Cached persistent data path — must be set from the main thread before any background use private static string _persistentDataPath; /// /// Call this from the main thread (e.g. Start/Awake) before any background Task.Run. /// Unity 2023+ throws if Application.persistentDataPath is read off the main thread. /// public static void CachePersistentDataPath(string path) => _persistentDataPath = path; /// /// Class that tracks each file downloaded. /// Used to determine whether an asset has been updated since last download. /// [Serializable] public class AssetInfo { // Repo uri public string RepoUri; // Asset's url public string Url; // Asset's updated at timestamp public string UpdatedAt; // Asset's tag name public string TagName; } /// /// Mapping of asset name to asset info above. This is persisted in the download manifest. /// [Serializable] public class AssetsManifest { // File name -> release metadata public Dictionary Metadata = new Dictionary(); // Last updated timestamp public DateTime LastUpdated; } /// /// Helper method to download assets asynchronously. After completion, scene is automatically reloaded /// if new assets have been downloaded. /// /// Current config /// Download progress interface - used to indicate download progress /// public static void DownloadAssetsAsync(Config config, IDownloadProgress downloadProgress = null, bool forceCheck = false) { _ = DownloadAssetsInternalAsync(config, downloadProgress, forceCheck); } private static async Task DownloadAssetsInternalAsync(Config config, IDownloadProgress downloadProgress, bool forceCheck) { // Mutual exclusion while loading assets await AssetsDownloader.semaphoreSlim.WaitAsync(); try { // UnityWebRequest must be created and used on the main thread. // Running as a plain async method (no Task.Run) keeps us on the Unity main-thread // synchronization context where web requests are allowed. AssetsDownloader assetsDownloader = new AssetsDownloader(); bool downloaded = await assetsDownloader.DownloadFromReposAsync(config, downloadProgress, forceCheck); if (downloaded) { // We downloaded new assets, so re-load the scene Debug.Log("Downloaded new assets. Re-populating panel"); _ = SceneManager.LoadSceneAsync(SceneManager.GetActiveScene().name); } } finally { AssetsDownloader.semaphoreSlim.Release(); } } /// /// Asynchronously download assets from configured repos. /// /// Current config /// Download progress interface /// private async Task DownloadFromReposAsync(Config config, IDownloadProgress downloadProgress = null, bool forceCheck = false) { if (null == config.downloadRepos) { // No repos configured, so return return false; } // Load the download manifest. This is used to compare if we're up-to-date or not. AssetsManifest manifest = LoadManifest(); if (null == manifest) { manifest = new AssetsManifest(); } // Rate limit update checks to one per couple of minutes, to avoid GitHub's rate limit if (!forceCheck && DateTime.Now.Subtract(manifest.LastUpdated).TotalMinutes < RateLimitInMins) { Debug.LogFormat("Exceeded rate limit of {0} mins - last checked for update on {1}", RateLimitInMins, manifest.LastUpdated); return false; } // Mark that we've just checked for updates & update manifest manifest.LastUpdated = DateTime.Now; SaveManifest(manifest); if (null != downloadProgress) { downloadProgress.OnCheckingForUpdates(); } // Download the assets metadata - used to determine whether we are up-to-date or not var assetsInfo = await DownloadAssetsMetadata(config, manifest, downloadProgress, forceCheck); if (assetsInfo.Count == 0) { // No updates have been found, so return Debug.Log("No updates found"); if (null != downloadProgress) { downloadProgress.OnNoUpdatesAvailable(); } return false; } // Download the assets var downloadedAssets = await DownloadFromReposInternalAsync(manifest, assetsInfo, downloadProgress); if (null != downloadProgress) { downloadProgress.OnUpdateFinish(); } return downloadedAssets; } /// /// Downloads assets metadata. This is used to determine whether our assets are up-to-date. /// /// Current config /// Download progress interface /// private async Task> DownloadAssetsMetadata( Config config, AssetsManifest manifest, IDownloadProgress downloadProgress = null, bool forceCheck = false) { // Gather asset metadata from every configured repo. // Duplicate URIs are skipped via the seenUris set. var assetsInfo = new Dictionary(StringComparer.OrdinalIgnoreCase); var seenUris = new HashSet(StringComparer.OrdinalIgnoreCase); var reposLoaded = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var item in config.downloadRepos) { if (null == item.type || null == item.repoUri || !seenUris.Add(item.repoUri)) { continue; } bool repoLoaded = false; if (string.Equals(item.type, Config.DownloadRepo_Type_GitHub, StringComparison.OrdinalIgnoreCase)) { repoLoaded = await GetAssetsInfoFromGithubRepoAsync(item.repoUri, assetsInfo, downloadProgress); } else if (string.Equals(item.type, Config.DownloadRepo_Type_Http, StringComparison.OrdinalIgnoreCase)) { repoLoaded = await GetAssetsInfoFromHttpEndpointAsync(item.repoUri, assetsInfo, downloadProgress); } if (repoLoaded) { reposLoaded.Add(item.repoUri); } } Debug.LogFormat("Assets info contains {0} entries", assetsInfo.Count); // Enumerate asset metadata in our download manifest var deletedAssets = new HashSet(); foreach (var entry in manifest.Metadata) { if (assetsInfo.ContainsKey(entry.Key)) { if (string.Equals(entry.Value.UpdatedAt, assetsInfo[entry.Key].UpdatedAt, StringComparison.OrdinalIgnoreCase) && File.Exists(Path.Combine(GetOrCreateDownloadPath(), entry.Key))) { // Matched on file name and updated-at timestamp, so we're up to date. Skip this one. Debug.LogFormat("Asset {0} is already up to date ({1})", entry.Key, entry.Value.UpdatedAt); assetsInfo.Remove(entry.Key); } } else if (reposLoaded.Contains(entry.Value.RepoUri)) { // The repo no longer has the cached file, so nuke it to keep things in sync var filePath = Path.Combine(GetOrCreateDownloadPath(), entry.Key); if (File.Exists(filePath)) { Debug.LogFormat("Deleting cached file that no longer exists on server: {0} @ {1}", filePath, entry.Value.RepoUri); File.Delete(filePath); } deletedAssets.Add(entry.Key); } } // Remove old cached files foreach (var entry in deletedAssets) { manifest.Metadata.Remove(entry); } if (deletedAssets.Count > 0) { // Persist manifest SaveManifest(manifest); } return assetsInfo; } /// /// Download assets from repos. The download manifest is also updated. /// /// Download manifest /// Assets to download /// Download progress interface /// private async Task DownloadFromReposInternalAsync( AssetsManifest manifest, Dictionary assetsInfo, IDownloadProgress downloadProgress = null) { var downloadedAsset = false; foreach (var entry in assetsInfo) { // Download asset var success = await DownloadAssetFromGitHubRepoAsync(entry.Key, entry.Value, downloadProgress); if (success) { // Update manifest manifest.Metadata[entry.Key] = entry.Value; downloadedAsset = true; } } if (downloadedAsset) { // Persist manifest SaveManifest(manifest); } return downloadedAsset; } /// /// Download asset metadata from given uri. The assets info parameter is populate with the metadata. /// /// The repo uri to download /// Assets info mapping that is populate by this function /// Download progress interface /// private async Task GetAssetsInfoFromGithubRepoAsync(string repoUri, Dictionary assetsInfo, IDownloadProgress downloadProgress = null) { var requestUrl = GithubApiUrl + repoUri; Debug.LogFormat("Reading assets from {0}", requestUrl); try { // Request asset url using (var req = new UnityWebRequest(requestUrl)) { req.downloadHandler = new DownloadHandlerBuffer(); await req.SendWebRequest(); if (req.result != UnityWebRequest.Result.Success) { // Error reading asset metadata, so return. Debug.LogFormat("Error reading asset info: {0}", req.error); var responseHeaders = req.GetResponseHeaders(); if (null != responseHeaders && responseHeaders.ContainsKey("X-RateLimit-Remaining") && responseHeaders["X-RateLimit-Remaining"] == "0") { // Github request limit reached. Display a friendly error message. Debug.LogFormat("Request limit reached"); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error updating: Request Limit Reached - try again later. ({1})", req.error, requestUrl)); } } else { if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error updating: {0} ({1})", req.error, requestUrl)); } } return false; } // Parse the returned asset metadata var jObject = JObject.Parse(req.downloadHandler.text); var tagName = jObject["tag_name"].Value(); foreach (var property in jObject["assets"]) { var updatedAt = property["updated_at"].Value(); var url = property["url"].Value(); var name = property["name"].Value(); if ((name.StartsWith("iconpack") && name.EndsWith(".zip")) || (name.StartsWith("appnames") && name.EndsWith(".json"))) { assetsInfo[name] = new AssetInfo { RepoUri = repoUri, Url = url, UpdatedAt = updatedAt, TagName = tagName }; } } } } catch (Exception e) { Debug.LogFormat("Exception reading asset info: {0} ", e.Message); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error updating: {0} ({1})", e.Message, requestUrl)); } return false; } return true; } /// /// Fetches the asset manifest from a plain HTTP endpoint and populates /// with the downloadable files. /// /// Expected manifest JSON: /// /// { /// "files": [ /// { "name": "iconpack_v1.zip", "url": "https://…/iconpack_v1.zip", "updatedAt": "2024-01-01T00:00:00Z" }, /// { "name": "appnames.json", "url": "https://…/appnames.json", "updatedAt": "2024-01-01T00:00:00Z" } /// ] /// } /// /// Only files whose names match the icon-pack or app-names naming conventions are accepted. /// private async Task GetAssetsInfoFromHttpEndpointAsync(string manifestUrl, Dictionary assetsInfo, IDownloadProgress downloadProgress = null) { Debug.LogFormat("Reading HTTP asset manifest from {0}", manifestUrl); try { using (var req = new UnityWebRequest(manifestUrl)) { req.downloadHandler = new DownloadHandlerBuffer(); await req.SendWebRequest(); if (req.result != UnityWebRequest.Result.Success) { Debug.LogFormat("Error reading HTTP manifest: {0}", req.error); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error reading manifest: {0} ({1})", req.error, manifestUrl)); } return false; } var jObject = JObject.Parse(req.downloadHandler.text); var files = jObject["files"]; if (null == files) { Debug.LogFormat("HTTP manifest at {0} has no 'files' array", manifestUrl); return true; } foreach (var file in files) { var name = file["name"]?.Value(); var url = file["url"]?.Value(); var updatedAt = file["updatedAt"]?.Value() ?? ""; if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(url)) { continue; } // Accept the same file naming conventions as the GitHub path if ((name.StartsWith("iconpack") && name.EndsWith(".zip")) || (name.StartsWith("appnames") && name.EndsWith(".json"))) { assetsInfo[name] = new AssetInfo { RepoUri = manifestUrl, Url = url, UpdatedAt = updatedAt }; } } } } catch (Exception e) { Debug.LogFormat("Exception reading HTTP manifest: {0}", e.Message); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error reading manifest: {0} ({1})", e.Message, manifestUrl)); } return false; } return true; } /// /// Download a single asset (file) from repo /// /// Name of the asset (file) to download /// Asset info (metadata) /// Download progress interface /// private async Task DownloadAssetFromGitHubRepoAsync(string name, AssetInfo assetInfo, IDownloadProgress downloadProgress) { var filePath = Path.Combine(GetOrCreateDownloadPath(), name); var tempFilePath = filePath + TempDownloadFileExtention; Debug.LogFormat("Downloading asset {0} from {1}", filePath, assetInfo.Url); try { // Request asset url using (var req = new UnityWebRequest(assetInfo.Url)) { // GitHub API requires Accept: application/octet-stream to return the raw // binary asset rather than a JSON metadata wrapper. // Plain HTTP endpoints don't need this header and some servers may reject it. if (!string.IsNullOrEmpty(assetInfo.TagName)) { req.SetRequestHeader("Accept", "application/octet-stream"); } if (null != downloadProgress) { downloadProgress.OnDownloadStart(name); } var downloadHandler = new DownloadHandlerFileWithProgress(tempFilePath, downloadProgress.OnDownloadProgress); downloadHandler.removeFileOnAbort = true; req.downloadHandler = downloadHandler; await req.SendWebRequest(); if (req.result != UnityWebRequest.Result.Success) { // Error reading asset metadata, so return. Debug.LogFormat("Error downloading asset: {0}", req.error); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error updating: {0} ({1})", req.error, assetInfo.Url)); } if (File.Exists(tempFilePath)) { File.Delete(tempFilePath); } return false; } // Rename temp file to desination file to ensure that we are not downloading // an error body message into the destination file. if (File.Exists(filePath)) { File.Delete(filePath); } File.Move(tempFilePath, filePath); if (null != downloadProgress) { downloadProgress.OnDownloadFinish(); } return true; } } catch (Exception e) { Debug.LogFormat("Exception downloading asset: {0} ", e.Message); if (null != downloadProgress) { downloadProgress.OnError(string.Format("Error updating: {0} ({1})", e.Message, assetInfo.Url)); } } return false; } /// /// Load download manifest /// /// Download manifest static private AssetsManifest LoadManifest() { var manifestPath = Path.Combine(GetOrCreateDownloadPath(), DownloadManifestFile); if (!File.Exists(manifestPath)) { return null; } Debug.Log("Found manifest: " + manifestPath); var jsonManifest = File.ReadAllText(manifestPath); try { AssetsManifest manifest = new AssetsManifest(); return JsonConvert.DeserializeObject(jsonManifest); } catch (Exception e) { Debug.Log(string.Format("Failed to read manifest: {0}", e.Message)); } return null; } /// /// Persist download manifest /// /// Download manifest to persist static private void SaveManifest(AssetsManifest manifest) { var manifestPath = Path.Combine(GetOrCreateDownloadPath(), DownloadManifestFile); Debug.Log("Saving manifest: " + manifestPath); try { File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifest, Formatting.Indented)); } catch (Exception e) { Debug.Log(string.Format("Failed to write manifest: {0}", e.Message)); } } static public string GetOrCreateDownloadPath() { string path = Path.Combine(_persistentDataPath, DownloadCacheFolder); Directory.CreateDirectory(path); return path; } } }