hibiscus-client/fabric-loom
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Multithreaded asset downloading, should be a lot quicker (#175)

Browse source 
* Multithreaded asset downloading, should be a lot quicker

* Force the string width

* Limit threads to max 10 and min 1

* Use ConcurrentLinkedDeque
This commit is contained in:
modmuss50 2020-01-29 15:23:13 +00:00 committed by GitHub
parent daf922d928
commit 46b616af0f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
src/main/java/net/fabricmc/loom/providers/MinecraftAssetsProvider.java
View file

@ -28,7 +28,12 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
import java.util.Deque;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.google.gson.Gson;
import org.gradle.api.GradleException;
@ -76,8 +81,11 @@ public class MinecraftAssetsProvider {
}
}
ProgressLogger progressLogger = ProgressLogger.getProgressFactory(project, MinecraftAssetsProvider.class.getName());
progressLogger.start("Downloading assets...", "assets");
project.getLogger().lifecycle(":downloading assets...");
Deque<ProgressLogger> loggers = new ConcurrentLinkedDeque<>();
ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)));
AssetIndex index;
try (FileReader fileReader = new FileReader(assetsInfo)) {
@ -85,9 +93,6 @@ public class MinecraftAssetsProvider {
}
Map<String, AssetObject> parent = index.getFileMap();
final int totalSize = parent.size();
int position = 0;
project.getLogger().lifecycle(":downloading assets...");
for (Map.Entry<String, AssetObject> entry : parent.entrySet()) {
AssetObject object = entry.getValue();
@ -103,22 +108,52 @@ public class MinecraftAssetsProvider {
throw new GradleException("Asset " + entry.getKey() + " not found at " + file.getAbsolutePath());
}
} else {
project.getLogger().debug(":downloading asset " + entry.getKey());
DownloadUtil.downloadIfChanged(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file, project.getLogger(), true);
executor.execute(() -> {
ProgressLogger progressLogger;
if (loggers.isEmpty()) {
//Create a new logger if we need one
progressLogger = ProgressLogger.getProgressFactory(project, MinecraftAssetsProvider.class.getName());
progressLogger.start("Downloading assets...", "assets");
} else {
// use a free logger if we can
progressLogger = loggers.pop();
}
String assetName = entry.getKey();
int end = assetName.lastIndexOf("/") + 1;
if (end > 0) {
assetName = assetName.substring(end);
}
project.getLogger().debug(":downloading asset " + assetName);
progressLogger.progress(String.format("%-30.30s", assetName) + " - " + sha1);
try {
DownloadUtil.downloadIfChanged(new URL(Constants.RESOURCES_BASE + sha1.substring(0, 2) + "/" + sha1), file, project.getLogger(), true);
} catch (IOException e) {
throw new RuntimeException("Failed to download: " + assetName, e);
}
//Give this logger back
loggers.add(progressLogger);
});
}
}
String assetName = entry.getKey();
int end = assetName.lastIndexOf("/") + 1;
if (end > 0) {
assetName = assetName.substring(end);
}
progressLogger.progress(assetName + " - " + position + "/" + totalSize + " (" + (int) ((position / (double) totalSize) * 100) + "%) assets downloaded");
position++;
}
progressLogger.completed();
//Wait for the assets to all download
executor.shutdown();
try {
if (executor.awaitTermination(2, TimeUnit.HOURS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
loggers.forEach(ProgressLogger::completed);
}
}