Cache the checksum of asset index files. (#335)

* Cache the checksum of asset index files.

* Resolve reviews
dev/0.11
shedaniel 2021-01-29 23:38:46 +08:00 committed by GitHub
parent 48d4243be6
commit 17fb2b4656
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 48 additions and 3 deletions

View File

@ -26,20 +26,27 @@ package net.fabricmc.loom.configuration.providers.minecraft.assets;
import java.io.File; import java.io.File;
import java.io.FileReader; import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.util.Deque; import java.util.Deque;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
import com.google.common.hash.Hashing;
import com.google.common.io.Files;
import com.google.gson.Gson; import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.gradle.api.GradleException; import org.gradle.api.GradleException;
import org.gradle.api.Project; import org.gradle.api.Project;
import net.fabricmc.loom.LoomGradleExtension; import net.fabricmc.loom.LoomGradleExtension;
import net.fabricmc.loom.LoomGradlePlugin;
import net.fabricmc.loom.configuration.providers.MinecraftProvider; import net.fabricmc.loom.configuration.providers.MinecraftProvider;
import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionInfo; import net.fabricmc.loom.configuration.providers.minecraft.MinecraftVersionInfo;
import net.fabricmc.loom.util.Checksum; import net.fabricmc.loom.util.Checksum;
@ -63,6 +70,7 @@ public class MinecraftAssetsProvider {
} }
File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.getFabricId(minecraftProvider.getMinecraftVersion()) + ".json"); File assetsInfo = new File(assets, "indexes" + File.separator + assetIndex.getFabricId(minecraftProvider.getMinecraftVersion()) + ".json");
File checksumInfo = new File(assets, "checksum" + File.separator + minecraftProvider.getMinecraftVersion() + ".json");
if (!assetsInfo.exists() || !Checksum.equals(assetsInfo, assetIndex.sha1)) { if (!assetsInfo.exists() || !Checksum.equals(assetsInfo, assetIndex.sha1)) {
project.getLogger().lifecycle(":downloading asset index"); project.getLogger().lifecycle(":downloading asset index");
@ -80,17 +88,28 @@ public class MinecraftAssetsProvider {
} }
} }
project.getLogger().lifecycle(":downloading assets..."); Gson gson = new Gson();
Map<String, String> checksumInfos = new HashMap<>();
if (checksumInfo.exists()) {
try (FileReader reader = new FileReader(checksumInfo)) {
checksumInfos.putAll(gson.fromJson(reader, new TypeToken<Map<String, String>>() {
}.getType()));
}
}
Deque<ProgressLogger> loggers = new ConcurrentLinkedDeque<>(); Deque<ProgressLogger> loggers = new ConcurrentLinkedDeque<>();
ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1))); ExecutorService executor = Executors.newFixedThreadPool(Math.min(10, Math.max(Runtime.getRuntime().availableProcessors() / 2, 1)));
int toDownload = 0;
AssetIndex index; AssetIndex index;
try (FileReader fileReader = new FileReader(assetsInfo)) { try (FileReader fileReader = new FileReader(assetsInfo)) {
index = new Gson().fromJson(fileReader, AssetIndex.class); index = gson.fromJson(fileReader, AssetIndex.class);
} }
Stopwatch stopwatch = Stopwatch.createStarted();
Map<String, AssetObject> parent = index.getFileMap(); Map<String, AssetObject> parent = index.getFileMap();
for (Map.Entry<String, AssetObject> entry : parent.entrySet()) { for (Map.Entry<String, AssetObject> entry : parent.entrySet()) {
@ -99,7 +118,16 @@ public class MinecraftAssetsProvider {
String filename = "objects" + File.separator + sha1.substring(0, 2) + File.separator + sha1; String filename = "objects" + File.separator + sha1.substring(0, 2) + File.separator + sha1;
File file = new File(assets, filename); File file = new File(assets, filename);
if (!file.exists() || !Checksum.equals(file, sha1)) { String localFileChecksum = !file.exists() ? null : checksumInfos.computeIfAbsent(entry.getKey(), path -> {
try {
return Files.asByteSource(file).hash(Hashing.sha1()).toString();
} catch (IOException e) {
e.printStackTrace();
return null;
}
});
if (LoomGradlePlugin.refreshDeps || localFileChecksum == null || !localFileChecksum.equals(sha1)) {
if (offline) { if (offline) {
if (file.exists()) { if (file.exists()) {
project.getLogger().warn("Outdated asset " + entry.getKey()); project.getLogger().warn("Outdated asset " + entry.getKey());
@ -107,6 +135,7 @@ public class MinecraftAssetsProvider {
throw new GradleException("Asset " + entry.getKey() + " not found at " + file.getAbsolutePath()); throw new GradleException("Asset " + entry.getKey() + " not found at " + file.getAbsolutePath());
} }
} else { } else {
toDownload++;
executor.execute(() -> { executor.execute(() -> {
ProgressLogger progressLogger; ProgressLogger progressLogger;
@ -135,6 +164,10 @@ public class MinecraftAssetsProvider {
throw new RuntimeException("Failed to download: " + assetName, e); throw new RuntimeException("Failed to download: " + assetName, e);
} }
if (localFileChecksum == null) {
checksumInfos.put(entry.getKey(), sha1);
}
//Give this logger back //Give this logger back
loggers.add(progressLogger); loggers.add(progressLogger);
}); });
@ -142,6 +175,18 @@ public class MinecraftAssetsProvider {
} }
} }
project.getLogger().info("Took " + stopwatch.stop() + " to iterate " + parent.size() + " asset index.");
if (toDownload > 0) {
project.getLogger().lifecycle(":downloading " + toDownload + " asset" + (toDownload == 1 ? "" : "s") + "...");
}
checksumInfo.getParentFile().mkdirs();
try (FileWriter writer = new FileWriter(checksumInfo)) {
gson.toJson(checksumInfos, writer);
}
//Wait for the assets to all download //Wait for the assets to all download
executor.shutdown(); executor.shutdown();