diff options
Diffstat (limited to 'patches/server/1030-Fix-and-optimise-world-force-upgrading.patch')
| -rw-r--r-- | patches/server/1030-Fix-and-optimise-world-force-upgrading.patch | 395 |
1 files changed, 395 insertions, 0 deletions
diff --git a/patches/server/1030-Fix-and-optimise-world-force-upgrading.patch b/patches/server/1030-Fix-and-optimise-world-force-upgrading.patch new file mode 100644 index 0000000000..3e94098edd --- /dev/null +++ b/patches/server/1030-Fix-and-optimise-world-force-upgrading.patch @@ -0,0 +1,395 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Spottedleaf <[email protected]> +Date: Thu, 20 May 2021 07:02:22 -0700 +Subject: [PATCH] Fix and optimise world force upgrading + +The WorldUpgrader class was incorrectly modified by +CB. It will store an IChunkLoader instance for all +dimension types in the world, but obviously with how +CB shifts around worlds only one dimension type exists +per world. But this would be OK if CB did this +change correctly. All IChunkLoader instances +will point to the same regionfiles. And all +IChunkLoader instances are going to be read from. + +This problem hasn't really been reported because +it relies on the persistent legacy data to be converted +as well to cause corruption. Why? Because the legacy +data is also shared, it will result in different +outputs from conversion (as once conversion for legacy +persistent data takes place, it is REMOVED - so the next +convert will _not_ have the data). Which means different +sizes on disk. Which means different regionfile sector +allocations. Which means there are 3 different possible +regionfile sector allocations in memory, and none of them +are going to be correct. + +I've fixed this by writing a world upgrader suited to +CB's changes to world folder format. It was brain dead +easy to add threading, so I did. + +== AT == +public net.minecraft.util.worldupdate.WorldUpgrader REGEX + +diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java +new file mode 100644 +index 0000000000000000000000000000000000000000..e049fbe4038aaea896f45b11ce9ce8f05922c898 +--- /dev/null ++++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java +@@ -0,0 +1,222 @@ ++package io.papermc.paper.world; ++ ++import com.mojang.datafixers.DataFixer; ++import com.mojang.serialization.MapCodec; ++import net.minecraft.SharedConstants; ++import net.minecraft.core.RegistryAccess; ++import net.minecraft.core.registries.Registries; ++import net.minecraft.nbt.CompoundTag; ++import net.minecraft.resources.ResourceKey; ++import net.minecraft.util.worldupdate.WorldUpgrader; ++import net.minecraft.world.level.ChunkPos; ++import net.minecraft.world.level.Level; ++import net.minecraft.world.level.chunk.ChunkGenerator; ++import net.minecraft.world.level.chunk.storage.ChunkStorage; ++import net.minecraft.world.level.chunk.storage.RegionFileStorage; ++import net.minecraft.world.level.chunk.storage.RegionStorageInfo; ++import net.minecraft.world.level.dimension.LevelStem; ++import net.minecraft.world.level.storage.DimensionDataStorage; ++import net.minecraft.world.level.storage.LevelStorageSource; ++import org.apache.logging.log4j.LogManager; ++import org.apache.logging.log4j.Logger; ++import java.io.File; ++import java.io.IOException; ++import java.text.DecimalFormat; ++import java.util.Optional; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.Executors; ++import java.util.concurrent.ThreadFactory; ++import java.util.concurrent.atomic.AtomicInteger; ++import java.util.concurrent.atomic.AtomicLong; ++import java.util.function.Supplier; ++ ++public class ThreadedWorldUpgrader { ++ ++ private static final Logger LOGGER = LogManager.getLogger(); ++ ++ private final ResourceKey<LevelStem> dimensionType; ++ private final String worldName; ++ private final File worldDir; ++ private final ExecutorService threadPool; ++ private final DataFixer dataFixer; ++ private final RegistryAccess registryLookup; ++ private final Optional<ResourceKey<MapCodec<? extends ChunkGenerator>>> generatorKey; ++ private final boolean removeCaches; ++ private final boolean recreateRegionFiles; // TODO ++ ++ public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final String worldName, final File worldDir, final int threads, ++ final DataFixer dataFixer, final RegistryAccess registryLookup, final Optional<ResourceKey<MapCodec<? extends ChunkGenerator>>> generatorKey, ++ final boolean removeCaches, final boolean recreateRegionFiles) { ++ this.dimensionType = dimensionType; ++ this.worldName = worldName; ++ this.worldDir = worldDir; ++ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() { ++ private final AtomicInteger threadCounter = new AtomicInteger(); ++ ++ @Override ++ public Thread newThread(final Runnable run) { ++ final Thread ret = new Thread(run); ++ ++ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement()); ++ ret.setUncaughtExceptionHandler((thread, throwable) -> { ++ LOGGER.fatal("Error upgrading world", throwable); ++ }); ++ ++ return ret; ++ } ++ }); ++ this.dataFixer = dataFixer; ++ this.registryLookup = registryLookup; ++ this.generatorKey = generatorKey; ++ this.removeCaches = removeCaches; ++ this.recreateRegionFiles = recreateRegionFiles; ++ } ++ ++ public void convert() { ++ final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile(); ++ final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer, this.registryLookup); ++ ++ final File regionFolder = new File(worldFolder, "region"); ++ ++ LOGGER.info("Force upgrading {}", this.worldName); ++ LOGGER.info("Counting regionfiles for {}", this.worldName); ++ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> { ++ return WorldUpgrader.REGEX.matcher(name).matches(); ++ }); ++ if (regionFiles == null) { ++ LOGGER.info("Found no regionfiles to convert for world {}", this.worldName); ++ return; ++ } ++ LOGGER.info("Found {} regionfiles to convert", regionFiles.length); ++ LOGGER.info("Starting conversion now for world {}", this.worldName); ++ ++ // Only used for profiling, let's fill it anyways just in case ++ final RegionStorageInfo storageInfo = new RegionStorageInfo( ++ this.worldName, ++ ResourceKey.create(Registries.DIMENSION, this.dimensionType.location()), ++ "region" ++ ); ++ ++ final WorldInfo info = new WorldInfo(() -> worldPersistentData, ++ new ChunkStorage(storageInfo, regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); ++ ++ long expectedChunks = (long)regionFiles.length * (32L * 32L); ++ ++ for (final File regionFile : regionFiles) { ++ final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath()); ++ if (regionPos == null) { ++ expectedChunks -= (32L * 32L); ++ continue; ++ } ++ ++ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5)); ++ } ++ this.threadPool.shutdown(); ++ ++ final DecimalFormat format = new DecimalFormat("#0.00"); ++ ++ final long start = System.nanoTime(); ++ ++ while (!this.threadPool.isTerminated()) { ++ final long current = info.convertedChunks.get(); ++ ++ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks); ++ ++ try { ++ Thread.sleep(1000L); ++ } catch (final InterruptedException ignore) {} ++ } ++ ++ final long end = System.nanoTime(); ++ ++ try { ++ info.loader.close(); ++ } catch (final IOException ex) { ++ LOGGER.fatal("Failed to close chunk loader", ex); ++ } ++ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)", ++ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0)); ++ } ++ ++ private static final class WorldInfo { ++ ++ public final Supplier<DimensionDataStorage> persistentDataSupplier; ++ public final ChunkStorage loader; ++ public final boolean removeCaches; ++ public final ResourceKey<LevelStem> worldKey; ++ public final Optional<ResourceKey<MapCodec<? extends ChunkGenerator>>> generatorKey; ++ public final AtomicLong convertedChunks = new AtomicLong(); ++ public final AtomicLong modifiedChunks = new AtomicLong(); ++ ++ private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches, ++ final ResourceKey<LevelStem> worldKey, Optional<ResourceKey<MapCodec<? extends ChunkGenerator>>> generatorKey) { ++ this.persistentDataSupplier = persistentDataSupplier; ++ this.loader = loader; ++ this.removeCaches = removeCaches; ++ this.worldKey = worldKey; ++ this.generatorKey = generatorKey; ++ } ++ } ++ ++ private static final class ConvertTask implements Runnable { ++ ++ private final WorldInfo worldInfo; ++ private final int regionX; ++ private final int regionZ; ++ ++ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) { ++ this.worldInfo = worldInfo; ++ this.regionX = regionX; ++ this.regionZ = regionZ; ++ } ++ ++ @Override ++ public void run() { ++ final int regionCX = this.regionX << 5; ++ final int regionCZ = this.regionZ << 5; ++ ++ final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier; ++ final ChunkStorage loader = this.worldInfo.loader; ++ final boolean removeCaches = this.worldInfo.removeCaches; ++ final ResourceKey<LevelStem> worldKey = this.worldInfo.worldKey; ++ ++ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) { ++ for (int cx = regionCX; cx < (regionCX + 32); ++cx) { ++ final ChunkPos chunkPos = new ChunkPos(cx, cz); ++ try { ++ // no need to check the coordinate of the chunk, the regionfilecache does that for us ++ ++ CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null); ++ ++ if (chunkNBT == null) { ++ continue; ++ } ++ ++ final int versionBefore = ChunkStorage.getVersion(chunkNBT); ++ ++ chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null); ++ ++ boolean modified = versionBefore < SharedConstants.getCurrentVersion().getDataVersion().getVersion(); ++ ++ if (removeCaches) { ++ final CompoundTag level = chunkNBT.getCompound("Level"); ++ modified |= level.contains("Heightmaps"); ++ level.remove("Heightmaps"); ++ modified |= level.contains("isLightOn"); ++ level.remove("isLightOn"); ++ } ++ ++ if (modified) { ++ this.worldInfo.modifiedChunks.getAndIncrement(); ++ loader.write(chunkPos, chunkNBT); ++ } ++ } catch (final Exception ex) { ++ LOGGER.error("Error upgrading chunk {}", chunkPos, ex); ++ } finally { ++ this.worldInfo.convertedChunks.getAndIncrement(); ++ } ++ } ++ } ++ } ++ } ++} +diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java +index 244a19ecd0234fa1d7a6ecfea20751595688605d..5443013060b62e3bfcc51cddca96d1c0bc59fe72 100644 +--- a/src/main/java/net/minecraft/server/Main.java ++++ b/src/main/java/net/minecraft/server/Main.java +@@ -392,6 +392,15 @@ public class Main { + return new WorldLoader.InitConfig(worldloader_d, Commands.CommandSelection.DEDICATED, serverPropertiesHandler.functionPermissionLevel); + } + ++ // Paper start - fix and optimise world upgrading ++ public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey<net.minecraft.world.level.dimension.LevelStem> dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession, ++ DataFixer dataFixer, RegistryAccess registryLookup, Optional<net.minecraft.resources.ResourceKey<com.mojang.serialization.MapCodec<? extends net.minecraft.world.level.chunk.ChunkGenerator>>> generatorKey, boolean removeCaches, boolean recreateRegionFiles) { ++ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8; ++ final io.papermc.paper.world.ThreadedWorldUpgrader worldUpgrader = new io.papermc.paper.world.ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, registryLookup, generatorKey, removeCaches, recreateRegionFiles); ++ worldUpgrader.convert(); ++ } ++ // Paper end - fix and optimise world upgrading ++ + public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, RegistryAccess dynamicRegistryManager, boolean recreateRegionFiles) { + Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit + WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, dynamicRegistryManager, eraseCache, recreateRegionFiles); +diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java +index 8515cec5e21ad291ca427baaafb4c2f337f01208..8dc2f9df367c849ca333bf1a1fd92ff91617b548 100644 +--- a/src/main/java/net/minecraft/server/MinecraftServer.java ++++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -599,11 +599,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + worlddata = new PrimaryLevelData(worldsettings, worldoptions, worlddimensions_b.specialWorldProperty(), lifecycle); + } + worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end) +- if (this.options.has("forceUpgrade")) { +- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.options.has("eraseCache"), () -> { +- return true; +- }, iregistrycustom_dimension, this.options.has("recreateRegionFiles")); +- } ++ // Paper - fix and optimise world upgrading; move down + + PrimaryLevelData iworlddataserver = worlddata; + boolean flag = worlddata.isDebugWorld(); +@@ -618,6 +614,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + biomeProvider = gen.getDefaultBiomeProvider(worldInfo); + } + ++ // Paper start - fix and optimise world upgrading ++ if (options.has("forceUpgrade")) { ++ net.minecraft.server.Main.convertWorldButItWorks( ++ dimensionKey, worldSession, DataFixers.getDataFixer(), iregistrycustom_dimension, worlddimension.generator().getTypeNameForDataFixer(), options.has("eraseCache"), options.has("recreateRegionFiles") ++ ); ++ } ++ // Paper end - fix and optimise world upgrading + ResourceKey<Level> worldKey = ResourceKey.create(Registries.DIMENSION, dimensionKey.location()); + + if (dimensionKey == LevelStem.OVERWORLD) { +diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java +index 0a8eeebb2d702ebcefd9f26cc0f41d1eab497902..b4ef3ad2c17168085372f1fe46809f02d9dfe74a 100644 +--- a/src/main/java/net/minecraft/world/level/Level.java ++++ b/src/main/java/net/minecraft/world/level/Level.java +@@ -178,6 +178,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable { + public final Map<Explosion.CacheKey, Float> explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions + public java.util.ArrayDeque<net.minecraft.world.level.block.RedstoneTorchBlock.Toggle> redstoneUpdateInfos; // Paper - Faster redstone torch rapid clock removal; Move from Map in BlockRedstoneTorch to here + ++ // Paper start - fix and optimise world upgrading ++ // copied from below ++ public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) { ++ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryAccess().registryOrThrow(net.minecraft.core.registries.Registries.DIMENSION_TYPE).getResourceKey(manager).orElseThrow(() -> { ++ return new IllegalStateException("Unregistered dimension type: " + manager); ++ }); ++ } ++ // Paper end - fix and optimise world upgrading ++ + public CraftWorld getWorld() { + return this.world; + } +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +index 249705ec1b8b692ef1d7fec34a04918afe6486bc..f6e3b745fc417354380d4a969f83aee430bad785 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +@@ -69,6 +69,29 @@ public class RegionFileStorage implements AutoCloseable { + } + + // Paper start ++ @Nullable ++ public static ChunkPos getRegionFileCoordinates(Path file) { ++ String fileName = file.getFileName().toString(); ++ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { ++ return null; ++ } ++ ++ String[] split = fileName.split("\\."); ++ ++ if (split.length != 4) { ++ return null; ++ } ++ ++ try { ++ int x = Integer.parseInt(split[1]); ++ int z = Integer.parseInt(split[2]); ++ ++ return new ChunkPos(x << 5, z << 5); ++ } catch (NumberFormatException ex) { ++ return null; ++ } ++ } ++ + public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { + return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ())); + } +diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java +index dd546e0680496d0626972b61b0eb183b07df0e6e..05e304f9fc8d0291fa779da589bd060ef4165b49 100644 +--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java ++++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java +@@ -1365,9 +1365,7 @@ public final class CraftServer implements Server { + worlddata.checkName(name); + worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified()); + +- if (this.console.options.has("forceUpgrade")) { +- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.console.options.has("eraseCache"), () -> true, iregistrycustom_dimension, this.console.options.has("recreateRegionFiles")); +- } ++ // Paper - fix and optimise world upgrading; move down + + long j = BiomeManager.obfuscateSeed(worlddata.worldGenOptions().seed()); // Paper - use world seed + List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata)); +@@ -1378,6 +1376,13 @@ public final class CraftServer implements Server { + biomeProvider = generator.getDefaultBiomeProvider(worldInfo); + } + ++ // Paper start - fix and optimise world upgrading ++ if (this.console.options.has("forceUpgrade")) { ++ net.minecraft.server.Main.convertWorldButItWorks( ++ actualDimension, worldSession, DataFixers.getDataFixer(), iregistrycustom_dimension, worlddimension.generator().getTypeNameForDataFixer(), this.console.options.has("eraseCache"), this.console.options.has("recreateRegionFiles") ++ ); ++ } ++ // Paper end - fix and optimise world upgrading + ResourceKey<net.minecraft.world.level.Level> worldKey; + String levelName = this.getServer().getProperties().levelName; + if (name.equals(levelName + "_nether")) { |