aboutsummaryrefslogtreecommitdiffhomepage
path: root/patches/server/0998-Fix-and-optimise-world-force-upgrading.patch
blob: 1e5f26c776e1083943840f238d8a0ff6a0a26fed (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Thu, 20 May 2021 07:02:22 -0700
Subject: [PATCH] Fix and optimise world force upgrading

The WorldUpgrader class was incorrectly modified by
CB. It will store an IChunkLoader instance for all
dimension types in the world, but obviously with how
CB shifts around worlds only one dimension type exists
per world. But this would be OK if CB did this
change correctly. All IChunkLoader instances
will point to the same regionfiles. And all
IChunkLoader instances are going to be read from.

This problem hasn't really been reported because
it relies on the persistent legacy data to be converted
as well to cause corruption. Why? Because the legacy
data is also shared, it will result in different
outputs from conversion (as once conversion for legacy
persistent data takes place, it is REMOVED - so the next
convert will _not_ have the data). Which means different
sizes on disk. Which means different regionfile sector
allocations. Which means there are 3 different possible
regionfile sector allocations in memory, and none of them
are going to be correct.

I've fixed this by writing a world upgrader suited to
CB's changes to world folder format. It was brain dead
easy to add threading, so I did.

== AT ==
public net.minecraft.util.worldupdate.WorldUpgrader REGEX

diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
new file mode 100644
index 0000000000000000000000000000000000000000..513833c2ea23df5b079d157bc5cb89d5c9754c0b
--- /dev/null
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
@@ -0,0 +1,209 @@
+package io.papermc.paper.world;
+
+import com.mojang.datafixers.DataFixer;
+import com.mojang.serialization.Codec;
+import net.minecraft.SharedConstants;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.resources.ResourceKey;
+import net.minecraft.util.worldupdate.WorldUpgrader;
+import net.minecraft.world.level.ChunkPos;
+import net.minecraft.world.level.Level;
+import net.minecraft.world.level.chunk.ChunkGenerator;
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
+import net.minecraft.world.level.dimension.DimensionType;
+import net.minecraft.world.level.dimension.LevelStem;
+import net.minecraft.world.level.levelgen.WorldGenSettings;
+import net.minecraft.world.level.storage.DimensionDataStorage;
+import net.minecraft.world.level.storage.LevelStorageSource;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+public class ThreadedWorldUpgrader {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private final ResourceKey<LevelStem> dimensionType;
+    private final String worldName;
+    private final File worldDir;
+    private final ExecutorService threadPool;
+    private final DataFixer dataFixer;
+    private final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
+    private final boolean removeCaches;
+
+    public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final String worldName, final File worldDir, final int threads,
+                                 final DataFixer dataFixer, final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey, final boolean removeCaches) {
+        this.dimensionType = dimensionType;
+        this.worldName = worldName;
+        this.worldDir = worldDir;
+        this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
+            private final AtomicInteger threadCounter = new AtomicInteger();
+
+            @Override
+            public Thread newThread(final Runnable run) {
+                final Thread ret = new Thread(run);
+
+                ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
+                ret.setUncaughtExceptionHandler((thread, throwable) -> {
+                    LOGGER.fatal("Error upgrading world", throwable);
+                });
+
+                return ret;
+            }
+        });
+        this.dataFixer = dataFixer;
+        this.generatorKey = generatorKey;
+        this.removeCaches = removeCaches;
+    }
+
+    public void convert() {
+        final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile();
+        final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer);
+
+        final File regionFolder = new File(worldFolder, "region");
+
+        LOGGER.info("Force upgrading " + this.worldName);
+        LOGGER.info("Counting regionfiles for " + this.worldName);
+        final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
+            return WorldUpgrader.REGEX.matcher(name).matches();
+        });
+        if (regionFiles == null) {
+            LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
+            return;
+        }
+        LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
+        LOGGER.info("Starting conversion now for world " + this.worldName);
+
+        final WorldInfo info = new WorldInfo(() -> worldPersistentData,
+                new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey);
+
+        long expectedChunks = (long)regionFiles.length * (32L * 32L);
+
+        for (final File regionFile : regionFiles) {
+            final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath());
+            if (regionPos == null) {
+                expectedChunks -= (32L * 32L);
+                continue;
+            }
+
+            this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
+        }
+        this.threadPool.shutdown();
+
+        final DecimalFormat format = new DecimalFormat("#0.00");
+
+        final long start = System.nanoTime();
+
+        while (!this.threadPool.isTerminated()) {
+            final long current = info.convertedChunks.get();
+
+            LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
+
+            try {
+                Thread.sleep(1000L);
+            } catch (final InterruptedException ignore) {}
+        }
+
+        final long end = System.nanoTime();
+
+        try {
+            info.loader.close();
+        } catch (final IOException ex) {
+            LOGGER.fatal("Failed to close chunk loader", ex);
+        }
+        LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
+                (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
+    }
+
+    private static final class WorldInfo {
+
+        public final Supplier<DimensionDataStorage> persistentDataSupplier;
+        public final ChunkStorage loader;
+        public final boolean removeCaches;
+        public final ResourceKey<LevelStem> worldKey;
+        public final Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey;
+        public final AtomicLong convertedChunks = new AtomicLong();
+        public final AtomicLong modifiedChunks = new AtomicLong();
+
+        private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches,
+                          final ResourceKey<LevelStem> worldKey, Optional<ResourceKey<Codec<? extends ChunkGenerator>>> generatorKey) {
+            this.persistentDataSupplier = persistentDataSupplier;
+            this.loader = loader;
+            this.removeCaches = removeCaches;
+            this.worldKey = worldKey;
+            this.generatorKey = generatorKey;
+        }
+    }
+
+    private static final class ConvertTask implements Runnable {
+
+        private final WorldInfo worldInfo;
+        private final int regionX;
+        private final int regionZ;
+
+        public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
+            this.worldInfo = worldInfo;
+            this.regionX = regionX;
+            this.regionZ = regionZ;
+        }
+
+        @Override
+        public void run() {
+            final int regionCX = this.regionX << 5;
+            final int regionCZ = this.regionZ << 5;
+
+            final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
+            final ChunkStorage loader = this.worldInfo.loader;
+            final boolean removeCaches = this.worldInfo.removeCaches;
+            final ResourceKey<LevelStem> worldKey = this.worldInfo.worldKey;
+
+            for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
+                for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
+                    final ChunkPos chunkPos = new ChunkPos(cx, cz);
+                    try {
+                        // no need to check the coordinate of the chunk, the regionfilecache does that for us
+
+                        CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null);
+
+                        if (chunkNBT == null) {
+                            continue;
+                        }
+
+                        final int versionBefore = ChunkStorage.getVersion(chunkNBT);
+
+                        chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null);
+
+                        boolean modified = versionBefore < SharedConstants.getCurrentVersion().getDataVersion().getVersion();
+
+                        if (removeCaches) {
+                            final CompoundTag level = chunkNBT.getCompound("Level");
+                            modified |= level.contains("Heightmaps");
+                            level.remove("Heightmaps");
+                            modified |= level.contains("isLightOn");
+                            level.remove("isLightOn");
+                        }
+
+                        if (modified) {
+                            this.worldInfo.modifiedChunks.getAndIncrement();
+                            loader.write(chunkPos, chunkNBT);
+                        }
+                    } catch (final Exception ex) {
+                        LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
+                    } finally {
+                        this.worldInfo.convertedChunks.getAndIncrement();
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
index 329471af4f40e0a74612707cce96bb00819e6cf2..bc391d27399d8c22e78735ca39aa8ab45efb6413 100644
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
@@ -388,6 +388,15 @@ public class Main {
         return new WorldLoader.InitConfig(worldloader_d, Commands.CommandSelection.DEDICATED, serverPropertiesHandler.functionPermissionLevel);
     }
 
+    // Paper start - fix and optimise world upgrading
+    public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey<net.minecraft.world.level.dimension.LevelStem> dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession,
+                                              DataFixer dataFixer, Optional<net.minecraft.resources.ResourceKey<com.mojang.serialization.Codec<? extends net.minecraft.world.level.chunk.ChunkGenerator>>> generatorKey, boolean removeCaches) {
+        int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
+        final io.papermc.paper.world.ThreadedWorldUpgrader worldUpgrader = new io.papermc.paper.world.ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, generatorKey, removeCaches);
+        worldUpgrader.convert();
+    }
+    // Paper end - fix and optimise world upgrading
+
     public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, Registry<LevelStem> dimensionOptionsRegistry) {
         Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit
         WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, dimensionOptionsRegistry, eraseCache);
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 79de3c639795cfc0bd86f842446e2bb3ab71d23a..1dfafbe508b4e4598339f412e5fb9d92717b5d26 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -584,11 +584,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 worlddata = new PrimaryLevelData(worldsettings, worldoptions, worlddimensions_b.specialWorldProperty(), lifecycle);
             }
             worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
-            if (this.options.has("forceUpgrade")) {
-                net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.options.has("eraseCache"), () -> {
-                    return true;
-                }, dimensions);
-            }
+            // Paper - fix and optimise world upgrading; move down
 
             PrimaryLevelData iworlddataserver = worlddata;
             boolean flag = worlddata.isDebugWorld();
@@ -603,6 +599,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 biomeProvider = gen.getDefaultBiomeProvider(worldInfo);
             }
 
+            // Paper start - fix and optimise world upgrading
+            if (options.has("forceUpgrade")) {
+                net.minecraft.server.Main.convertWorldButItWorks(
+                    dimensionKey, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), options.has("eraseCache")
+                );
+            }
+            // Paper end - fix and optimise world upgrading
             ResourceKey<Level> worldKey = ResourceKey.create(Registries.DIMENSION, dimensionKey.location());
 
             if (dimensionKey == LevelStem.OVERWORLD) {
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
index 29368a97c6348e2d1d6241db3d57e84b9537231a..23643b18e9352289ddad1eb1e04d1c3ada5570ad 100644
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
@@ -181,6 +181,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
     public final Map<Explosion.CacheKey, Float> explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions
     public java.util.ArrayDeque<net.minecraft.world.level.block.RedstoneTorchBlock.Toggle> redstoneUpdateInfos; // Paper - Faster redstone torch rapid clock removal; Move from Map in BlockRedstoneTorch to here
 
+    // Paper start - fix and optimise world upgrading
+    // copied from below
+    public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) {
+        return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryAccess().registryOrThrow(net.minecraft.core.registries.Registries.DIMENSION_TYPE).getResourceKey(manager).orElseThrow(() -> {
+            return new IllegalStateException("Unregistered dimension type: " + manager);
+        });
+    }
+    // Paper end - fix and optimise world upgrading
+
     public CraftWorld getWorld() {
         return this.world;
     }
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
index 0db8ee3b640e6d1268e9c1cccda85459bd447105..42d37bee3a459adcd46408596ccf93abbcff51fe 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
@@ -60,6 +60,29 @@ public class RegionFileStorage implements AutoCloseable {
     }
 
     // Paper start
+    @Nullable
+    public static ChunkPos getRegionFileCoordinates(Path file) {
+        String fileName = file.getFileName().toString();
+        if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
+            return null;
+        }
+
+        String[] split = fileName.split("\\.");
+
+        if (split.length != 4) {
+            return null;
+        }
+
+        try {
+            int x = Integer.parseInt(split[1]);
+            int z = Integer.parseInt(split[2]);
+
+            return new ChunkPos(x << 5, z << 5);
+        } catch (NumberFormatException ex) {
+            return null;
+        }
+    }
+    
     public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) {
         return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
     }
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index c9137151f0d2978adb432c40da68689465d2325d..ab7bc27e870227e6746b77a7b5e109e2cf198b5f 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
@@ -1357,9 +1357,7 @@ public final class CraftServer implements Server {
         worlddata.checkName(name);
         worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified());
 
-        if (this.console.options.has("forceUpgrade")) {
-            net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.console.options.has("eraseCache"), () -> true, iregistry);
-        }
+        // Paper - fix and optimise world upgrading; move down
 
         long j = BiomeManager.obfuscateSeed(worlddata.worldGenOptions().seed()); // Paper - use world seed
         List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata));
@@ -1370,6 +1368,13 @@ public final class CraftServer implements Server {
             biomeProvider = generator.getDefaultBiomeProvider(worldInfo);
         }
 
+        // Paper start - fix and optimise world upgrading
+        if (this.console.options.has("forceUpgrade")) {
+            net.minecraft.server.Main.convertWorldButItWorks(
+                actualDimension,  worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), this.console.options.has("eraseCache")
+            );
+        }
+        // Paper end - fix and optimise world upgrading
         ResourceKey<net.minecraft.world.level.Level> worldKey;
         String levelName = this.getServer().getProperties().levelName;
         if (name.equals(levelName + "_nether")) {