Rewrite chunk system (#8177)

Patch documentation to come

Issues with the old system that are fixed now:
- World generation does not scale with cpu cores effectively.
- Relies on the main thread for scheduling and maintaining chunk state, dropping chunk load/generate rates at lower tps.
- Unreliable prioritisation of chunk gen/load calls that block the main thread.
- Shutdown logic is utterly unreliable, as it has to wait for all chunks to unload - is it guaranteed that the chunk system is in a state on shutdown that it can reliably do this? Watchdog shutdown also typically failed due to thread checks, which is now resolved.
- Saving of data is not unified (i.e can save chunk data without saving entity data, poses problems for desync if shutdown is really abnormal.
- Entities are not loaded with chunks. This caused quite a bit of headache for Chunk#getEntities API, but now the new chunk system loads entities with chunks so that they are ready whenever the chunk loads in. Effectively brings the behavior back to 1.16 era, but still storing entities in their own separate regionfiles.

The above list is not complete. The patch documentation will complete it.

New chunk system hard relies on starlight and dataconverter, and most importantly the new concurrent utilities in ConcurrentUtil.

Some of the old async chunk i/o interface (i.e the old file io thread reroutes _some_ calls to the new file io thread) is kept for plugin compat reasons. It will be removed in the next major version of minecraft.

The old legacy chunk system patches have been moved to the removed folder in case we need them again.

1 files changed, 557 insertions, 0 deletions

diff --git a/patches/server/0379-Improved-Watchdog-Support.patch b/patches/server/0379-Improved-Watchdog-Support.patch
new file mode 100644
index 0000000000..96f8055ba4
--- /dev/null
+++ b/patches/server/0379-Improved-Watchdog-Support.patch
@@ -0,0 +1,557 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Aikar <[email protected]>
+Date: Sun, 12 Apr 2020 15:50:48 -0400
+Subject: [PATCH] Improved Watchdog Support
+
+Forced Watchdog Crash support and Improve Async Shutdown
+
+If the request to shut down the server is received while we are in
+a watchdog hang, immediately treat it as a crash and begin the shutdown
+process. Shutdown process is now improved to also shutdown cleanly when
+not using restart scripts either.
+
+If a server is deadlocked, a server owner can send SIGUP (or any other signal
+the JVM understands to shut down as it currently does) and the watchdog
+will no longer need to wait until the full timeout, allowing you to trigger
+a close process and try to shut the server down gracefully, saving player and
+world data.
+
+Previously there was no way to trigger this outside of waiting for a full watchdog
+timeout, which may be set to a really long time...
+
+Additionally, fix everything to do with shutting the server down asynchronously.
+
+Previously, nearly everything about the process was fragile and unsafe. Main might
+not have actually been frozen, and might still be manipulating state.
+
+Or, some reuest might ask main to do something in the shutdown but main is dead.
+
+Or worse, other things might start closing down items such as the Console or Thread Pool
+before we are fully shutdown.
+
+This change tries to resolve all of these issues by moving everything into the stop
+method and guaranteeing only one thread is stopping the server.
+
+We then issue Thread Death to the main thread of another thread initiates the stop process.
+We have to ensure Thread Death propagates correctly though to stop main completely.
+
+This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.
+
+This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
+are properly accounted for and wont trip watchdog on init.
+
+diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
+index 5a19e30a9b7e65a70f68a429b8ca741f788a303b..7b1843e16745ca8db2244e17490d291401f22679 100644
+--- a/src/main/java/com/destroystokyo/paper/Metrics.java
++++ b/src/main/java/com/destroystokyo/paper/Metrics.java
+@@ -92,7 +92,12 @@ public class Metrics {
+      * Starts the Scheduler which submits our data every 30 minutes.
+      */
+     private void startSubmitting() {
+-        final Runnable submitTask = this::submitData;
++        final Runnable submitTask = () -> {
++            if (MinecraftServer.getServer().hasStopped()) {
++                return;
++            }
++            submitData();
++        };
+ 
+         // Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the
+         // bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay.
+diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
+index 336795dff742b7c6957fbd3476aff31d25a5e659..30a58229aa6dac5039511d0c0df5f2912ea7de9f 100644
+--- a/src/main/java/net/minecraft/CrashReport.java
++++ b/src/main/java/net/minecraft/CrashReport.java
+@@ -230,6 +230,7 @@ public class CrashReport {
+     }
+ 
+     public static CrashReport forThrowable(Throwable cause, String title) {
++        if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
+         while (cause instanceof CompletionException && cause.getCause() != null) {
+             cause = cause.getCause();
+         }
+diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
+index a8cbbe13f7da0fe89191196acfa8617e683a66e7..1205d5f7d2bee68f6b3cd8e2ccbcd3d056963d8e 100644
+--- a/src/main/java/net/minecraft/server/MinecraftServer.java
++++ b/src/main/java/net/minecraft/server/MinecraftServer.java
+@@ -284,7 +284,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+     public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
+     public int autosavePeriod;
+     public Commands vanillaCommandDispatcher;
+-    private boolean forceTicks;
++    public boolean forceTicks; // Paper
+     // CraftBukkit end
+     // Spigot start
+     public static final int TPS = 20;
+@@ -295,6 +295,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+     public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations;
+     public static long currentTickLong = 0L; // Paper
+ 
++    public volatile Thread shutdownThread; // Paper
++    public volatile boolean abnormalExit = false; // Paper
++
+     public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
+         AtomicReference<S> atomicreference = new AtomicReference();
+         Thread thread = new io.papermc.paper.util.TickThread(() -> { // Paper - rewrite chunk system
+@@ -877,6 +880,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ 
+     // CraftBukkit start
+     private boolean hasStopped = false;
++    public volatile boolean hasFullyShutdown = false; // Paper
+     private final Object stopLock = new Object();
+     public final boolean hasStopped() {
+         synchronized (this.stopLock) {
+@@ -891,6 +895,19 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+             if (this.hasStopped) return;
+             this.hasStopped = true;
+         }
++        // Paper start - kill main thread, and kill it hard
++        shutdownThread = Thread.currentThread();
++        org.spigotmc.WatchdogThread.doStop(); // Paper
++        if (!isSameThread()) {
++            MinecraftServer.LOGGER.info("Stopping main thread (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)");
++            while (this.getRunningThread().isAlive()) {
++                this.getRunningThread().stop();
++                try {
++                    Thread.sleep(1);
++                } catch (InterruptedException e) {}
++            }
++        }
++        // Paper end
+         // CraftBukkit end
+         if (this.metricsRecorder.isRecording()) {
+             this.cancelRecordingMetrics();
+@@ -947,7 +964,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+             this.getProfileCache().save(false); // Paper
+         }
+         // Spigot end
++        // Paper start - move final shutdown items here
++        LOGGER.info("Flushing Chunk IO");
+         io.papermc.paper.chunk.system.io.RegionFileIOThread.close(true); // Paper // Paper - rewrite chunk system
++        LOGGER.info("Closing Thread Pool");
++        Util.shutdownExecutors(); // Paper
++        LOGGER.info("Closing Server");
++        try {
++            net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++        } catch (Exception e) {
++        }
++        this.onServerExit();
++        // Paper end
+     }
+ 
+     public String getLocalIp() {
+@@ -1042,6 +1070,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ 
+     protected void runServer() {
+         try {
++            long serverStartTime = Util.getNanos(); // Paper
+             if (!this.initServer()) {
+                 throw new IllegalStateException("Failed to initialize server");
+             }
+@@ -1054,6 +1083,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+             this.updateStatusIcon(this.status);
+ 
+             // Spigot start
++            // Paper start - move done tracking
++            LOGGER.info("Running delayed init tasks");
++            this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init,
++            // this is going to be the first thing the tick process does anyways, so move done and run it after
++            // everything is init before watchdog tick.
++            // anything at 3+ won't be caught here but also will trip watchdog....
++            // tasks are default scheduled at -1 + delay, and first tick will tick at 1
++            String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (Util.getNanos() - serverStartTime) / 1.0E9D);
++            LOGGER.info("Done ({})! For help, type \"help\"", doneTime);
++            // Paper end
++
++            org.spigotmc.WatchdogThread.tick(); // Paper
+             org.spigotmc.WatchdogThread.hasStarted = true; // Paper
+             Arrays.fill( recentTps, 20 );
+             long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
+@@ -1114,6 +1155,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+                 JvmProfiler.INSTANCE.onServerTick(this.averageTickTime);
+             }
+         } catch (Throwable throwable) {
++            // Paper start
++            if (throwable instanceof ThreadDeath) {
++                MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
++                return;
++            }
++            // Paper end
+             MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
+             // Spigot Start
+             if ( throwable.getCause() != null )
+@@ -1144,14 +1191,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+                     this.services.profileCache().clearExecutor();
+                 }
+ 
+-                org.spigotmc.WatchdogThread.doStop(); // Spigot
++                //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
+                 // CraftBukkit start - Restore terminal to original settings
+                 try {
+-                    net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++                    //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
+                 } catch (Exception ignored) {
+                 }
+                 // CraftBukkit end
+-                this.onServerExit();
++                //this.onServerExit(); // Paper - moved into stop
+             }
+ 
+         }
+@@ -1220,6 +1267,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ 
+     @Override
+     public TickTask wrapRunnable(Runnable runnable) {
++        // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
++        if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
++            runnable.run();
++            runnable = () -> {};
++        }
++        // Paper end
+         return new TickTask(this.tickCount, runnable);
+     }
+ 
+@@ -1446,6 +1499,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+                 try {
+                     crashreport = CrashReport.forThrowable(throwable, "Exception ticking world");
+                 } catch (Throwable t) {
++                    if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper
+                     throw new RuntimeException("Error generating crash report", t);
+                 }
+                 // Spigot End
+@@ -1937,7 +1991,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+             this.packRepository.setSelected(dataPacks);
+             this.worldData.setDataPackConfig(MinecraftServer.getSelectedPacks(this.packRepository));
+             this.resources.managers.updateRegistryTags(this.registryAccess());
+-            this.getPlayerList().saveAll();
++            // Paper start
++            if (Thread.currentThread() != this.serverThread) {
++                return;
++            }
++            // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements
++            for (ServerPlayer player : this.getPlayerList().getPlayers()) {
++                player.getAdvancements().save();
++            }
++            // Paper end
+             this.getPlayerList().reloadResources();
+             this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary());
+             this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager);
+diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+index 2a84d4b07b89672b76d31fa60bb4a0c1ba394831..abfcba1c3db090563191e8adea6d8250ae2d138e 100644
+--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
++++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+@@ -274,7 +274,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+             long j = Util.getNanos() - i;
+             String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
+ 
+-            DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
++            //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init
+             if (dedicatedserverproperties.announcePlayerAchievements != null) {
+                 ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this);
+             }
+@@ -404,7 +404,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+             //this.remoteStatusListener.b(); // Paper - don't wait for remote connections
+         }
+ 
+-        System.exit(0); // CraftBukkit
++        hasFullyShutdown = true; // Paper
++        System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
+     }
+ 
+     @Override
+@@ -772,7 +773,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+     @Override
+     public void stopServer() {
+         super.stopServer();
+-        Util.shutdownExecutors();
++        //Util.shutdownExecutors(); // Paper - moved into super
+         SkullBlockEntity.clear();
+     }
+ 
+diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
+index f738012c97024e81488ea6683160ee8236458cb1..401b1f440b7b1e4f12ba5e8080ca004971c56ae6 100644
+--- a/src/main/java/net/minecraft/server/players/PlayerList.java
++++ b/src/main/java/net/minecraft/server/players/PlayerList.java
+@@ -518,7 +518,7 @@ public abstract class PlayerList {
+         this.cserver.getPluginManager().callEvent(playerQuitEvent);
+         entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
+ 
+-        entityplayer.doTick(); // SPIGOT-924
++        if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
+         // CraftBukkit end
+ 
+         // Paper start - Remove from collideRule team if needed
+diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+index 6fefa619299d3202158490630d62c16aef71e831..7a4ade1a4190bf4fbb048919ae2be230f7b80fff 100644
+--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
++++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+@@ -152,6 +152,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
+         try {
+             task.run();
+         } catch (Exception var3) {
++            if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper
+             LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3);
+         }
+ 
+diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
+index 0dd6ccc5b281ea46d2d12eb99c28335bdbe66d7e..426fe552d2444a4977a3e261d7e60fbda0893756 100644
+--- a/src/main/java/net/minecraft/world/level/Level.java
++++ b/src/main/java/net/minecraft/world/level/Level.java
+@@ -799,6 +799,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
+         try {
+             tickConsumer.accept(entity);
+         } catch (Throwable throwable) {
++            if (throwable instanceof ThreadDeath) throw throwable; // Paper
+             // Paper start - Prevent tile entity and entity crashes
+             final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level.getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
+             MinecraftServer.LOGGER.error(msg, throwable);
+diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+index bd1c957a9405ccf18f110c7976cf8e0af922cf78..b0f53c99a89b900ffe49bdd277329829b44775d4 100644
+--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
++++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+@@ -1200,6 +1200,7 @@ public class LevelChunk extends ChunkAccess {
+ 
+                         gameprofilerfiller.pop();
+                     } catch (Throwable throwable) {
++                        if (throwable instanceof ThreadDeath) throw throwable; // Paper
+                         // Paper start - Prevent tile entity and entity crashes
+                         final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
+                         net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
+diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java
+index 08e74f41516a545a2371a7418d995ab288831834..cce6886bb3973eed8f0c7ca7b1189547324fd4e2 100644
+--- a/src/main/java/org/bukkit/craftbukkit/Main.java
++++ b/src/main/java/org/bukkit/craftbukkit/Main.java
+@@ -12,6 +12,8 @@ import java.util.logging.Level;
+ import java.util.logging.Logger;
+ import joptsimple.OptionParser;
+ import joptsimple.OptionSet;
++import net.minecraft.util.ExceptionCollector;
++import net.minecraft.world.level.lighting.LayerLightEventListener;
+ import net.minecrell.terminalconsole.TerminalConsoleAppender; // Paper
+ 
+ public class Main {
+@@ -169,6 +171,36 @@ public class Main {
+ 
+         OptionSet options = null;
+ 
++        // Paper start - preload logger classes to avoid plugins mixing versions
++        tryPreloadClass("org.apache.logging.log4j.core.Core");
++        tryPreloadClass("org.apache.logging.log4j.core.appender.AsyncAppender");
++        tryPreloadClass("org.apache.logging.log4j.core.Appender");
++        tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector");
++        tryPreloadClass("org.apache.logging.log4j.core.Filter");
++        tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler");
++        tryPreloadClass("org.apache.logging.log4j.core.LogEvent");
++        tryPreloadClass("org.apache.logging.log4j.core.Logger");
++        tryPreloadClass("org.apache.logging.log4j.core.LoggerContext");
++        tryPreloadClass("org.apache.logging.log4j.core.LogEventListener");
++        tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent");
++        tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable");
++        tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage");
++        tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage");
++        tryPreloadClass("org.apache.logging.log4j.message.Message");
++        tryPreloadClass("org.apache.logging.log4j.message.MessageFactory");
++        tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage");
++        tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage");
++        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger");
++        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext");
++        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy");
++        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor");
++        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent");
++        tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil");
++        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler");
++        tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy");
++        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo");
++        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement");
++        // Paper end
+         try {
+             options = parser.parse(args);
+         } catch (joptsimple.OptionException ex) {
+@@ -264,8 +296,64 @@ public class Main {
+             } catch (Throwable t) {
+                 t.printStackTrace();
+             }
++            // Paper start
++            // load some required classes to avoid errors during shutdown if jar is replaced
++            // also to guarantee our version loads over plugins
++            tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow");
++            tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl");
++            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values");
++            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator");
++            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry");
++            tryPreloadClass("com.google.common.collect.Iterables");
++            for (int i = 1; i <= 15; i++) {
++                tryPreloadClass("com.google.common.collect.Iterables$" + i, false);
++            }
++            tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean");
++            tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt");
++            tryPreloadClass("org.jline.terminal.impl.MouseSupport");
++            tryPreloadClass("org.jline.terminal.impl.MouseSupport$1");
++            tryPreloadClass("org.jline.terminal.Terminal$MouseTracking");
++            tryPreloadClass("co.aikar.timings.TimingHistory");
++            tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport");
++            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext");
++            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11");
++            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$12");
++            tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8");
++            tryPreloadClass("io.netty.util.concurrent.DefaultPromise");
++            tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1");
++            tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil");
++            tryPreloadClass("io.netty.util.internal.SystemPropertyUtil");
++            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler");
++            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1");
++            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2");
++            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3");
++            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4");
++            tryPreloadClass("org.slf4j.helpers.MessageFormatter");
++            tryPreloadClass("org.slf4j.helpers.FormattingTuple");
++            tryPreloadClass("org.slf4j.helpers.BasicMarker");
++            tryPreloadClass("org.slf4j.helpers.Util");
++            tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent");
++            tryPreloadClass("com.destroystokyo.paper.event.entity.EntityRemoveFromWorldEvent");
++            // Minecraft, seen during saving
++            tryPreloadClass(LayerLightEventListener.DummyLightLayerEventListener.class.getName());
++            tryPreloadClass(LayerLightEventListener.class.getName());
++            tryPreloadClass(ExceptionCollector.class.getName());
++            // Paper end
++        }
++    }
++
++    // Paper start
++    private static void tryPreloadClass(String className) {
++        tryPreloadClass(className, true);
++    }
++    private static void tryPreloadClass(String className, boolean printError) {
++        try {
++            Class.forName(className);
++        } catch (ClassNotFoundException e) {
++            if (printError) System.err.println("An expected class  " + className + " was not found for preloading: " + e.getMessage());
+         }
+     }
++    // Paper end
+ 
+     private static List<String> asList(String... params) {
+         return Arrays.asList(params);
+diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+index b4a19d80bbf71591f25729fd0e98590350cb31d0..e948ec5a573b22645664eb53bc3e9932246544e4 100644
+--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
++++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+@@ -12,12 +12,28 @@ public class ServerShutdownThread extends Thread {
+     @Override
+     public void run() {
+         try {
++            // Paper start - try to shutdown on main
++            server.safeShutdown(false, false);
++            for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
++                Thread.sleep(100);
++            }
++            if (server.hasStopped()) {
++                while (!server.hasFullyShutdown) Thread.sleep(1000);
++                return;
++            }
++            // Looks stalled, close async
+             org.spigotmc.AsyncCatcher.enabled = false; // Spigot
+             org.spigotmc.AsyncCatcher.shuttingDown = true; // Paper
++            server.forceTicks = true;
+             this.server.close();
++            while (!server.hasFullyShutdown) Thread.sleep(1000);
++        } catch (InterruptedException e) {
++            e.printStackTrace();
++            // Paper end
+         } finally {
++            org.apache.logging.log4j.LogManager.shutdown(); // Paper
+             try {
+-                net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++                //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
+             } catch (Exception e) {
+             }
+         }
+diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
+index a142a56a920e153ed84c08cece993f10d76f7793..92d97a5810a379b427a99b4c63fb9844d823a84f 100644
+--- a/src/main/java/org/spigotmc/RestartCommand.java
++++ b/src/main/java/org/spigotmc/RestartCommand.java
+@@ -139,7 +139,7 @@ public class RestartCommand extends Command
+     // Paper end
+ 
+     // Paper start - copied from above and modified to return if the hook registered
+-    private static boolean addShutdownHook(String restartScript)
++    public static boolean addShutdownHook(String restartScript)
+     {
+         String[] split = restartScript.split( " " );
+         if ( split.length > 0 && new File( split[0] ).isFile() )
+diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
+index b47d043144c499b1499f6b4be5a16a3f75c9fcb8..383c52c62f49b17db2fbf58009d6ea132d124bea 100644
+--- a/src/main/java/org/spigotmc/WatchdogThread.java
++++ b/src/main/java/org/spigotmc/WatchdogThread.java
+@@ -11,6 +11,7 @@ import org.bukkit.Bukkit;
+ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Paper - rewrite chunk system
+ {
+ 
++    public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
+     private static WatchdogThread instance;
+     private long timeoutTime;
+     private boolean restart;
+@@ -39,6 +40,7 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+     {
+         if ( WatchdogThread.instance == null )
+         {
++            if (timeoutTime <= 0) timeoutTime = 300; // Paper
+             WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
+             WatchdogThread.instance.start();
+         } else
+@@ -70,12 +72,13 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+             // Paper start
+             Logger log = Bukkit.getServer().getLogger();
+             long currentTime = WatchdogThread.monotonicMillis();
+-            if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
++            MinecraftServer server = MinecraftServer.getServer();
++            if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
+             {
+-                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
++                boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
+                 // Don't spam early warning dumps
+                 if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+-                if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
++                if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+                 lastEarlyWarning = currentTime;
+                 if (isLongTimeout) {
+                 // Paper end
+@@ -137,9 +140,25 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+ 
+                 if ( isLongTimeout )
+                 {
+-                if ( this.restart && !MinecraftServer.getServer().hasStopped() )
++                if ( !server.hasStopped() )
+                 {
+-                    RestartCommand.restart();
++                    AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
++                    AsyncCatcher.shuttingDown = true;
++                    server.forceTicks = true;
++                    if (restart) {
++                        RestartCommand.addShutdownHook( SpigotConfig.restartScript );
++                    }
++                    // try one last chance to safe shutdown on main incase it 'comes back'
++                    server.abnormalExit = true;
++                    server.safeShutdown(false, restart);
++                    try {
++                        Thread.sleep(1000);
++                    } catch (InterruptedException e) {
++                        e.printStackTrace();
++                    }
++                    if (!server.hasStopped()) {
++                        server.close();
++                    }
+                 }
+                 break;
+                 } // Paper end
+diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
+index 3dc317e466e1b93dff030794dd7f29ca1b266778..d285dbec16272db6b8a71865e05924ad66087407 100644
+--- a/src/main/resources/log4j2.xml
++++ b/src/main/resources/log4j2.xml
+@@ -1,5 +1,5 @@
+ <?xml version="1.0" encoding="UTF-8"?>
+-<Configuration status="WARN" packages="com.mojang.util">
++<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable">
+     <Appenders>
+         <Queue name="ServerGuiConsole">
+             <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />