aboutsummaryrefslogtreecommitdiffhomepage
path: root/patches/server/0366-Improved-Watchdog-Support.patch
diff options
context:
space:
mode:
Diffstat (limited to 'patches/server/0366-Improved-Watchdog-Support.patch')
-rw-r--r--patches/server/0366-Improved-Watchdog-Support.patch548
1 files changed, 548 insertions, 0 deletions
diff --git a/patches/server/0366-Improved-Watchdog-Support.patch b/patches/server/0366-Improved-Watchdog-Support.patch
new file mode 100644
index 0000000000..520fe4923f
--- /dev/null
+++ b/patches/server/0366-Improved-Watchdog-Support.patch
@@ -0,0 +1,548 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Aikar <[email protected]>
+Date: Sun, 12 Apr 2020 15:50:48 -0400
+Subject: [PATCH] Improved Watchdog Support
+
+Forced Watchdog Crash support and Improve Async Shutdown
+
+If the request to shut down the server is received while we are in
+a watchdog hang, immediately treat it as a crash and begin the shutdown
+process. Shutdown process is now improved to also shutdown cleanly when
+not using restart scripts either.
+
+If a server is deadlocked, a server owner can send SIGUP (or any other signal
+the JVM understands to shut down as it currently does) and the watchdog
+will no longer need to wait until the full timeout, allowing you to trigger
+a close process and try to shut the server down gracefully, saving player and
+world data.
+
+Previously there was no way to trigger this outside of waiting for a full watchdog
+timeout, which may be set to a really long time...
+
+Additionally, fix everything to do with shutting the server down asynchronously.
+
+Previously, nearly everything about the process was fragile and unsafe. Main might
+not have actually been frozen, and might still be manipulating state.
+
+Or, some reuest might ask main to do something in the shutdown but main is dead.
+
+Or worse, other things might start closing down items such as the Console or Thread Pool
+before we are fully shutdown.
+
+This change tries to resolve all of these issues by moving everything into the stop
+method and guaranteeing only one thread is stopping the server.
+
+We then issue Thread Death to the main thread of another thread initiates the stop process.
+We have to ensure Thread Death propagates correctly though to stop main completely.
+
+This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.
+
+This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
+are properly accounted for and wont trip watchdog on init.
+
+diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
+index 6aaed8e8bf8c721fc834da5c76ac72a4c3e92458..4b002e8b75d117b726b0de274a76d3596fce015b 100644
+--- a/src/main/java/com/destroystokyo/paper/Metrics.java
++++ b/src/main/java/com/destroystokyo/paper/Metrics.java
+@@ -92,7 +92,12 @@ public class Metrics {
+ * Starts the Scheduler which submits our data every 30 minutes.
+ */
+ private void startSubmitting() {
+- final Runnable submitTask = this::submitData;
++ final Runnable submitTask = () -> {
++ if (MinecraftServer.getServer().hasStopped()) {
++ return;
++ }
++ submitData();
++ };
+
+ // Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the
+ // bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay.
+diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
+index a9a0248b1bd1ac454064e977b61f9b7d80962ff8..059b1a0bf048af6a28c322f35da3d3cbbe426546 100644
+--- a/src/main/java/net/minecraft/CrashReport.java
++++ b/src/main/java/net/minecraft/CrashReport.java
+@@ -233,6 +233,7 @@ public class CrashReport {
+ }
+
+ public static CrashReport forThrowable(Throwable cause, String title) {
++ if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
+ while (cause instanceof CompletionException && cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
+index 2deb639d404943ef5b028c4ede59cab99b31a40f..08f7f287af32597d8a39f429013adec9266020bf 100644
+--- a/src/main/java/net/minecraft/server/MinecraftServer.java
++++ b/src/main/java/net/minecraft/server/MinecraftServer.java
+@@ -297,7 +297,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
+ public int autosavePeriod;
+ public Commands vanillaCommandDispatcher;
+- private boolean forceTicks;
++ public boolean forceTicks; // Paper
+ // CraftBukkit end
+ // Spigot start
+ public static final int TPS = 20;
+@@ -308,6 +308,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations;
+ public static long currentTickLong = 0L; // Paper
+
++ public volatile Thread shutdownThread; // Paper
++ public volatile boolean abnormalExit = false; // Paper
++
+ public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
+ AtomicReference<S> atomicreference = new AtomicReference();
+ Thread thread = new io.papermc.paper.util.TickThread(() -> { // Paper - rewrite chunk system
+@@ -914,6 +917,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+
+ // CraftBukkit start
+ private boolean hasStopped = false;
++ public volatile boolean hasFullyShutdown = false; // Paper
+ private final Object stopLock = new Object();
+ public final boolean hasStopped() {
+ synchronized (this.stopLock) {
+@@ -928,6 +932,19 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ if (this.hasStopped) return;
+ this.hasStopped = true;
+ }
++ // Paper start - kill main thread, and kill it hard
++ shutdownThread = Thread.currentThread();
++ org.spigotmc.WatchdogThread.doStop(); // Paper
++ if (!isSameThread()) {
++ MinecraftServer.LOGGER.info("Stopping main thread (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)");
++ while (this.getRunningThread().isAlive()) {
++ this.getRunningThread().stop();
++ try {
++ Thread.sleep(1);
++ } catch (InterruptedException e) {}
++ }
++ }
++ // Paper end
+ // CraftBukkit end
+ if (this.metricsRecorder.isRecording()) {
+ this.cancelRecordingMetrics();
+@@ -981,7 +998,20 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ this.getProfileCache().save(false); // Paper
+ }
+ // Spigot end
++ // Paper start - move final shutdown items here
++ LOGGER.info("Flushing Chunk IO");
++ // Paper end - move final shutdown items here
+ io.papermc.paper.chunk.system.io.RegionFileIOThread.close(true); // Paper // Paper - rewrite chunk system
++ // Paper start - move final shutdown items here
++ LOGGER.info("Closing Thread Pool");
++ Util.shutdownExecutors(); // Paper
++ LOGGER.info("Closing Server");
++ try {
++ net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++ } catch (Exception e) {
++ }
++ this.onServerExit();
++ // Paper end - move final shutdown items here
+ }
+
+ public String getLocalIp() {
+@@ -1076,6 +1106,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+
+ protected void runServer() {
+ try {
++ long serverStartTime = Util.getNanos(); // Paper
+ if (!this.initServer()) {
+ throw new IllegalStateException("Failed to initialize server");
+ }
+@@ -1085,6 +1116,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ this.status = this.buildServerStatus();
+
+ // Spigot start
++ // Paper start - move done tracking
++ LOGGER.info("Running delayed init tasks");
++ this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init,
++ // this is going to be the first thing the tick process does anyways, so move done and run it after
++ // everything is init before watchdog tick.
++ // anything at 3+ won't be caught here but also will trip watchdog....
++ // tasks are default scheduled at -1 + delay, and first tick will tick at 1
++ String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (Util.getNanos() - serverStartTime) / 1.0E9D);
++ LOGGER.info("Done ({})! For help, type \"help\"", doneTime);
++ // Paper end
++
++ org.spigotmc.WatchdogThread.tick(); // Paper
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
+ Arrays.fill( this.recentTps, 20 );
+ long tickSection = Util.getNanos(), curTime, tickCount = 1; // Paper
+@@ -1163,6 +1206,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ JvmProfiler.INSTANCE.onServerTick(this.smoothedTickTimeMillis);
+ }
+ } catch (Throwable throwable) {
++ // Paper start
++ if (throwable instanceof ThreadDeath) {
++ MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
++ return;
++ }
++ // Paper end
+ MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
+ // Spigot Start
+ if ( throwable.getCause() != null )
+@@ -1193,14 +1242,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ this.services.profileCache().clearExecutor();
+ }
+
+- org.spigotmc.WatchdogThread.doStop(); // Spigot
++ //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
+ // CraftBukkit start - Restore terminal to original settings
+ try {
+- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
+ } catch (Exception ignored) {
+ }
+ // CraftBukkit end
+- this.onServerExit();
++ //this.onServerExit(); // Paper - moved into stop
+ }
+
+ }
+@@ -1269,6 +1318,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+
+ @Override
+ public TickTask wrapRunnable(Runnable runnable) {
++ // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
++ if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
++ runnable.run();
++ runnable = () -> {};
++ }
++ // Paper end
+ return new TickTask(this.tickCount, runnable);
+ }
+
+@@ -1547,6 +1602,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+ try {
+ crashreport = CrashReport.forThrowable(throwable, "Exception ticking world");
+ } catch (Throwable t) {
++ if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper
+ throw new RuntimeException("Error generating crash report", t);
+ }
+ // Spigot End
+@@ -2060,7 +2116,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
+
+ this.worldData.setDataConfiguration(worlddataconfiguration);
+ this.resources.managers.updateRegistryTags(this.registryAccess());
+- this.getPlayerList().saveAll();
++ // Paper start
++ if (Thread.currentThread() != this.serverThread) {
++ return;
++ }
++ // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements
++ for (ServerPlayer player : this.getPlayerList().getPlayers()) {
++ player.getAdvancements().save();
++ }
++ // Paper end
+ this.getPlayerList().reloadResources();
+ this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary());
+ this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager);
+diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+index 109c7ff78d4c1f5496d294f52ecfd9df2070db1e..fe47a38137f7b7fa94c507e790eec4fb7303595f 100644
+--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
++++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+@@ -269,7 +269,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+ long j = Util.getNanos() - i;
+ String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
+
+- DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
++ //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init
+ if (dedicatedserverproperties.announcePlayerAchievements != null) {
+ ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this);
+ }
+@@ -397,7 +397,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+ // this.remoteStatusListener.stop(); // Paper - don't wait for remote connections
+ }
+
+- System.exit(0); // CraftBukkit
++ hasFullyShutdown = true; // Paper
++ System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
+ }
+
+ @Override
+@@ -774,7 +775,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
+ @Override
+ public void stopServer() {
+ super.stopServer();
+- Util.shutdownExecutors();
++ //Util.shutdownExecutors(); // Paper - moved into super
+ SkullBlockEntity.clear();
+ }
+
+diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
+index 9036d9941b1597e14d3b25a5143d914248338800..f644c8b89e96d35fb9f2ac7941dcfcf9e8b66451 100644
+--- a/src/main/java/net/minecraft/server/players/PlayerList.java
++++ b/src/main/java/net/minecraft/server/players/PlayerList.java
+@@ -532,7 +532,7 @@ public abstract class PlayerList {
+ this.cserver.getPluginManager().callEvent(playerQuitEvent);
+ entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
+
+- entityplayer.doTick(); // SPIGOT-924
++ if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
+ // CraftBukkit end
+
+ // Paper start - Remove from collideRule team if needed
+diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+index f5829ae484d93b547a5437b85a9621346384a11b..83701fbfaa56a232593ee8f11a3afb8941238bfa 100644
+--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
++++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+@@ -152,6 +152,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
+ try {
+ task.run();
+ } catch (Exception var3) {
++ if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper
+ LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3);
+ }
+
+diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
+index 35fd928c87403d98a80b6c13fa9df83496576c8f..179a7708a5f4674b31a87c49ca3d5c4c4dbd4b65 100644
+--- a/src/main/java/net/minecraft/world/level/Level.java
++++ b/src/main/java/net/minecraft/world/level/Level.java
+@@ -915,6 +915,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
+ try {
+ tickConsumer.accept(entity);
+ } catch (Throwable throwable) {
++ if (throwable instanceof ThreadDeath) throw throwable; // Paper
+ // Paper start - Prevent tile entity and entity crashes
+ final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level().getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
+ MinecraftServer.LOGGER.error(msg, throwable);
+diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+index 12834995b2bab9e82a40feb01f038532961296cf..12af77215bfd6df3b6802a567ac3c013a4cdf06a 100644
+--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
++++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+@@ -1177,6 +1177,7 @@ public class LevelChunk extends ChunkAccess {
+
+ gameprofilerfiller.pop();
+ } catch (Throwable throwable) {
++ if (throwable instanceof ThreadDeath) throw throwable; // Paper
+ // Paper start - Prevent tile entity and entity crashes
+ final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
+ net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
+diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java
+index 17d74aa6d857e49b61f5aed30be2b17be871df4c..ff16522a4ae4e0cb32f4c6b919bcaffc4507dfe8 100644
+--- a/src/main/java/org/bukkit/craftbukkit/Main.java
++++ b/src/main/java/org/bukkit/craftbukkit/Main.java
+@@ -179,6 +179,36 @@ public class Main {
+
+ OptionSet options = null;
+
++ // Paper start - preload logger classes to avoid plugins mixing versions
++ tryPreloadClass("org.apache.logging.log4j.core.Core");
++ tryPreloadClass("org.apache.logging.log4j.core.appender.AsyncAppender");
++ tryPreloadClass("org.apache.logging.log4j.core.Appender");
++ tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector");
++ tryPreloadClass("org.apache.logging.log4j.core.Filter");
++ tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler");
++ tryPreloadClass("org.apache.logging.log4j.core.LogEvent");
++ tryPreloadClass("org.apache.logging.log4j.core.Logger");
++ tryPreloadClass("org.apache.logging.log4j.core.LoggerContext");
++ tryPreloadClass("org.apache.logging.log4j.core.LogEventListener");
++ tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent");
++ tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable");
++ tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage");
++ tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage");
++ tryPreloadClass("org.apache.logging.log4j.message.Message");
++ tryPreloadClass("org.apache.logging.log4j.message.MessageFactory");
++ tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage");
++ tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage");
++ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger");
++ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext");
++ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy");
++ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor");
++ tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent");
++ tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil");
++ tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler");
++ tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy");
++ tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo");
++ tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement");
++ // Paper end
+ try {
+ options = parser.parse(args);
+ } catch (joptsimple.OptionException ex) {
+@@ -289,8 +319,64 @@ public class Main {
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
++ // Paper start
++ // load some required classes to avoid errors during shutdown if jar is replaced
++ // also to guarantee our version loads over plugins
++ tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow");
++ tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl");
++ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values");
++ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator");
++ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry");
++ tryPreloadClass("com.google.common.collect.Iterables");
++ for (int i = 1; i <= 15; i++) {
++ tryPreloadClass("com.google.common.collect.Iterables$" + i, false);
++ }
++ tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean");
++ tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt");
++ tryPreloadClass("org.jline.terminal.impl.MouseSupport");
++ tryPreloadClass("org.jline.terminal.impl.MouseSupport$1");
++ tryPreloadClass("org.jline.terminal.Terminal$MouseTracking");
++ tryPreloadClass("co.aikar.timings.TimingHistory");
++ tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport");
++ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext");
++ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11");
++ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$12");
++ tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8");
++ tryPreloadClass("io.netty.util.concurrent.DefaultPromise");
++ tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1");
++ tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil");
++ tryPreloadClass("io.netty.util.internal.SystemPropertyUtil");
++ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler");
++ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1");
++ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2");
++ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3");
++ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4");
++ tryPreloadClass("org.slf4j.helpers.MessageFormatter");
++ tryPreloadClass("org.slf4j.helpers.FormattingTuple");
++ tryPreloadClass("org.slf4j.helpers.BasicMarker");
++ tryPreloadClass("org.slf4j.helpers.Util");
++ tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent");
++ tryPreloadClass("com.destroystokyo.paper.event.entity.EntityRemoveFromWorldEvent");
++ // Minecraft, seen during saving
++ tryPreloadClass(net.minecraft.world.level.lighting.LayerLightEventListener.DummyLightLayerEventListener.class.getName());
++ tryPreloadClass(net.minecraft.world.level.lighting.LayerLightEventListener.class.getName());
++ tryPreloadClass(net.minecraft.util.ExceptionCollector.class.getName());
++ // Paper end
++ }
++ }
++
++ // Paper start
++ private static void tryPreloadClass(String className) {
++ tryPreloadClass(className, true);
++ }
++ private static void tryPreloadClass(String className, boolean printError) {
++ try {
++ Class.forName(className);
++ } catch (ClassNotFoundException e) {
++ if (printError) System.err.println("An expected class " + className + " was not found for preloading: " + e.getMessage());
+ }
+ }
++ // Paper end
+
+ private static List<String> asList(String... params) {
+ return Arrays.asList(params);
+diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+index c6e8441e299f477ddb22c1ce2618710763978f1a..e8e93538dfd71de86515d9405f728db1631e949a 100644
+--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
++++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+@@ -12,11 +12,27 @@ public class ServerShutdownThread extends Thread {
+ @Override
+ public void run() {
+ try {
++ // Paper start - try to shutdown on main
++ server.safeShutdown(false, false);
++ for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
++ Thread.sleep(100);
++ }
++ if (server.hasStopped()) {
++ while (!server.hasFullyShutdown) Thread.sleep(1000);
++ return;
++ }
++ // Looks stalled, close async
+ org.spigotmc.AsyncCatcher.enabled = false; // Spigot
++ server.forceTicks = true;
+ this.server.close();
++ while (!server.hasFullyShutdown) Thread.sleep(1000);
++ } catch (InterruptedException e) {
++ e.printStackTrace();
++ // Paper end
+ } finally {
++ org.apache.logging.log4j.LogManager.shutdown(); // Paper
+ try {
+- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
++ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
+ } catch (Exception e) {
+ }
+ }
+diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
+index 051b9e3a5d29a5840d596468e3ddd013bedc8da3..631457374fda492776f4af0515cee2e43a934d2c 100644
+--- a/src/main/java/org/spigotmc/RestartCommand.java
++++ b/src/main/java/org/spigotmc/RestartCommand.java
+@@ -138,7 +138,7 @@ public class RestartCommand extends Command
+ // Paper end
+
+ // Paper start - copied from above and modified to return if the hook registered
+- private static boolean addShutdownHook(String restartScript)
++ public static boolean addShutdownHook(String restartScript)
+ {
+ String[] split = restartScript.split( " " );
+ if ( split.length > 0 && new File( split[0] ).isFile() )
+diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
+index 230d55820778e84c1c8aa2b013ae0e5e35568ea1..5ca863aa1859922fa359eba32539229db40e5b98 100644
+--- a/src/main/java/org/spigotmc/WatchdogThread.java
++++ b/src/main/java/org/spigotmc/WatchdogThread.java
+@@ -11,6 +11,7 @@ import org.bukkit.Bukkit;
+ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Paper - rewrite chunk system
+ {
+
++ public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
+ private static WatchdogThread instance;
+ private long timeoutTime;
+ private boolean restart;
+@@ -39,6 +40,7 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+ {
+ if ( WatchdogThread.instance == null )
+ {
++ if (timeoutTime <= 0) timeoutTime = 300; // Paper
+ WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
+ WatchdogThread.instance.start();
+ } else
+@@ -70,12 +72,13 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+ // Paper start
+ Logger log = Bukkit.getServer().getLogger();
+ long currentTime = WatchdogThread.monotonicMillis();
+- if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
++ MinecraftServer server = MinecraftServer.getServer();
++ if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
+ {
+- boolean isLongTimeout = currentTime > lastTick + timeoutTime;
++ boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
+ // Don't spam early warning dumps
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+- if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
++ if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+ lastEarlyWarning = currentTime;
+ if (isLongTimeout) {
+ // Paper end
+@@ -137,9 +140,24 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
+
+ if ( isLongTimeout )
+ {
+- if ( this.restart && !MinecraftServer.getServer().hasStopped() )
++ if ( !server.hasStopped() )
+ {
+- RestartCommand.restart();
++ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
++ server.forceTicks = true;
++ if (restart) {
++ RestartCommand.addShutdownHook( SpigotConfig.restartScript );
++ }
++ // try one last chance to safe shutdown on main incase it 'comes back'
++ server.abnormalExit = true;
++ server.safeShutdown(false, restart);
++ try {
++ Thread.sleep(1000);
++ } catch (InterruptedException e) {
++ e.printStackTrace();
++ }
++ if (!server.hasStopped()) {
++ server.close();
++ }
+ }
+ break;
+ } // Paper end
+diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
+index ea4e2161c0bd43884055cc6b8d70b2139f70e720..266b4e6fb3988b5848021c83fdc68e342c70b188 100644
+--- a/src/main/resources/log4j2.xml
++++ b/src/main/resources/log4j2.xml
+@@ -1,5 +1,5 @@
+ <?xml version="1.0" encoding="UTF-8"?>
+-<Configuration status="WARN" packages="com.mojang.util">
++<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable">
+ <Appenders>
+ <Queue name="ServerGuiConsole">
+ <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />