diff options
Diffstat (limited to 'patches/server/1016-Improved-Watchdog-Support.patch')
| -rw-r--r-- | patches/server/1016-Improved-Watchdog-Support.patch | 475 |
1 files changed, 475 insertions, 0 deletions
diff --git a/patches/server/1016-Improved-Watchdog-Support.patch b/patches/server/1016-Improved-Watchdog-Support.patch new file mode 100644 index 0000000000..4c18803565 --- /dev/null +++ b/patches/server/1016-Improved-Watchdog-Support.patch @@ -0,0 +1,475 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aikar <[email protected]> +Date: Sun, 12 Apr 2020 15:50:48 -0400 +Subject: [PATCH] Improved Watchdog Support + +Forced Watchdog Crash support and Improve Async Shutdown + +If the request to shut down the server is received while we are in +a watchdog hang, immediately treat it as a crash and begin the shutdown +process. Shutdown process is now improved to also shutdown cleanly when +not using restart scripts either. + +If a server is deadlocked, a server owner can send SIGUP (or any other signal +the JVM understands to shut down as it currently does) and the watchdog +will no longer need to wait until the full timeout, allowing you to trigger +a close process and try to shut the server down gracefully, saving player and +world data. + +Previously there was no way to trigger this outside of waiting for a full watchdog +timeout, which may be set to a really long time... + +Additionally, fix everything to do with shutting the server down asynchronously. + +Previously, nearly everything about the process was fragile and unsafe. Main might +not have actually been frozen, and might still be manipulating state. + +Or, some reuest might ask main to do something in the shutdown but main is dead. + +Or worse, other things might start closing down items such as the Console or Thread Pool +before we are fully shutdown. + +This change tries to resolve all of these issues by moving everything into the stop +method and guaranteeing only one thread is stopping the server. + +We then issue Thread Death to the main thread of another thread initiates the stop process. +We have to ensure Thread Death propagates correctly though to stop main completely. + +This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save. + +This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they +are properly accounted for and wont trip watchdog on init. + +diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java +index 6aaed8e8bf8c721fc834da5c76ac72a4c3e92458..4b002e8b75d117b726b0de274a76d3596fce015b 100644 +--- a/src/main/java/com/destroystokyo/paper/Metrics.java ++++ b/src/main/java/com/destroystokyo/paper/Metrics.java +@@ -92,7 +92,12 @@ public class Metrics { + * Starts the Scheduler which submits our data every 30 minutes. + */ + private void startSubmitting() { +- final Runnable submitTask = this::submitData; ++ final Runnable submitTask = () -> { ++ if (MinecraftServer.getServer().hasStopped()) { ++ return; ++ } ++ submitData(); ++ }; + + // Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the + // bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay. +diff --git a/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java +new file mode 100644 +index 0000000000000000000000000000000000000000..183e141d0c13190c6905dc4510d891992afef878 +--- /dev/null ++++ b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java +@@ -0,0 +1,26 @@ ++package io.papermc.paper.util; ++ ++public class LogManagerShutdownThread extends Thread { ++ ++ static LogManagerShutdownThread INSTANCE = new LogManagerShutdownThread(); ++ public static final void hook() { ++ if (INSTANCE == null) { ++ throw new IllegalStateException("Cannot re-hook after being unhooked"); ++ } ++ Runtime.getRuntime().addShutdownHook(INSTANCE); ++ } ++ ++ public static final void unhook() { ++ Runtime.getRuntime().removeShutdownHook(INSTANCE); ++ INSTANCE = null; ++ } ++ ++ private LogManagerShutdownThread() { ++ super("Log4j2 Shutdown Thread"); ++ } ++ ++ @Override ++ public void run() { ++ org.apache.logging.log4j.LogManager.shutdown(); ++ } ++} +diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java +index 589a8bf75be6ccc59f1e5dd5d8d9afed41c4772d..b24265573fdef5d9a964bcd76146f34542c420cf 100644 +--- a/src/main/java/net/minecraft/CrashReport.java ++++ b/src/main/java/net/minecraft/CrashReport.java +@@ -237,6 +237,7 @@ public class CrashReport { + } + + public static CrashReport forThrowable(Throwable cause, String title) { ++ if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper + while (cause instanceof CompletionException && cause.getCause() != null) { + cause = cause.getCause(); + } +diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java +index 244a19ecd0234fa1d7a6ecfea20751595688605d..581bd217304e0f9e0b2113c335694805dfb4e2a1 100644 +--- a/src/main/java/net/minecraft/server/Main.java ++++ b/src/main/java/net/minecraft/server/Main.java +@@ -77,6 +77,7 @@ public class Main { + + @DontObfuscate + public static void main(final OptionSet optionset) { // CraftBukkit - replaces main(String[] astring) ++ io.papermc.paper.util.LogManagerShutdownThread.hook(); // Paper + SharedConstants.tryDetectVersion(); + /* CraftBukkit start - Replace everything + OptionParser optionparser = new OptionParser(); +diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java +index c13b4f39c19dd6a62ae39688bde5a4739b391e59..d25d500a2206d4f7e821de3766e8a523df4370e3 100644 +--- a/src/main/java/net/minecraft/server/MinecraftServer.java ++++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -307,7 +307,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>(); + public int autosavePeriod; + // Paper - don't store the vanilla dispatcher +- private boolean forceTicks; ++ public boolean forceTicks; // Paper - Improved watchdog support + // CraftBukkit end + // Spigot start + public static final int TPS = 20; +@@ -319,6 +319,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + public final io.papermc.paper.configuration.PaperConfigurations paperConfigurations; // Paper - add paper configuration files + public boolean isIteratingOverLevels = false; // Paper - Throw exception on world create while being ticked + ++ public volatile Thread shutdownThread; // Paper ++ public volatile boolean abnormalExit = false; // Paper ++ + public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) { + AtomicReference<S> atomicreference = new AtomicReference(); + Thread thread = new ca.spottedleaf.moonrise.common.util.TickThread(() -> { // Paper - rewrite chunk system +@@ -488,6 +491,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + } + */ + // Paper end ++ io.papermc.paper.util.LogManagerShutdownThread.unhook(); // Paper + Runtime.getRuntime().addShutdownHook(new org.bukkit.craftbukkit.util.ServerShutdownThread(this)); + // CraftBukkit end + this.paperConfigurations = services.paperConfigurations(); // Paper - add paper configuration files +@@ -1004,6 +1008,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + // CraftBukkit start + private boolean hasStopped = false; + private boolean hasLoggedStop = false; // Paper - Debugging ++ public volatile boolean hasFullyShutdown = false; // Paper + private final Object stopLock = new Object(); + public final boolean hasStopped() { + synchronized (this.stopLock) { +@@ -1019,6 +1024,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + this.hasStopped = true; + } + if (!hasLoggedStop && isDebugging()) io.papermc.paper.util.TraceUtil.dumpTraceForThread("Server stopped"); // Paper - Debugging ++ // Paper start - kill main thread, and kill it hard ++ shutdownThread = Thread.currentThread(); ++ org.spigotmc.WatchdogThread.doStop(); // Paper ++ // Paper end + // CraftBukkit end + if (this.metricsRecorder.isRecording()) { + this.cancelRecordingMetrics(); +@@ -1092,7 +1101,17 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + } + // Spigot end + ++ // Paper end - Improved watchdog support; move final shutdown items here + ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread.deinit(); // Paper - rewrite chunk system ++ // Paper start - move final shutdown items here ++ Util.shutdownExecutors(); ++ try { ++ net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender ++ } catch (final Exception ignored) { ++ } ++ io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown ++ this.onServerExit(); ++ // Paper end - Improved watchdog support + } + + public String getLocalIp() { +@@ -1187,6 +1206,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + + protected void runServer() { + try { ++ long serverStartTime = Util.getNanos(); // Paper + if (!this.initServer()) { + throw new IllegalStateException("Failed to initialize server"); + } +@@ -1196,6 +1216,17 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + this.status = this.buildServerStatus(); + + // Spigot start ++ // Paper start - Improved Watchdog Support ++ LOGGER.info("Running delayed init tasks"); ++ this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init, ++ // this is going to be the first thing the tick process does anyways, so move done and run it after ++ // everything is init before watchdog tick. ++ // anything at 3+ won't be caught here but also will trip watchdog.... ++ // tasks are default scheduled at -1 + delay, and first tick will tick at 1 ++ final long actualDoneTimeMs = System.currentTimeMillis() - org.bukkit.craftbukkit.Main.BOOT_TIME.toEpochMilli(); // Paper - Add total time ++ LOGGER.info("Done ({})! For help, type \"help\"", String.format(java.util.Locale.ROOT, "%.3fs", actualDoneTimeMs / 1000.00D)); // Paper - Add total time ++ org.spigotmc.WatchdogThread.tick(); ++ // Paper end - Improved Watchdog Support + org.spigotmc.WatchdogThread.hasStarted = true; // Paper + Arrays.fill( this.recentTps, 20 ); + // Paper start - further improve server tick loop +@@ -1290,6 +1321,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + JvmProfiler.INSTANCE.onServerTick(this.smoothedTickTimeMillis); + } + } catch (Throwable throwable) { ++ // Paper start ++ if (throwable instanceof ThreadDeath) { ++ MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable); ++ return; ++ } ++ // Paper end + MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable); + CrashReport crashreport = MinecraftServer.constructOrExtractCrashReport(throwable); + +@@ -1314,15 +1351,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + this.services.profileCache().clearExecutor(); + } + +- org.spigotmc.WatchdogThread.doStop(); // Spigot ++ //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop + // CraftBukkit start - Restore terminal to original settings + try { +- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender ++ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop + } catch (Exception ignored) { + } + // CraftBukkit end +- io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown +- this.onServerExit(); ++ //io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown ++ //this.onServerExit(); // Paper - moved into stop + } + + } +@@ -1445,6 +1482,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + + @Override + public TickTask wrapRunnable(Runnable runnable) { ++ // Paper start - anything that does try to post to main during watchdog crash, run on watchdog ++ if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) { ++ runnable.run(); ++ runnable = () -> {}; ++ } ++ // Paper end + return new TickTask(this.tickCount, runnable); + } + +@@ -2264,7 +2307,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa + this.worldData.setDataConfiguration(worlddataconfiguration); + this.resources.managers.updateRegistryTags(); + this.potionBrewing = this.potionBrewing.reload(this.worldData.enabledFeatures()); // Paper - Custom Potion Mixes +- this.getPlayerList().saveAll(); ++ // Paper start ++ if (Thread.currentThread() != this.serverThread) { ++ return; ++ } ++ // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements // TODO Move this to a different patch ++ for (ServerPlayer player : this.getPlayerList().getPlayers()) { ++ player.getAdvancements().save(); ++ } ++ // Paper end + this.getPlayerList().reloadResources(); + this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary()); + this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager); +diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java +index 7d2896918ff5fed37e5de5a22c37b0c7f32634a8..d43b98bdfcb00603737a309c0fb7793d42289b8c 100644 +--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java ++++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java +@@ -328,7 +328,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface + long j = Util.getNanos() - i; + String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D); + +- DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); ++ DedicatedServer.LOGGER.info("Done preparing level \"{}\" ({})", this.getLevelIdName(), s); // Paper - clarify startup log messages & add total time + if (dedicatedserverproperties.announcePlayerAchievements != null) { + ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this.overworld()); // CraftBukkit - per-world + } +@@ -462,7 +462,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface + // this.remoteStatusListener.stop(); // Paper - don't wait for remote connections + } + +- System.exit(0); // CraftBukkit ++ hasFullyShutdown = true; // Paper ++ System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper + } + + @Override +@@ -842,7 +843,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface + @Override + public void stopServer() { + super.stopServer(); +- Util.shutdownExecutors(); ++ //Util.shutdownExecutors(); // Paper - moved into super + SkullBlockEntity.clear(); + } + +diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java +index ec058eb6a10500831f173dcb47576c32c7516318..0cfaf31273bbec7640eb3a2b16a8d2c816b0ed61 100644 +--- a/src/main/java/net/minecraft/server/players/PlayerList.java ++++ b/src/main/java/net/minecraft/server/players/PlayerList.java +@@ -604,7 +604,7 @@ public abstract class PlayerList { + this.cserver.getPluginManager().callEvent(playerQuitEvent); + entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage()); + +- entityplayer.doTick(); // SPIGOT-924 ++ if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog) + // CraftBukkit end + + // Paper start - Configurable player collision; Remove from collideRule team if needed +diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java +index ca94a1aaccdcc9f28b5f7936b871216a75ab762a..f22fb84c7e7929d6c80c44b13179cf385d8a43f9 100644 +--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java ++++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java +@@ -150,6 +150,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler + try { + task.run(); + } catch (Exception var3) { ++ if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper + LOGGER.error(LogUtils.FATAL_MARKER, "Error executing task on {}", this.name(), var3); + } + } +diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java +index 5c4eaa6bcf20b0fcec14bd5ef76ea6f29a8613a2..e2a0487089eb5a7bdc1433e4c75f69d8e9f9d5f9 100644 +--- a/src/main/java/net/minecraft/world/level/Level.java ++++ b/src/main/java/net/minecraft/world/level/Level.java +@@ -1419,6 +1419,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable, ca.spottedl + try { + tickConsumer.accept(entity); + } catch (Throwable throwable) { ++ if (throwable instanceof ThreadDeath) throw throwable; // Paper + // Paper start - Prevent block entity and entity crashes + final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level().getWorld().getName(), entity.getX(), entity.getY(), entity.getZ()); + MinecraftServer.LOGGER.error(msg, throwable); +diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java +index b1a6dc25b04ab6a43e8d62378e14d88d1c60bbbe..7c11853c5090fbc4fa5b3e73a69acf166158fdec 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java ++++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java +@@ -1044,6 +1044,7 @@ public class LevelChunk extends ChunkAccess implements ca.spottedleaf.moonrise.p + + gameprofilerfiller.pop(); + } catch (Throwable throwable) { ++ if (throwable instanceof ThreadDeath) throw throwable; // Paper + // Paper start - Prevent block entity and entity crashes + final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ()); + net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable); +diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java +index c6e8441e299f477ddb22c1ce2618710763978f1a..e8e93538dfd71de86515d9405f728db1631e949a 100644 +--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java ++++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java +@@ -12,11 +12,27 @@ public class ServerShutdownThread extends Thread { + @Override + public void run() { + try { ++ // Paper start - try to shutdown on main ++ server.safeShutdown(false, false); ++ for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) { ++ Thread.sleep(100); ++ } ++ if (server.hasStopped()) { ++ while (!server.hasFullyShutdown) Thread.sleep(1000); ++ return; ++ } ++ // Looks stalled, close async + org.spigotmc.AsyncCatcher.enabled = false; // Spigot ++ server.forceTicks = true; + this.server.close(); ++ while (!server.hasFullyShutdown) Thread.sleep(1000); ++ } catch (InterruptedException e) { ++ e.printStackTrace(); ++ // Paper end + } finally { ++ org.apache.logging.log4j.LogManager.shutdown(); // Paper + try { +- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender ++ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop + } catch (Exception e) { + } + } +diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java +index 39e56b95aaafbcd8ebe68fdefaace83702e9510d..3ba27955548a26367a87d6b87c3c61beb299dfb9 100644 +--- a/src/main/java/org/spigotmc/RestartCommand.java ++++ b/src/main/java/org/spigotmc/RestartCommand.java +@@ -139,7 +139,7 @@ public class RestartCommand extends Command + // Paper end + + // Paper start - copied from above and modified to return if the hook registered +- private static boolean addShutdownHook(String restartScript) ++ public static boolean addShutdownHook(String restartScript) // Paper + { + String[] split = restartScript.split( " " ); + if ( split.length > 0 && new File( split[0] ).isFile() ) +diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java +index 529df2a41dd93d6e1505053bd04032dbf0cdaa31..c9e17225bc52fe5e7b2dc0908db225a86c6e94d1 100644 +--- a/src/main/java/org/spigotmc/WatchdogThread.java ++++ b/src/main/java/org/spigotmc/WatchdogThread.java +@@ -11,6 +11,7 @@ import org.bukkit.Bukkit; + public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThread // Paper - rewrite chunk system + { + ++ public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper - Improved watchdog support + private static WatchdogThread instance; + private long timeoutTime; + private boolean restart; +@@ -39,6 +40,7 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre + { + if ( WatchdogThread.instance == null ) + { ++ if (timeoutTime <= 0) timeoutTime = 300; // Paper + WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart ); + WatchdogThread.instance.start(); + } else +@@ -70,12 +72,13 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre + // Paper start + Logger log = Bukkit.getServer().getLogger(); + long currentTime = WatchdogThread.monotonicMillis(); +- if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable ++ MinecraftServer server = MinecraftServer.getServer(); ++ if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable + { +- boolean isLongTimeout = currentTime > lastTick + timeoutTime; ++ boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000); + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; +- if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... ++ if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + lastEarlyWarning = currentTime; + if (isLongTimeout) { + // Paper end +@@ -136,9 +139,24 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre + + if ( isLongTimeout ) + { +- if ( this.restart && !MinecraftServer.getServer().hasStopped() ) ++ if ( !server.hasStopped() ) + { +- RestartCommand.restart(); ++ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us ++ server.forceTicks = true; ++ if (restart) { ++ RestartCommand.addShutdownHook( SpigotConfig.restartScript ); ++ } ++ // try one last chance to safe shutdown on main incase it 'comes back' ++ server.abnormalExit = true; ++ server.safeShutdown(false, restart); ++ try { ++ Thread.sleep(1000); ++ } catch (InterruptedException e) { ++ e.printStackTrace(); ++ } ++ if (!server.hasStopped()) { ++ server.close(); ++ } + } + break; + } // Paper end +diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml +index 637d64da9938e51a97338b9253b43889585c67bb..d2a75850af9c6ad2aca66a5f994f1b587d73eac4 100644 +--- a/src/main/resources/log4j2.xml ++++ b/src/main/resources/log4j2.xml +@@ -1,5 +1,5 @@ + <?xml version="1.0" encoding="UTF-8"?> +-<Configuration status="WARN"> ++<Configuration status="WARN" shutdownHook="disable"> + <Appenders> + <Queue name="ServerGuiConsole"> + <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg{nolookups}%n" /> |