Rewrite chunk system (#8177)

Patch documentation to come

Issues with the old system that are fixed now:
- World generation does not scale with cpu cores effectively.
- Relies on the main thread for scheduling and maintaining chunk state, dropping chunk load/generate rates at lower tps.
- Unreliable prioritisation of chunk gen/load calls that block the main thread.
- Shutdown logic is utterly unreliable, as it has to wait for all chunks to unload - is it guaranteed that the chunk system is in a state on shutdown that it can reliably do this? Watchdog shutdown also typically failed due to thread checks, which is now resolved.
- Saving of data is not unified (i.e can save chunk data without saving entity data, poses problems for desync if shutdown is really abnormal.
- Entities are not loaded with chunks. This caused quite a bit of headache for Chunk#getEntities API, but now the new chunk system loads entities with chunks so that they are ready whenever the chunk loads in. Effectively brings the behavior back to 1.16 era, but still storing entities in their own separate regionfiles.

The above list is not complete. The patch documentation will complete it.

New chunk system hard relies on starlight and dataconverter, and most importantly the new concurrent utilities in ConcurrentUtil.

Some of the old async chunk i/o interface (i.e the old file io thread reroutes _some_ calls to the new file io thread) is kept for plugin compat reasons. It will be removed in the next major version of minecraft.

The old legacy chunk system patches have been moved to the removed folder in case we need them again.

1 files changed, 81 insertions, 0 deletions

diff --git a/patches/server/0704-Add-Raw-Byte-Entity-Serialization.patch b/patches/server/0704-Add-Raw-Byte-Entity-Serialization.patch
new file mode 100644
index 0000000000..d7d990702b
--- /dev/null
+++ b/patches/server/0704-Add-Raw-Byte-Entity-Serialization.patch
@@ -0,0 +1,81 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Mariell Hoversholm <[email protected]>
+Date: Sun, 24 Oct 2021 16:20:31 -0400
+Subject: [PATCH] Add Raw Byte Entity Serialization
+
+
+diff --git a/src/main/java/net/minecraft/world/entity/Entity.java b/src/main/java/net/minecraft/world/entity/Entity.java
+index 07ba08de35bac64eeac2657e15dcc60d1dce3cca..bdfa91daf69e71e8c2e1e820f14360c567bdeb87 100644
+--- a/src/main/java/net/minecraft/world/entity/Entity.java
++++ b/src/main/java/net/minecraft/world/entity/Entity.java
+@@ -1986,6 +1986,15 @@ public abstract class Entity implements Nameable, EntityAccess, CommandSource {
+         }
+     }
+ 
++    // Paper start - Entity serialization api
++    public boolean serializeEntity(CompoundTag compound) {
++        List<Entity> pass = new java.util.ArrayList<>(this.getPassengers());
++        this.passengers = ImmutableList.of();
++        boolean result = save(compound);
++        this.passengers = ImmutableList.copyOf(pass);
++        return result;
++    }
++    // Paper end
+     public boolean save(CompoundTag nbt) {
+         return this.isPassenger() ? false : this.saveAsPassenger(nbt);
+     }
+diff --git a/src/main/java/org/bukkit/craftbukkit/entity/CraftEntity.java b/src/main/java/org/bukkit/craftbukkit/entity/CraftEntity.java
+index b30fb13db5524dcd05a155b014b93089af05c994..a1ef4ba683b1721359fb162b5f97ceefd7207184 100644
+--- a/src/main/java/org/bukkit/craftbukkit/entity/CraftEntity.java
++++ b/src/main/java/org/bukkit/craftbukkit/entity/CraftEntity.java
+@@ -1312,5 +1312,15 @@ public abstract class CraftEntity implements org.bukkit.entity.Entity {
+         }
+         return set;
+     }
++
++    @Override
++    public boolean spawnAt(Location location, org.bukkit.event.entity.CreatureSpawnEvent.SpawnReason reason) {
++        Preconditions.checkNotNull(location, "location cannot be null");
++        Preconditions.checkNotNull(reason, "reason cannot be null");
++        entity.level = ((CraftWorld) location.getWorld()).getHandle();
++        entity.setPos(location.getX(), location.getY(), location.getZ());
++        entity.setRot(location.getYaw(), location.getPitch());
++        return !entity.valid && entity.level.addFreshEntity(entity, reason);
++    }
+     // Paper end
+ }
+diff --git a/src/main/java/org/bukkit/craftbukkit/util/CraftMagicNumbers.java b/src/main/java/org/bukkit/craftbukkit/util/CraftMagicNumbers.java
+index 0a3d447f87698dc786d6cab6ded27eb1b5780204..d3b37adfe6d66e82db18d94f143af3aba4543f79 100644
+--- a/src/main/java/org/bukkit/craftbukkit/util/CraftMagicNumbers.java
++++ b/src/main/java/org/bukkit/craftbukkit/util/CraftMagicNumbers.java
+@@ -457,6 +457,30 @@ public final class CraftMagicNumbers implements UnsafeValues {
+         return CraftItemStack.asCraftMirror(net.minecraft.world.item.ItemStack.of((CompoundTag) converted.getValue()));
+     }
+ 
++    @Override
++    public byte[] serializeEntity(org.bukkit.entity.Entity entity) {
++        Preconditions.checkNotNull(entity, "null cannot be serialized");
++        Preconditions.checkArgument(entity instanceof org.bukkit.craftbukkit.entity.CraftEntity, "only CraftEntities can be serialized");
++
++        CompoundTag compound = new CompoundTag();
++        ((org.bukkit.craftbukkit.entity.CraftEntity) entity).getHandle().serializeEntity(compound);
++        return serializeNbtToBytes(compound);
++    }
++
++    @Override
++    public org.bukkit.entity.Entity deserializeEntity(byte[] data, org.bukkit.World world, boolean preserveUUID) {
++        Preconditions.checkNotNull(data, "null cannot be deserialized");
++        Preconditions.checkArgument(data.length > 0, "cannot deserialize nothing");
++
++        CompoundTag compound = deserializeNbtFromBytes(data);
++        int dataVersion = compound.getInt("DataVersion");
++        Dynamic<Tag> converted = DataFixers.getDataFixer().update(References.ENTITY_TREE, new Dynamic<>(NbtOps.INSTANCE, compound), dataVersion, getDataVersion());
++        compound = (CompoundTag) converted.getValue();
++        if (!preserveUUID) compound.remove("UUID"); // Generate a new UUID so we don't have to worry about deserializing the same entity twice
++        return net.minecraft.world.entity.EntityType.create(compound, ((org.bukkit.craftbukkit.CraftWorld) world).getHandle())
++            .orElseThrow(() -> new IllegalArgumentException("An ID was not found for the data. Did you downgrade?")).getBukkitEntity();
++    }
++
+     private byte[] serializeNbtToBytes(CompoundTag compound) {
+         compound.putInt("DataVersion", getDataVersion());
+         java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream();