Skip to content

Commit f2355fa

Browse files
feat: Linear region file format support (extended from #415) (#418)
* Linear support Fix region rendering & bitwise operators Close streams * Make mca region-file types extensible * Fix file-name verification not working --------- Co-authored-by: Sofiane H. Djerbi <46628754+kugge@users.noreply.github.com>
1 parent bc9e688 commit f2355fa

File tree

6 files changed

+297
-19
lines changed

6 files changed

+297
-19
lines changed

BlueMapCore/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ repositories {
5656

5757
@Suppress("GradlePackageUpdate")
5858
dependencies {
59+
implementation("com.github.luben:zstd-jni:1.5.4-1")
5960
api ("com.github.ben-manes.caffeine:caffeine:2.8.5")
6061
api ("org.apache.commons:commons-lang3:3.6")
6162
api ("commons-io:commons-io:2.5")

BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/MCAWorld.java

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import de.bluecolored.bluemap.api.debug.DebugDump;
3232
import de.bluecolored.bluemap.core.BlueMap;
3333
import de.bluecolored.bluemap.core.logger.Logger;
34+
import de.bluecolored.bluemap.core.mca.region.RegionType;
3435
import de.bluecolored.bluemap.core.util.Vector2iCache;
3536
import de.bluecolored.bluemap.core.world.*;
3637
import net.querz.nbt.CompoundTag;
@@ -131,7 +132,7 @@ public Collection<Vector2i> listRegions() {
131132
List<Vector2i> regions = new ArrayList<>(regionFiles.length);
132133

133134
for (File file : regionFiles) {
134-
if (!file.getName().endsWith(".mca")) continue;
135+
if (RegionType.forFileName(file.getName()) == null) continue;
135136
if (file.length() <= 0) continue;
136137

137138
try {
@@ -213,17 +214,12 @@ public boolean isIgnoreMissingLightData() {
213214
return ignoreMissingLightData;
214215
}
215216

216-
private File getMCAFile(int regionX, int regionZ) {
217-
return getRegionFolder().resolve("r." + regionX + "." + regionZ + ".mca").toFile();
218-
}
219-
220217
private Region loadRegion(Vector2i regionPos) {
221218
return loadRegion(regionPos.getX(), regionPos.getY());
222219
}
223220

224221
Region loadRegion(int x, int z) {
225-
File regionPath = getMCAFile(x, z);
226-
return new MCARegion(this, regionPath);
222+
return RegionType.loadRegion(this, getRegionFolder(), x, z);
227223
}
228224

229225
private Chunk loadChunk(Vector2i chunkPos) {
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* This file is part of BlueMap, licensed under the MIT License (MIT).
3+
*
4+
* Copyright (c) Blue (Lukas Rieger) <https://bluecolored.de>
5+
* Copyright (c) contributors
6+
*
7+
* Permission is hereby granted, free of charge, to any person obtaining a copy
8+
* of this software and associated documentation files (the "Software"), to deal
9+
* in the Software without restriction, including without limitation the rights
10+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
* copies of the Software, and to permit persons to whom the Software is
12+
* furnished to do so, subject to the following conditions:
13+
*
14+
* The above copyright notice and this permission notice shall be included in
15+
* all copies or substantial portions of the Software.
16+
*
17+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
* THE SOFTWARE.
24+
*/
25+
package de.bluecolored.bluemap.core.mca.region;
26+
27+
import com.flowpowered.math.vector.Vector2i;
28+
import com.github.luben.zstd.ZstdInputStream;
29+
import de.bluecolored.bluemap.core.logger.Logger;
30+
import de.bluecolored.bluemap.core.mca.MCAChunk;
31+
import de.bluecolored.bluemap.core.mca.MCAWorld;
32+
import de.bluecolored.bluemap.core.world.Chunk;
33+
import de.bluecolored.bluemap.core.world.EmptyChunk;
34+
import de.bluecolored.bluemap.core.world.Region;
35+
import net.querz.nbt.CompoundTag;
36+
import net.querz.nbt.Tag;
37+
38+
import java.io.*;
39+
import java.nio.file.Files;
40+
import java.nio.file.Path;
41+
import java.util.ArrayList;
42+
import java.util.Collection;
43+
import java.util.Collections;
44+
import java.util.List;
45+
46+
public class LinearRegion implements Region {
47+
48+
public static final String FILE_SUFFIX = ".linear";
49+
50+
private static final long SUPERBLOCK = -4323716122432332390L;
51+
private static final byte VERSION = 1;
52+
private static final int HEADER_SIZE = 32;
53+
private static final int FOOTER_SIZE = 8;
54+
55+
private final MCAWorld world;
56+
private final Path regionFile;
57+
private final Vector2i regionPos;
58+
59+
60+
public LinearRegion(MCAWorld world, Path regionFile) throws IllegalArgumentException {
61+
this.world = world;
62+
this.regionFile = regionFile;
63+
64+
String[] filenameParts = regionFile.getFileName().toString().split("\\.");
65+
int rX = Integer.parseInt(filenameParts[1]);
66+
int rZ = Integer.parseInt(filenameParts[2]);
67+
68+
this.regionPos = new Vector2i(rX, rZ);
69+
}
70+
71+
@Override
72+
public Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) throws IOException {
73+
if (Files.notExists(regionFile)) return EmptyChunk.INSTANCE;
74+
75+
long fileLength = Files.size(regionFile);
76+
if (fileLength == 0) return EmptyChunk.INSTANCE;
77+
78+
try (InputStream inputStream = Files.newInputStream(regionFile);
79+
DataInputStream rawDataStream = new DataInputStream(inputStream)) {
80+
81+
long superBlock = rawDataStream.readLong();
82+
if (superBlock != SUPERBLOCK)
83+
throw new RuntimeException("Superblock invalid: " + superBlock + " file " + regionFile);
84+
85+
byte version = rawDataStream.readByte();
86+
if (version != VERSION)
87+
throw new RuntimeException("Version invalid: " + version + " file " + regionFile);
88+
89+
rawDataStream.skipBytes(11); // newestTimestamp + compression level + chunk count
90+
91+
int dataCount = rawDataStream.readInt();
92+
if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE)
93+
throw new RuntimeException("File length invalid " + this.regionFile + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE));
94+
95+
rawDataStream.skipBytes(8); // Data Hash
96+
97+
byte[] rawCompressed = new byte[dataCount];
98+
rawDataStream.readFully(rawCompressed, 0, dataCount);
99+
100+
superBlock = rawDataStream.readLong();
101+
if (superBlock != SUPERBLOCK)
102+
throw new RuntimeException("Footer superblock invalid " + this.regionFile);
103+
104+
try (DataInputStream dis = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) {
105+
int x = chunkX - (regionPos.getX() << 5);
106+
int z = chunkZ - (regionPos.getY() << 5);
107+
int pos = (z << 5) + x;
108+
int skip = 0;
109+
110+
for (int i = 0; i < pos; i++) {
111+
skip += dis.readInt(); // size of the chunk (bytes) to skip
112+
dis.skipBytes(4); // skip 0 (will be timestamps)
113+
}
114+
115+
int size = dis.readInt();
116+
if (size <= 0) return EmptyChunk.INSTANCE;
117+
118+
dis.skipBytes(((1024 - pos - 1) << 3) + 4); // Skip current chunk 0 and unneeded other chunks zero/size
119+
dis.skipBytes(skip); // Skip unneeded chunks data
120+
121+
Tag<?> tag = Tag.deserialize(dis, Tag.DEFAULT_MAX_DEPTH);
122+
if (tag instanceof CompoundTag) {
123+
MCAChunk chunk = MCAChunk.create(world, (CompoundTag) tag);
124+
if (!chunk.isGenerated()) return EmptyChunk.INSTANCE;
125+
return chunk;
126+
} else {
127+
throw new IOException("Invalid data tag: " + (tag == null ? "null" : tag.getClass().getName()));
128+
}
129+
}
130+
} catch (RuntimeException e) {
131+
throw new IOException(e);
132+
}
133+
}
134+
135+
@Override
136+
public Collection<Vector2i> listChunks(long modifiedSince) {
137+
if (Files.notExists(regionFile)) return Collections.emptyList();
138+
139+
try {
140+
long fileLength = Files.size(regionFile);
141+
if (fileLength == 0) return Collections.emptyList();
142+
} catch (IOException ex) {
143+
Logger.global.logWarning("Failed to read file-size for file: " + regionFile);
144+
return Collections.emptyList();
145+
}
146+
147+
List<Vector2i> chunks = new ArrayList<>(1024); //1024 = 32 x 32 chunks per region-file
148+
try (InputStream inputStream = Files.newInputStream(regionFile);
149+
DataInputStream rawDataStream = new DataInputStream(inputStream)) {
150+
151+
long superBlock = rawDataStream.readLong();
152+
if (superBlock != SUPERBLOCK) throw new RuntimeException("Superblock invalid: " + superBlock + " file " + regionFile);
153+
154+
byte version = rawDataStream.readByte();
155+
if (version != VERSION) throw new RuntimeException("Version invalid: " + version + " file " + regionFile);
156+
157+
long newestTimestamp = rawDataStream.readLong();
158+
159+
// If whole region is the same - skip.
160+
if (newestTimestamp < modifiedSince / 1000) return Collections.emptyList();
161+
162+
// Linear files store whole region timestamp, not chunk timestamp. We need to render the while region file.
163+
// TODO: Add per-chunk timestamps when .linear add support for per-chunk timestamps (soon)
164+
for(int i = 0 ; i < 1024; i++)
165+
chunks.add(new Vector2i((regionPos.getX() << 5) + (i & 31), (regionPos.getY() << 5) + (i >> 5)));
166+
return chunks;
167+
} catch (RuntimeException | IOException ex) {
168+
Logger.global.logWarning("Failed to read .linear file: " + regionFile + " (" + ex + ")");
169+
}
170+
return chunks;
171+
}
172+
173+
@Override
174+
public Path getRegionFile() {
175+
return regionFile;
176+
}
177+
178+
public static String getRegionFileName(int regionX, int regionZ) {
179+
return "r." + regionX + "." + regionZ + FILE_SUFFIX;
180+
}
181+
182+
}

BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/MCARegion.java renamed to BlueMapCore/src/main/java/de/bluecolored/bluemap/core/mca/region/MCARegion.java

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2323
* THE SOFTWARE.
2424
*/
25-
package de.bluecolored.bluemap.core.mca;
25+
package de.bluecolored.bluemap.core.mca.region;
2626

2727
import com.flowpowered.math.vector.Vector2i;
2828
import de.bluecolored.bluemap.core.logger.Logger;
29+
import de.bluecolored.bluemap.core.mca.MCAChunk;
30+
import de.bluecolored.bluemap.core.mca.MCAWorld;
2931
import de.bluecolored.bluemap.core.world.Chunk;
3032
import de.bluecolored.bluemap.core.world.EmptyChunk;
3133
import de.bluecolored.bluemap.core.world.Region;
@@ -34,22 +36,26 @@
3436
import net.querz.nbt.mca.CompressionType;
3537

3638
import java.io.*;
39+
import java.nio.file.Files;
40+
import java.nio.file.Path;
3741
import java.util.ArrayList;
3842
import java.util.Collection;
3943
import java.util.Collections;
4044
import java.util.List;
4145

4246
public class MCARegion implements Region {
4347

48+
public static final String FILE_SUFFIX = ".mca";
49+
4450
private final MCAWorld world;
45-
private final File regionFile;
51+
private final Path regionFile;
4652
private final Vector2i regionPos;
4753

48-
public MCARegion(MCAWorld world, File regionFile) throws IllegalArgumentException {
54+
public MCARegion(MCAWorld world, Path regionFile) throws IllegalArgumentException {
4955
this.world = world;
5056
this.regionFile = regionFile;
5157

52-
String[] filenameParts = regionFile.getName().split("\\.");
58+
String[] filenameParts = regionFile.getFileName().toString().split("\\.");
5359
int rX = Integer.parseInt(filenameParts[1]);
5460
int rZ = Integer.parseInt(filenameParts[2]);
5561

@@ -58,9 +64,12 @@ public MCARegion(MCAWorld world, File regionFile) throws IllegalArgumentExceptio
5864

5965
@Override
6066
public Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) throws IOException {
61-
if (!regionFile.exists() || regionFile.length() == 0) return EmptyChunk.INSTANCE;
67+
if (Files.notExists(regionFile)) return EmptyChunk.INSTANCE;
68+
69+
long fileLength = Files.size(regionFile);
70+
if (fileLength == 0) return EmptyChunk.INSTANCE;
6271

63-
try (RandomAccessFile raf = new RandomAccessFile(regionFile, "r")) {
72+
try (RandomAccessFile raf = new RandomAccessFile(regionFile.toFile(), "r")) {
6473

6574
int xzChunk = Math.floorMod(chunkZ, 32) * 32 + Math.floorMod(chunkX, 32);
6675

@@ -100,11 +109,19 @@ public Chunk loadChunk(int chunkX, int chunkZ, boolean ignoreMissingLightData) t
100109

101110
@Override
102111
public Collection<Vector2i> listChunks(long modifiedSince) {
103-
if (!regionFile.exists() || regionFile.length() == 0) return Collections.emptyList();
112+
if (Files.notExists(regionFile)) return Collections.emptyList();
113+
114+
try {
115+
long fileLength = Files.size(regionFile);
116+
if (fileLength == 0) return Collections.emptyList();
117+
} catch (IOException ex) {
118+
Logger.global.logWarning("Failed to read file-size for file: " + regionFile);
119+
return Collections.emptyList();
120+
}
104121

105122
List<Vector2i> chunks = new ArrayList<>(1024); //1024 = 32 x 32 chunks per region-file
106123

107-
try (RandomAccessFile raf = new RandomAccessFile(regionFile, "r")) {
124+
try (RandomAccessFile raf = new RandomAccessFile(regionFile.toFile(), "r")) {
108125
for (int x = 0; x < 32; x++) {
109126
for (int z = 0; z < 32; z++) {
110127
Vector2i chunk = new Vector2i(regionPos.getX() * 32 + x, regionPos.getY() * 32 + z);
@@ -127,15 +144,19 @@ public Collection<Vector2i> listChunks(long modifiedSince) {
127144
}
128145
}
129146
} catch (RuntimeException | IOException ex) {
130-
Logger.global.logWarning("Failed to read .mca file: " + regionFile.getAbsolutePath() + " (" + ex + ")");
147+
Logger.global.logWarning("Failed to read .mca file: " + regionFile + " (" + ex + ")");
131148
}
132149

133150
return chunks;
134151
}
135152

136153
@Override
137-
public File getRegionFile() {
154+
public Path getRegionFile() {
138155
return regionFile;
139156
}
140157

158+
public static String getRegionFileName(int regionX, int regionZ) {
159+
return "r." + regionX + "." + regionZ + FILE_SUFFIX;
160+
}
161+
141162
}

0 commit comments

Comments
 (0)