Skip to content

Commit a933b6d

Browse files
bchapuisbchapuis
committed
Generate different queries depending on the postgresql version
1 parent 8d858f1 commit a933b6d

File tree

6 files changed

+170
-21
lines changed

6 files changed

+170
-21
lines changed

‎baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ public Integer call() throws Exception {
8787
var objectMapper = objectMapper();
8888
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
8989
var datasource = PostgresUtils.createDataSourceFromObject(tileset.getDatabase());
90+
var postgresVersion = PostgresUtils.getPostgresVersion(datasource);
9091

9192
var tilesetSupplier = (Supplier<Tileset>) () -> {
9293
try {
@@ -99,7 +100,7 @@ public Integer call() throws Exception {
99100

100101
var tileStoreSupplier = (Supplier<TileStore<ByteBuffer>>) () -> {
101102
var tileJSON = tilesetSupplier.get();
102-
return new PostgresTileStore(datasource, tileJSON);
103+
return new PostgresTileStore(datasource, tileJSON, postgresVersion);
103104
};
104105

105106
var styleSupplier = (Supplier<Style>) () -> {

‎baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,10 @@ public Integer call() throws Exception {
9393
var caffeineSpec = CaffeineSpec.parse(cache);
9494
var tileset = objectMapper.readValue(configReader.read(tilesetPath), Tileset.class);
9595
var datasource = PostgresUtils.createDataSourceFromObject(tileset.getDatabase());
96+
var postgresVersion = PostgresUtils.getPostgresVersion(datasource);
9697

9798
try (
98-
var tileStore = new PostgresTileStore(datasource, tileset);
99+
var tileStore = new PostgresTileStore(datasource, tileset, postgresVersion);
99100
var tileCache = new VectorTileCache(tileStore, caffeineSpec)) {
100101

101102
var tileStoreSupplier = (Supplier<TileStore<ByteBuffer>>) () -> tileCache;

‎baremaps-core/src/main/java/org/apache/baremaps/tasks/ExportVectorTiles.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@
2222
import com.fasterxml.jackson.core.JsonProcessingException;
2323
import com.fasterxml.jackson.databind.ObjectMapper;
2424
import java.io.IOException;
25+
import java.nio.ByteBuffer;
2526
import java.nio.file.Files;
2627
import java.nio.file.Path;
2728
import java.nio.file.StandardOpenOption;
29+
import java.sql.SQLException;
2830
import java.util.*;
2931
import java.util.stream.Collectors;
3032
import javax.sql.DataSource;
@@ -34,6 +36,7 @@
3436
import org.apache.baremaps.maplibre.tileset.TilesetQuery;
3537
import org.apache.baremaps.openstreetmap.stream.ProgressLogger;
3638
import org.apache.baremaps.openstreetmap.stream.StreamUtils;
39+
import org.apache.baremaps.postgres.utils.PostgresUtils;
3740
import org.apache.baremaps.tilestore.*;
3841
import org.apache.baremaps.tilestore.file.FileTileStore;
3942
import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
@@ -146,7 +149,7 @@ public void execute(WorkflowContext context) throws Exception {
146149

147150
var bufferedTileEntryStream = StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> {
148151
try {
149-
return new TileEntry(tile, sourceTileStore.read(tile));
152+
return new TileEntry<>(tile, sourceTileStore.read(tile));
150153
} catch (TileStoreException e) {
151154
throw new WorkflowException(e);
152155
}
@@ -166,11 +169,14 @@ public void execute(WorkflowContext context) throws Exception {
166169
}
167170
}
168171

169-
private TileStore sourceTileStore(Tileset tileset, DataSource datasource) {
170-
return new PostgresTileStore(datasource, tileset);
172+
private TileStore<ByteBuffer> sourceTileStore(Tileset tileset, DataSource datasource)
173+
throws SQLException {
174+
var postgresVersion = PostgresUtils.getPostgresVersion(datasource);
175+
return new PostgresTileStore(datasource, tileset, postgresVersion);
171176
}
172177

173-
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
178+
private TileStore<ByteBuffer> targetTileStore(Tileset source)
179+
throws TileStoreException, IOException {
174180
switch (format) {
175181
case FILE:
176182
return new FileTileStore(repository.resolve("tiles"));

‎baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java‎

Lines changed: 120 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,18 @@ public class PostgresTileStore implements TileStore<ByteBuffer> {
4646

4747
private final Tileset tileset;
4848

49+
private final int postgresVersion;
50+
4951
/**
5052
* Constructs a {@code PostgresTileStore}.
5153
*
5254
* @param datasource the datasource
5355
* @param tileset the tileset
5456
*/
55-
public PostgresTileStore(DataSource datasource, Tileset tileset) {
57+
public PostgresTileStore(DataSource datasource, Tileset tileset, int postgresVersion) {
5658
this.datasource = datasource;
5759
this.tileset = tileset;
60+
this.postgresVersion = postgresVersion;
5861
}
5962

6063
/**
@@ -76,7 +79,7 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
7679
var start = System.currentTimeMillis();
7780

7881
// Prepare and cache the query
79-
var query = cache.computeIfAbsent(tileCoord.z(), z -> prepareQuery(tileset, z));
82+
var query = cache.computeIfAbsent(tileCoord.z(), this::prepareQuery);
8083

8184
// Fetch and compress the tile data
8285
try (var connection = datasource.getConnection();
@@ -119,14 +122,121 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
119122
}
120123

121124
/**
122-
* Prepare the sql query for a given tileset and zoom level.
125+
* Prepare the sql query for a given zoom level.
126+
*
127+
* @param zoom the zoom level
128+
* @return the prepared query
129+
*/
130+
protected Query prepareQuery(int zoom) {
131+
if (postgresVersion >= 16) {
132+
return prepareNewQuery(zoom);
133+
} else {
134+
return prepareLegacyQuery(zoom);
135+
}
136+
}
137+
138+
/**
139+
* Prepare the sql query for a given zoom level that uses the new version of postgresql (>= 16).
140+
*
141+
* @param zoom the zoom level
142+
* @return the prepared query
143+
*/
144+
@SuppressWarnings("squid:S3776")
145+
private Query prepareNewQuery(int zoom) {
146+
// Initialize a builder for the tile sql
147+
var tileSql = new StringBuilder();
148+
tileSql.append("SELECT ");
149+
150+
// Iterate over the layers and keep track of the number of layers and parameters included in the
151+
// final sql
152+
var layers = tileset.getVectorLayers();
153+
var layerCount = 0;
154+
var paramCount = 0;
155+
for (var layer : layers) {
156+
157+
// Initialize a builder for the layer sql
158+
var layerSql = new StringBuilder();
159+
var layerHead = String.format("(SELECT ST_AsMVT(mvtGeom.*, '%s') FROM (", layer.getId());
160+
layerSql.append(layerHead);
161+
162+
// Iterate over the queries and keep track of the number of queries included in the final
163+
// sql
164+
var queries = layer.getQueries();
165+
var queryCount = 0;
166+
for (var query : queries) {
167+
168+
// Only include the sql if the zoom level is in the range
169+
if (query.getMinzoom() <= zoom && zoom < query.getMaxzoom()) {
170+
171+
// Add a union between queries
172+
if (queryCount > 0) {
173+
layerSql.append("UNION ALL ");
174+
}
175+
176+
// Add the sql to the layer sql
177+
var querySql = query.getSql().trim()
178+
.replaceAll("\\s+", " ")
179+
.replace(";", "")
180+
.replace("?", "??")
181+
.replace("$zoom", String.valueOf(zoom));
182+
var querySqlWithParams = String.format(
183+
"""
184+
SELECT
185+
mvtData.id AS id,
186+
mvtData.tags - 'id' AS tags,
187+
ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom
188+
FROM (%s) AS mvtData
189+
WHERE mvtData.geom IS NOT NULL
190+
AND mvtData.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))
191+
""",
192+
querySql);
193+
layerSql.append(querySqlWithParams);
194+
195+
// Increase the parameter count (e.g. ?) and sql count
196+
paramCount += 6;
197+
queryCount++;
198+
}
199+
}
200+
201+
// Add the tail of the layer sql
202+
var layerQueryTail = ") AS mvtGeom)";
203+
layerSql.append(layerQueryTail);
204+
205+
// Only include the layer sql if queries were included for this layer
206+
if (queryCount > 0) {
207+
208+
// Add the concatenation between layer queries
209+
if (layerCount > 0) {
210+
tileSql.append(" || ");
211+
}
212+
213+
// Add the layer sql to the mvt sql
214+
tileSql.append(layerSql);
215+
216+
// Increase the layer count
217+
layerCount++;
218+
}
219+
}
220+
221+
// Add the tail of the tile sql
222+
var tileQueryTail = " AS mvtTile";
223+
tileSql.append(tileQueryTail);
224+
225+
// Format the sql query
226+
var sql = tileSql.toString().replace("\n", " ");
227+
228+
return new Query(sql, paramCount);
229+
}
230+
231+
/**
232+
* Prepare the sql query for a given zoom level that uses the legacy versions of postgresql (<
233+
* 16).
123234
*
124-
* @param tileset the tileset
125235
* @param zoom the zoom level
126-
* @return
236+
* @return the prepared query
127237
*/
128238
@SuppressWarnings("squid:S3776")
129-
protected static Query prepareQuery(Tileset tileset, int zoom) {
239+
private Query prepareLegacyQuery(int zoom) {
130240
// Initialize a builder for the tile sql
131241
var tileSql = new StringBuilder();
132242
tileSql.append("SELECT ");
@@ -178,10 +288,10 @@ protected static Query prepareQuery(Tileset tileset, int zoom) {
178288
var querySqlWithParams = String.format(
179289
"""
180290
SELECT
181-
tile.id AS id,
182-
tile.tags - 'id' AS tags,
183-
ST_AsMVTGeom(tile.geom, ST_TileEnvelope(?, ?, ?)) AS geom
184-
FROM (%s) as tile
291+
mvtData.id AS id,
292+
mvtData.tags - 'id' AS tags,
293+
ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom
294+
FROM (%s) as mvtData
185295
""",
186296
querySql);
187297
layerSql.append(querySqlWithParams);

‎baremaps-core/src/test/java/org/apache/baremaps/tilestore/postgres/PostgresTileStoreTest.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,41 @@
2424
import org.apache.baremaps.maplibre.tileset.Tileset;
2525
import org.apache.baremaps.maplibre.tileset.TilesetLayer;
2626
import org.apache.baremaps.maplibre.tileset.TilesetQuery;
27+
import org.junit.jupiter.api.BeforeEach;
2728
import org.junit.jupiter.api.Test;
2829

2930
class PostgresTileStoreTest {
3031

31-
@Test
32-
void prepareQuery() {
33-
var tileset = new Tileset();
32+
private Tileset tileset;
33+
34+
@BeforeEach
35+
void prepare() {
36+
tileset = new Tileset();
3437
tileset.setMinzoom(0);
3538
tileset.setMaxzoom(20);
3639
tileset.setVectorLayers(List.of(
3740
new TilesetLayer("a", Map.of(), "", 0, 20,
3841
List.of(new TilesetQuery(0, 20, "SELECT id, tags, geom FROM table"))),
3942
new TilesetLayer("b", Map.of(), "", 0, 20,
4043
List.of(new TilesetQuery(0, 20, "SELECT id, tags, geom FROM table")))));
41-
var query = PostgresTileStore.prepareQuery(tileset, 10);
44+
45+
}
46+
47+
@Test
48+
void prepareNewQuery() {
49+
var postgresTileStore = new PostgresTileStore(null, tileset, 16);
50+
var query = postgresTileStore.prepareQuery(10);
51+
assertEquals(
52+
"SELECT (SELECT ST_AsMVT(mvtGeom.*, 'a') FROM (SELECT mvtData.id AS id, mvtData.tags - 'id' AS tags, ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table) AS mvtData WHERE mvtData.geom IS NOT NULL AND mvtData.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096)) ) AS mvtGeom) || (SELECT ST_AsMVT(mvtGeom.*, 'b') FROM (SELECT mvtData.id AS id, mvtData.tags - 'id' AS tags, ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table) AS mvtData WHERE mvtData.geom IS NOT NULL AND mvtData.geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096)) ) AS mvtGeom) AS mvtTile",
53+
query.sql());
54+
}
55+
56+
@Test
57+
void prepareLegacyQuery() {
58+
var postgresTileStore = new PostgresTileStore(null, tileset, 15);
59+
var query = postgresTileStore.prepareQuery(10);
4260
assertEquals(
43-
"SELECT (SELECT ST_AsMVT(mvtGeom.*, 'a') FROM (SELECT tile.id AS id, tile.tags - 'id' AS tags, ST_AsMVTGeom(tile.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table WHERE geom IS NOT NULL AND geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) as tile ) AS mvtGeom) || (SELECT ST_AsMVT(mvtGeom.*, 'b') FROM (SELECT tile.id AS id, tile.tags - 'id' AS tags, ST_AsMVTGeom(tile.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table WHERE geom IS NOT NULL AND geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) as tile ) AS mvtGeom) AS mvtTile",
61+
"SELECT (SELECT ST_AsMVT(mvtGeom.*, 'a') FROM (SELECT mvtData.id AS id, mvtData.tags - 'id' AS tags, ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table WHERE geom IS NOT NULL AND geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) as mvtData ) AS mvtGeom) || (SELECT ST_AsMVT(mvtGeom.*, 'b') FROM (SELECT mvtData.id AS id, mvtData.tags - 'id' AS tags, ST_AsMVTGeom(mvtData.geom, ST_TileEnvelope(?, ?, ?)) AS geom FROM (SELECT id, tags, geom FROM table WHERE geom IS NOT NULL AND geom && ST_TileEnvelope(?, ?, ?, margin => (64.0/4096))) as mvtData ) AS mvtGeom) AS mvtTile",
4462
query.sql());
4563
}
4664
}

‎baremaps-postgres/src/main/java/org/apache/baremaps/postgres/utils/PostgresUtils.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,4 +178,17 @@ public static void executeResource(Connection connection, String resource)
178178
statement.execute(queries);
179179
}
180180
}
181+
182+
/**
183+
* Gets the version of the Postgres database.
184+
*
185+
* @param datasource the data source
186+
* @return the version of the Postgres database
187+
* @throws SQLException if a database access error occurs
188+
*/
189+
public static int getPostgresVersion(DataSource datasource) throws SQLException {
190+
try (Connection connection = datasource.getConnection()) {
191+
return connection.getMetaData().getDatabaseMajorVersion();
192+
}
193+
}
181194
}

0 commit comments

Comments
 (0)