Skip to content

Commit bb2d5dd

Browse files
hubgeterhubgeter
committed
[fix](fe) Fix OceanBase JDBC catalog compatibility with newer drivers
1 parent 90f1eb8 commit bb2d5dd

File tree

6 files changed

+275
-45
lines changed

6 files changed

+275
-45
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
import java.util.Arrays;
5353
import java.util.List;
5454
import java.util.Map;
55+
import java.util.regex.Matcher;
56+
import java.util.regex.Pattern;
5557

5658
/**
5759
* External JDBC Catalog resource for external table query.
@@ -163,6 +165,8 @@ public class JdbcResource extends Resource {
163165

164166
// timeout for both connection and read. 10 seconds is long enough.
165167
private static final int HTTP_TIMEOUT_MS = 10000;
168+
private static final Pattern OCEANBASE_DRIVER_VERSION_PATTERN =
169+
Pattern.compile("oceanbase-client-(\\d+(?:\\.\\d+){1,3})\\.jar", Pattern.CASE_INSENSITIVE);
166170
@SerializedName(value = "configs")
167171
private Map<String, String> configs;
168172

@@ -185,7 +189,7 @@ public void modifyProperties(Map<String, String> properties) throws DdlException
185189
for (String propertyKey : ALL_PROPERTIES) {
186190
replaceIfEffectiveValue(this.configs, propertyKey, properties.get(propertyKey));
187191
}
188-
this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL)));
192+
this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL), getProperty(DRIVER_URL)));
189193
super.modifyProperties(properties);
190194
}
191195

@@ -216,7 +220,7 @@ protected void setProperties(ImmutableMap<String, String> properties) throws Ddl
216220
throw new DdlException("JdbcResource Missing " + property + " in properties");
217221
}
218222
}
219-
this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL)));
223+
this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL), getProperty(DRIVER_URL)));
220224
configs.put(CHECK_SUM, computeObjectChecksum(getProperty(DRIVER_URL)));
221225
}
222226

@@ -396,6 +400,10 @@ public static String parseDbType(String url) throws DdlException {
396400
}
397401

398402
public static String handleJdbcUrl(String jdbcUrl) throws DdlException {
403+
return handleJdbcUrl(jdbcUrl, null);
404+
}
405+
406+
public static String handleJdbcUrl(String jdbcUrl, String driverUrl) throws DdlException {
399407
// delete all space in jdbcUrl
400408
String newJdbcUrl = jdbcUrl.replaceAll(" ", "");
401409
String dbType = parseDbType(newJdbcUrl);
@@ -415,6 +423,10 @@ public static String handleJdbcUrl(String jdbcUrl) throws DdlException {
415423
if (dbType.equals(OCEANBASE)) {
416424
// set useCursorFetch to true
417425
newJdbcUrl = checkAndSetJdbcBoolParam(dbType, newJdbcUrl, "useCursorFetch", "false", "true");
426+
if (shouldDisableOceanBaseLegacyDatetimeCode(driverUrl)) {
427+
newJdbcUrl = checkAndSetJdbcBoolParam(dbType, newJdbcUrl,
428+
"useLegacyDatetimeCode", "true", "false");
429+
}
418430
}
419431
}
420432
if (dbType.equals(POSTGRESQL)) {
@@ -429,6 +441,31 @@ public static String handleJdbcUrl(String jdbcUrl) throws DdlException {
429441
return newJdbcUrl;
430442
}
431443

444+
private static boolean shouldDisableOceanBaseLegacyDatetimeCode(String driverUrl) {
445+
if (driverUrl == null || driverUrl.isEmpty()) {
446+
return false;
447+
}
448+
Matcher matcher = OCEANBASE_DRIVER_VERSION_PATTERN.matcher(driverUrl);
449+
if (!matcher.find()) {
450+
return false;
451+
}
452+
return compareVersion(matcher.group(1), "2.4.15") >= 0;
453+
}
454+
455+
private static int compareVersion(String leftVersion, String rightVersion) {
456+
String[] left = leftVersion.split("\\.");
457+
String[] right = rightVersion.split("\\.");
458+
int maxLength = Math.max(left.length, right.length);
459+
for (int i = 0; i < maxLength; i++) {
460+
int leftPart = i < left.length ? Integer.parseInt(left[i]) : 0;
461+
int rightPart = i < right.length ? Integer.parseInt(right[i]) : 0;
462+
if (leftPart != rightPart) {
463+
return Integer.compare(leftPart, rightPart);
464+
}
465+
}
466+
return 0;
467+
}
468+
432469
/**
433470
* Check jdbcUrl param, if the param is not set, set it to the expected value.
434471
* If the param is set to an unexpected value, replace it with the expected value.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ protected Map<String, String> processCompatibleProperties(Map<String, String> pr
159159
}
160160
String jdbcUrl = properties.getOrDefault(JdbcResource.JDBC_URL, "");
161161
if (!Strings.isNullOrEmpty(jdbcUrl)) {
162-
jdbcUrl = JdbcResource.handleJdbcUrl(jdbcUrl);
162+
jdbcUrl = JdbcResource.handleJdbcUrl(jdbcUrl, properties.get(JdbcResource.DRIVER_URL));
163163
properties.put(JdbcResource.JDBC_URL, jdbcUrl);
164164
}
165165
return properties;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java‎

Lines changed: 129 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
package org.apache.doris.datasource.jdbc.client;
1919

2020
import org.apache.doris.catalog.ArrayType;
21+
import org.apache.doris.catalog.JdbcResource;
2122
import org.apache.doris.catalog.ScalarType;
2223
import org.apache.doris.catalog.Type;
2324
import org.apache.doris.common.util.Util;
2425
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
2526

27+
import com.google.common.annotations.VisibleForTesting;
2628
import com.google.common.base.Preconditions;
2729
import com.google.common.base.Strings;
2830
import com.google.common.collect.ImmutableList;
@@ -35,7 +37,9 @@
3537
import java.sql.ResultSet;
3638
import java.sql.SQLException;
3739
import java.sql.Statement;
40+
import java.sql.Types;
3841
import java.util.List;
42+
import java.util.Locale;
3943
import java.util.Map;
4044
import java.util.Optional;
4145
import java.util.Set;
@@ -168,12 +172,18 @@ public List<JdbcFieldSchema> getJdbcColumnsInfo(String remoteDbName, String remo
168172
rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName);
169173

170174
Map<String, String> mapFieldtoType = Maps.newHashMap();
171-
if (isDoris) {
175+
if (requiresFullTypeDefinition()) {
172176
mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName, remoteTableName);
173177
}
174178

175179
while (rs.next()) {
176-
JdbcFieldSchema field = new JdbcFieldSchema(rs, mapFieldtoType);
180+
JdbcFieldSchema field = new JdbcFieldSchema(rs);
181+
String fullTypeName = mapFieldtoType.get(field.getColumnName());
182+
if (isDoris) {
183+
field.setDataTypeName(Optional.ofNullable(fullTypeName));
184+
} else {
185+
field.setFullDataTypeName(Optional.ofNullable(fullTypeName));
186+
}
177187
tableSchema.add(field);
178188
}
179189
} catch (SQLException e) {
@@ -225,27 +235,31 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
225235
if (isDoris) {
226236
return dorisTypeToDoris(fieldSchema);
227237
}
228-
// For mysql type: "INT UNSIGNED":
229-
// fieldSchema.getDataTypeName().orElse("unknown").split(" ")[0] == "INT"
230-
// fieldSchema.getDataTypeName().orElse("unknown").split(" ")[1] == "UNSIGNED"
231-
String[] typeFields = fieldSchema.getDataTypeName().orElse("unknown").split(" ");
232-
String mysqlType = typeFields[0];
238+
return mysqlTypeToDoris(fieldSchema, enableMappingVarbinary, enableMappingTimestampTz, convertDateToNull);
239+
}
240+
241+
@VisibleForTesting
242+
static Type mysqlTypeToDoris(JdbcFieldSchema fieldSchema, boolean enableMappingVarbinary,
243+
boolean enableMappingTimestampTz, boolean convertDateToNull) {
244+
MySqlTypeDescriptor typeDescriptor = MySqlTypeDescriptor.from(fieldSchema);
245+
String mysqlType = typeDescriptor.baseType;
233246
// For unsigned int, should extend the type.
234-
if (typeFields.length > 1 && "UNSIGNED".equals(typeFields[1])) {
247+
if (typeDescriptor.unsigned) {
235248
switch (mysqlType) {
236249
case "TINYINT":
237250
return Type.SMALLINT;
238251
case "SMALLINT":
239252
case "MEDIUMINT":
240253
return Type.INT;
241254
case "INT":
255+
case "INTEGER":
242256
return Type.BIGINT;
243257
case "BIGINT":
244258
return Type.LARGEINT;
245259
case "DECIMAL": {
246-
int precision = fieldSchema.requiredColumnSize() + 1;
247-
int scale = fieldSchema.requiredDecimalDigits();
248-
return createDecimalOrStringType(precision, scale);
260+
int precision = getColumnLength(fieldSchema, typeDescriptor) + 1;
261+
int scale = getDecimalScale(fieldSchema, typeDescriptor);
262+
return createMysqlDecimalOrStringType(precision, scale);
249263
}
250264
case "DOUBLE":
251265
// As of MySQL 8.0.17, the UNSIGNED attribute is deprecated
@@ -260,15 +274,20 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
260274
}
261275
}
262276
switch (mysqlType) {
277+
case "BOOL":
263278
case "BOOLEAN":
264279
return Type.BOOLEAN;
265280
case "TINYINT":
281+
if (fieldSchema.getDataType() == Types.BIT && getOptionalColumnLength(fieldSchema, typeDescriptor) == 1) {
282+
return Type.BOOLEAN;
283+
}
266284
return Type.TINYINT;
267285
case "SMALLINT":
268286
case "YEAR":
269287
return Type.SMALLINT;
270288
case "MEDIUMINT":
271289
case "INT":
290+
case "INTEGER":
272291
return Type.INT;
273292
case "BIGINT":
274293
return Type.BIGINT;
@@ -278,11 +297,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
278297
}
279298
return ScalarType.createDateV2Type();
280299
case "TIMESTAMP": {
281-
int columnSize = fieldSchema.requiredColumnSize();
282-
int scale = columnSize > 19 ? columnSize - 20 : 0;
283-
if (scale > 6) {
284-
scale = 6;
285-
}
300+
int scale = getDateTimeScale(fieldSchema, typeDescriptor);
286301
if (convertDateToNull) {
287302
fieldSchema.setAllowNull(true);
288303
}
@@ -291,12 +306,8 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
291306
}
292307
case "DATETIME": {
293308
// mysql can support microsecond
294-
// use columnSize to calculate the precision of timestamp/datetime
295-
int columnSize = fieldSchema.requiredColumnSize();
296-
int scale = columnSize > 19 ? columnSize - 20 : 0;
297-
if (scale > 6) {
298-
scale = 6;
299-
}
309+
// use type definition when available, otherwise fall back to column size metadata
310+
int scale = getDateTimeScale(fieldSchema, typeDescriptor);
300311
if (convertDateToNull) {
301312
fieldSchema.setAllowNull(true);
302313
}
@@ -307,24 +318,24 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
307318
case "DOUBLE":
308319
return Type.DOUBLE;
309320
case "DECIMAL": {
310-
int precision = fieldSchema.requiredColumnSize();
311-
int scale = fieldSchema.requiredDecimalDigits();
312-
return createDecimalOrStringType(precision, scale);
321+
int precision = getColumnLength(fieldSchema, typeDescriptor);
322+
int scale = getDecimalScale(fieldSchema, typeDescriptor);
323+
return createMysqlDecimalOrStringType(precision, scale);
313324
}
314325
case "CHAR":
315-
return ScalarType.createCharType(fieldSchema.requiredColumnSize());
326+
return ScalarType.createCharType(getColumnLength(fieldSchema, typeDescriptor));
316327
case "VARCHAR":
317-
return ScalarType.createVarcharType(fieldSchema.requiredColumnSize());
328+
return ScalarType.createVarcharType(getColumnLength(fieldSchema, typeDescriptor));
318329
case "TINYBLOB":
319330
case "BLOB":
320331
case "MEDIUMBLOB":
321332
case "LONGBLOB":
322333
case "BINARY":
323334
case "VARBINARY":
324-
return enableMappingVarbinary ? ScalarType.createVarbinaryType(fieldSchema.requiredColumnSize())
335+
return enableMappingVarbinary ? ScalarType.createVarbinaryType(getColumnLength(fieldSchema, typeDescriptor))
325336
: ScalarType.createStringType();
326337
case "BIT":
327-
if (fieldSchema.requiredColumnSize() == 1) {
338+
if (getOptionalColumnLength(fieldSchema, typeDescriptor) == 1) {
328339
return Type.BOOLEAN;
329340
} else {
330341
return ScalarType.createStringType();
@@ -344,6 +355,10 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
344355
}
345356
}
346357

358+
private boolean requiresFullTypeDefinition() {
359+
return isDoris || JdbcResource.OCEANBASE.equals(dbType);
360+
}
361+
347362
private boolean isConvertDatetimeToNull(JdbcClientConfig jdbcClientConfig) {
348363
// Check if the JDBC URL contains "zeroDateTimeBehavior=convertToNull" or "zeroDateTimeBehavior=convert_to_null"
349364
String jdbcUrl = jdbcClientConfig.getJdbcUrl().toLowerCase();
@@ -462,4 +477,89 @@ private Type dorisTypeToDoris(JdbcFieldSchema fieldSchema) {
462477
return Type.UNSUPPORTED;
463478
}
464479
}
480+
481+
private static int getColumnLength(JdbcFieldSchema fieldSchema, MySqlTypeDescriptor typeDescriptor) {
482+
return typeDescriptor.length.orElseGet(fieldSchema::requiredColumnSize);
483+
}
484+
485+
private static int getOptionalColumnLength(JdbcFieldSchema fieldSchema, MySqlTypeDescriptor typeDescriptor) {
486+
return typeDescriptor.length.orElseGet(() -> fieldSchema.getColumnSize().orElse(0));
487+
}
488+
489+
private static int getDecimalScale(JdbcFieldSchema fieldSchema, MySqlTypeDescriptor typeDescriptor) {
490+
return typeDescriptor.scale.orElseGet(fieldSchema::requiredDecimalDigits);
491+
}
492+
493+
private static int getDateTimeScale(JdbcFieldSchema fieldSchema, MySqlTypeDescriptor typeDescriptor) {
494+
int scale = typeDescriptor.length.orElseGet(() -> {
495+
int columnSize = fieldSchema.getColumnSize().orElse(0);
496+
return columnSize > 19 ? columnSize - 20 : 0;
497+
});
498+
return Math.min(scale, 6);
499+
}
500+
501+
private static Type createMysqlDecimalOrStringType(int precision, int scale) {
502+
if (precision <= ScalarType.MAX_DECIMAL128_PRECISION && precision > 0) {
503+
return ScalarType.createDecimalV3Type(precision, scale);
504+
}
505+
return ScalarType.createStringType();
506+
}
507+
508+
private static class MySqlTypeDescriptor {
509+
private final String baseType;
510+
private final boolean unsigned;
511+
private final Optional<Integer> length;
512+
private final Optional<Integer> scale;
513+
514+
private MySqlTypeDescriptor(String baseType, boolean unsigned, Optional<Integer> length,
515+
Optional<Integer> scale) {
516+
this.baseType = baseType;
517+
this.unsigned = unsigned;
518+
this.length = length;
519+
this.scale = scale;
520+
}
521+
522+
private static MySqlTypeDescriptor from(JdbcFieldSchema fieldSchema) {
523+
String typeName = fieldSchema.getFullDataTypeName()
524+
.orElse(fieldSchema.getDataTypeName().orElse("unknown"));
525+
String normalized = typeName == null ? "" : typeName.trim().replaceAll("\\s+", " ");
526+
if (normalized.isEmpty()) {
527+
return new MySqlTypeDescriptor("UNKNOWN", false, Optional.empty(), Optional.empty());
528+
}
529+
530+
String upperType = normalized.toUpperCase(Locale.ROOT);
531+
boolean unsigned = upperType.contains(" UNSIGNED");
532+
int openParen = upperType.indexOf('(');
533+
int firstSpace = upperType.indexOf(' ');
534+
int endIndex = upperType.length();
535+
if (openParen >= 0) {
536+
endIndex = openParen;
537+
} else if (firstSpace >= 0) {
538+
endIndex = firstSpace;
539+
}
540+
String baseType = upperType.substring(0, endIndex);
541+
542+
Optional<Integer> length = Optional.empty();
543+
Optional<Integer> scale = Optional.empty();
544+
if (openParen >= 0) {
545+
int closeParen = upperType.indexOf(')', openParen + 1);
546+
if (closeParen > openParen + 1) {
547+
String[] parameters = upperType.substring(openParen + 1, closeParen).split(",");
548+
length = parseTypeParameter(parameters[0]);
549+
if (parameters.length > 1) {
550+
scale = parseTypeParameter(parameters[1]);
551+
}
552+
}
553+
}
554+
return new MySqlTypeDescriptor(baseType, unsigned, length, scale);
555+
}
556+
557+
private static Optional<Integer> parseTypeParameter(String parameter) {
558+
try {
559+
return Optional.of(Integer.parseInt(parameter.trim()));
560+
} catch (NumberFormatException e) {
561+
return Optional.empty();
562+
}
563+
}
564+
}
465565
}

0 commit comments

Comments
 (0)