diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 04f63c2a7..1da1490be 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,6 +5,7 @@ on: branches: - master - develop + - release* pull_request: type: [opened, reopened, edited] @@ -34,6 +35,7 @@ jobs: run: mvn $MAVEN_ARGS verify coverage: + if: github.repository == 'ydb-platform/ydb-java-sdk' name: Coverage YDB Java SDK runs-on: ubuntu-latest needs: build diff --git a/CHANGELOG.md b/CHANGELOG.md index f8d2ebe8f..9cf8d0c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.1.11 ## + +* Transports: Add option withGrpcKeepAliveTime to enable grpc keep-alives +* Table: Fixed creation and altering of dataColumns indexes in createTable/alterTable +* Upgraded version of yc-auth to avoid jackson-databind vulnerability + ## 2.1.10 ## * Topics: Added message metadata support diff --git a/README.md b/README.md index 024349a02..ea76baac8 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Firstly you can import YDB Java BOM to specify correct versions of SDK modules. tech.ydb ydb-sdk-bom - 2.1.10 + 2.1.11 pom import diff --git a/bom/pom.xml b/bom/pom.xml index 90c48d37d..5ff1f627a 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -6,7 +6,7 @@ 4.0.0 tech.ydb - 2.1.10 + 2.1.11 ydb-sdk-bom Java SDK Bill of Materials Java SDK Bill of Materials (BOM) @@ -16,7 +16,7 @@ 1.0.0 1.5.2 - 2.1.0 + 2.1.1 diff --git a/coordination/pom.xml b/coordination/pom.xml index 244a70089..54ceb02af 100644 --- a/coordination/pom.xml +++ b/coordination/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ydb-sdk-coordination diff --git a/core/pom.xml b/core/pom.xml index 5b53086c7..8899add12 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ydb-sdk-core diff --git a/core/src/main/java/tech/ydb/core/grpc/GrpcTransportBuilder.java b/core/src/main/java/tech/ydb/core/grpc/GrpcTransportBuilder.java index 045b46b8b..3b1daece7 100644 --- a/core/src/main/java/tech/ydb/core/grpc/GrpcTransportBuilder.java +++ b/core/src/main/java/tech/ydb/core/grpc/GrpcTransportBuilder.java @@ -51,6 +51,7 @@ public class GrpcTransportBuilder { * can cause leaks https://github.com/grpc/grpc-java/issues/9340 */ private boolean grpcRetry = false; + private Long grpcKeepAliveTimeMillis = null; GrpcTransportBuilder(@Nullable String endpoint, @Nullable HostAndPort host, @Nonnull String database) { this.endpoint = endpoint; @@ -132,6 +133,10 @@ public boolean isEnableRetry() { return grpcRetry; } + public Long getGrpcKeepAliveTimeMillis() { + return grpcKeepAliveTimeMillis; + } + public boolean useDefaultGrpcResolver() { return useDefaultGrpcResolver; } @@ -236,6 +241,18 @@ public GrpcTransportBuilder withDiscoveryTimeout(long timeout, TimeUnit unit) { return this; } + public GrpcTransportBuilder withGrpcKeepAliveTime(Duration time) { + this.grpcKeepAliveTimeMillis = time.toMillis(); + Preconditions.checkArgument(grpcKeepAliveTimeMillis > 0, "grpcKeepAliveTime must be greater than 0"); + return this; + } + + public GrpcTransportBuilder withGrpcKeepAliveTime(long time, TimeUnit unit) { + this.grpcKeepAliveTimeMillis = unit.toMillis(time); + Preconditions.checkArgument(grpcKeepAliveTimeMillis > 0, "grpcKeepAliveTime must be greater than 0"); + return this; + } + public GrpcTransportBuilder withCallExecutor(Executor executor) { this.callExecutor = Objects.requireNonNull(executor); return this; diff --git a/core/src/main/java/tech/ydb/core/impl/pool/ManagedChannelFactory.java b/core/src/main/java/tech/ydb/core/impl/pool/ManagedChannelFactory.java index ea49daf59..7ddee2fe9 100644 --- a/core/src/main/java/tech/ydb/core/impl/pool/ManagedChannelFactory.java +++ b/core/src/main/java/tech/ydb/core/impl/pool/ManagedChannelFactory.java @@ -1,6 +1,7 @@ package tech.ydb.core.impl.pool; import java.io.ByteArrayInputStream; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import javax.net.ssl.SSLException; @@ -39,6 +40,7 @@ public class ManagedChannelFactory { private final boolean retryEnabled; private final long connectTimeoutMs; private final boolean useDefaultGrpcResolver; + private final Long grpcKeepAliveTimeMillis; private ManagedChannelFactory(GrpcTransportBuilder builder) { this.database = builder.getDatabase(); @@ -49,6 +51,7 @@ private ManagedChannelFactory(GrpcTransportBuilder builder) { this.retryEnabled = builder.isEnableRetry(); this.connectTimeoutMs = builder.getConnectTimeoutMillis(); this.useDefaultGrpcResolver = builder.useDefaultGrpcResolver(); + this.grpcKeepAliveTimeMillis = builder.getGrpcKeepAliveTimeMillis(); } public long getConnectTimeoutMs() { @@ -80,6 +83,10 @@ public ManagedChannel newManagedChannel(String host, int port) { .defaultLoadBalancingPolicy(DEFAULT_BALANCER_POLICY); } + if (grpcKeepAliveTimeMillis != null) { + channelBuilder.keepAliveTime(grpcKeepAliveTimeMillis, TimeUnit.MILLISECONDS); + } + if (channelInitializer != null) { channelInitializer.accept(channelBuilder); } diff --git a/core/src/main/resources/version.properties b/core/src/main/resources/version.properties index 58483400e..1a30fbf9f 100644 --- a/core/src/main/resources/version.properties +++ b/core/src/main/resources/version.properties @@ -1 +1 @@ -version=2.1.10 +version=2.1.11 diff --git a/pom.xml b/pom.xml index a1835b9e3..723b06cf9 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 Java SDK for YDB Java SDK for YDB diff --git a/scheme/pom.xml b/scheme/pom.xml index a0fcaba09..e7f29c888 100644 --- a/scheme/pom.xml +++ b/scheme/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ydb-sdk-scheme diff --git a/table/pom.xml b/table/pom.xml index f833dbcc2..9bcb0962c 100644 --- a/table/pom.xml +++ b/table/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ydb-sdk-table diff --git a/table/src/main/java/tech/ydb/table/description/ColumnFamily.java b/table/src/main/java/tech/ydb/table/description/ColumnFamily.java index cb26a29cb..699a02856 100644 --- a/table/src/main/java/tech/ydb/table/description/ColumnFamily.java +++ b/table/src/main/java/tech/ydb/table/description/ColumnFamily.java @@ -4,13 +4,19 @@ public final class ColumnFamily { private final String name; private final StoragePool data; private final Compression compression; - private final boolean keepInMemory; + @Deprecated public ColumnFamily(String name, StoragePool data, Compression compression, boolean keepInMemory) { this.name = name; this.data = data; this.compression = compression; - this.keepInMemory = keepInMemory; +// this.keepInMemory = keepInMemory; + } + + public ColumnFamily(String name, StoragePool data, Compression compression) { + this.name = name; + this.data = data; + this.compression = compression; } public String getName() { @@ -25,8 +31,9 @@ public Compression getCompression() { return compression; } + @Deprecated public boolean isKeepInMemory() { - return keepInMemory; + return false; } public enum Compression { diff --git a/table/src/main/java/tech/ydb/table/impl/BaseSession.java b/table/src/main/java/tech/ydb/table/impl/BaseSession.java index 2e23eaeaa..490a92af9 100644 --- a/table/src/main/java/tech/ydb/table/impl/BaseSession.java +++ b/table/src/main/java/tech/ydb/table/impl/BaseSession.java @@ -53,6 +53,7 @@ import tech.ydb.table.settings.AutoPartitioningPolicy; import tech.ydb.table.settings.BeginTxSettings; import tech.ydb.table.settings.BulkUpsertSettings; +import tech.ydb.table.settings.Changefeed; import tech.ydb.table.settings.CommitTxSettings; import tech.ydb.table.settings.CopyTableSettings; import tech.ydb.table.settings.CopyTablesSettings; @@ -153,13 +154,7 @@ public static CompletableFuture> createSessionId(TableRpc tableRp .thenApply(result -> result.map(YdbTable.CreateSessionResult::getSessionId)); } - private static void applyPartitioningSettings( - PartitioningSettings partitioningSettings, - Consumer consumer) { - if (partitioningSettings == null) { - return; - } - + private static YdbTable.PartitioningSettings buildPartitioningSettings(PartitioningSettings partitioningSettings) { YdbTable.PartitioningSettings.Builder builder = YdbTable.PartitioningSettings.newBuilder(); if (partitioningSettings.getPartitioningByLoad() != null) { builder.setPartitioningByLoad( @@ -185,85 +180,126 @@ private static void applyPartitioningSettings( builder.setMaxPartitionsCount(partitioningSettings.getMaxPartitionsCount()); } - consumer.accept(builder.build()); + return builder.build(); + } + + private static YdbTable.ColumnMeta buildColumnMeta(TableColumn column) { + YdbTable.ColumnMeta.Builder builder = YdbTable.ColumnMeta.newBuilder() + .setName(column.getName()) + .setType(column.getType().toPb()); + if (column.getFamily() != null) { + builder.setFamily(column.getFamily()); + } + return builder.build(); + } + + private static YdbTable.Changefeed buildChangefeed(Changefeed changefeed) { + YdbTable.Changefeed.Builder builder = YdbTable.Changefeed.newBuilder() + .setName(changefeed.getName()) + .setFormat(changefeed.getFormat().toProto()) + .setVirtualTimestamps(changefeed.hasVirtualTimestamps()) + .setInitialScan(changefeed.hasInitialScan()) + .setMode(changefeed.getMode().toPb()); + + Duration retentionPeriod = changefeed.getRetentionPeriod(); + if (retentionPeriod != null) { + builder.setRetentionPeriod(com.google.protobuf.Duration.newBuilder() + .setSeconds(retentionPeriod.getSeconds()) + .setNanos(retentionPeriod.getNano()) + .build()); + } + + return builder.build(); + } + + private static YdbTable.TableIndex buildIndex(TableIndex index) { + YdbTable.TableIndex.Builder builder = YdbTable.TableIndex.newBuilder(); + builder.setName(index.getName()); + builder.addAllIndexColumns(index.getColumns()); + builder.addAllDataColumns(index.getDataColumns()); + switch (index.getType()) { + case GLOBAL_ASYNC: + builder.setGlobalAsyncIndex(YdbTable.GlobalAsyncIndex.getDefaultInstance()); + break; + case GLOBAL: + default: + builder.setGlobalIndex(YdbTable.GlobalIndex.getDefaultInstance()); + break; + } + return builder.build(); + } + + private static YdbTable.ColumnFamily buildColumnFamity(ColumnFamily family) { + YdbTable.ColumnFamily.Compression compression; + switch (family.getCompression()) { + case COMPRESSION_NONE: + compression = YdbTable.ColumnFamily.Compression.COMPRESSION_NONE; + break; + case COMPRESSION_LZ4: + compression = YdbTable.ColumnFamily.Compression.COMPRESSION_LZ4; + break; + default: + compression = YdbTable.ColumnFamily.Compression.COMPRESSION_UNSPECIFIED; + } + + return YdbTable.ColumnFamily.newBuilder() + .setCompression(compression) + .setData(YdbTable.StoragePool.newBuilder().setMedia(family.getData().getMedia())) + .setName(family.getName()) + .build(); + } + + private static YdbTable.TtlSettings buildTtlSettings(TtlSettings settings) { + return YdbTable.TtlSettings.newBuilder() + .setDateTypeColumn(YdbTable.DateTypeColumnModeSettings.newBuilder() + .setColumnName(settings.getDateTimeColumn()) + .setExpireAfterSeconds(settings.getExpireAfterSeconds()) + .build()) + .build(); } @Override public CompletableFuture createTable( String path, - TableDescription tableDescription, + TableDescription description, CreateTableSettings settings ) { YdbTable.CreateTableRequest.Builder request = YdbTable.CreateTableRequest.newBuilder() .setSessionId(id) .setPath(path) .setOperationParams(Operations.createParams(settings.toOperationSettings())) - .addAllPrimaryKey(tableDescription.getPrimaryKeys()); + .addAllPrimaryKey(description.getPrimaryKeys()); - applyPartitioningSettings(tableDescription.getPartitioningSettings(), request::setPartitioningSettings); - - for (TableColumn column : tableDescription.getColumns()) { - YdbTable.ColumnMeta.Builder builder = YdbTable.ColumnMeta.newBuilder() - .setName(column.getName()) - .setType(column.getType().toPb()); - if (column.getFamily() != null) { - builder.setFamily(column.getFamily()); - } - request.addColumns(builder.build()); + for (ColumnFamily family: description.getColumnFamilies()) { + request.addColumnFamilies(buildColumnFamity(family)); } - for (TableIndex index : tableDescription.getIndexes()) { - YdbTable.TableIndex.Builder b = request.addIndexesBuilder(); - b.setName(index.getName()); - b.addAllIndexColumns(index.getColumns()); - if (index.getType() == TableIndex.Type.GLOBAL) { - b.setGlobalIndex(YdbTable.GlobalIndex.getDefaultInstance()); - } + for (TableColumn column : description.getColumns()) { + request.addColumns(buildColumnMeta(column)); + } - if (index.getType() == TableIndex.Type.GLOBAL_ASYNC) { - b.setGlobalAsyncIndex(YdbTable.GlobalAsyncIndex.getDefaultInstance()); - } + for (TableIndex index : description.getIndexes()) { + request.addIndexes(buildIndex(index)); } + if (settings.getTtlSettings() != null) { + request.setTtlSettings(buildTtlSettings(settings.getTtlSettings())); + } - for (ColumnFamily family : tableDescription.getColumnFamilies()) { - YdbTable.ColumnFamily.Compression compression; - switch (family.getCompression()) { - case COMPRESSION_NONE: - compression = YdbTable.ColumnFamily.Compression.COMPRESSION_NONE; - break; - case COMPRESSION_LZ4: - compression = YdbTable.ColumnFamily.Compression.COMPRESSION_LZ4; - break; - default: - compression = YdbTable.ColumnFamily.Compression.COMPRESSION_UNSPECIFIED; - } - request.addColumnFamilies( - YdbTable.ColumnFamily.newBuilder() - .setKeepInMemoryValue(family.isKeepInMemory() ? - tech.ydb.proto.common.CommonProtos.FeatureFlag.Status.ENABLED.getNumber() : - tech.ydb.proto.common.CommonProtos.FeatureFlag.Status.DISABLED.getNumber()) - .setCompression(compression) - .setData(YdbTable.StoragePool.newBuilder().setMedia(family.getData().getMedia())) - .setName(family.getName()) - .build()); + if (description.getPartitioningSettings() != null) { + request.setPartitioningSettings(buildPartitioningSettings(description.getPartitioningSettings())); } if (settings.getPresetName() != null) { - request.getProfileBuilder() - .setPresetName(settings.getPresetName()); + request.getProfileBuilder().setPresetName(settings.getPresetName()); } if (settings.getExecutionPolicy() != null) { - request.getProfileBuilder() - .getExecutionPolicyBuilder() - .setPresetName(settings.getExecutionPolicy()); + request.getProfileBuilder().getExecutionPolicyBuilder().setPresetName(settings.getExecutionPolicy()); } if (settings.getCompactionPolicy() != null) { - request.getProfileBuilder() - .getCompactionPolicyBuilder() - .setPresetName(settings.getCompactionPolicy()); + request.getProfileBuilder().getCompactionPolicyBuilder().setPresetName(settings.getCompactionPolicy()); } PartitioningPolicy partitioningPolicy = settings.getPartitioningPolicy(); @@ -325,14 +361,6 @@ public CompletableFuture createTable( CommonProtos.FeatureFlag.Status.ENABLED : CommonProtos.FeatureFlag.Status.DISABLED); } - TtlSettings ttlSettings = settings.getTtlSettings(); - if (ttlSettings != null) { - YdbTable.DateTypeColumnModeSettings.Builder dateTypeColumnBuilder = request.getTtlSettingsBuilder() - .getDateTypeColumnBuilder(); - dateTypeColumnBuilder.setColumnName(ttlSettings.getDateTimeColumn()); - dateTypeColumnBuilder.setExpireAfterSeconds(ttlSettings.getExpireAfterSeconds()); - } - final GrpcRequestSettings grpcRequestSettings = makeGrpcRequestSettings(settings.getTimeoutDuration()); return tableRpc.createTable(request.build(), grpcRequestSettings); } @@ -369,28 +397,37 @@ public CompletableFuture alterTable(String path, AlterTableSettings sett .setPath(path) .setOperationParams(Operations.createParams(settings.toOperationSettings())); - settings.forEachAddColumn((name, type) -> { - builder.addAddColumns(YdbTable.ColumnMeta.newBuilder() - .setName(name) - .setType(type.toPb()) - .build()); - }); + for (TableColumn addColumn: settings.getAddColumns()) { + builder.addAddColumns(buildColumnMeta(addColumn)); + } + + for (Changefeed addChangefeed: settings.getAddChangefeeds()) { + builder.addAddChangefeeds(buildChangefeed(addChangefeed)); + } - settings.forEachDropColumn(builder::addDropColumns); + for (TableIndex index : settings.getAddIndexes()) { + builder.addAddIndexes(buildIndex(index)); + } + + if (settings.getTtlSettings() != null) { + builder.setSetTtlSettings(buildTtlSettings(settings.getTtlSettings())); + } - settings.forEachAddChangefeed(changefeed -> builder.addAddChangefeeds(changefeed.toProto())); + if (settings.getPartitioningSettings() != null) { + builder.setAlterPartitioningSettings(buildPartitioningSettings(settings.getPartitioningSettings())); + } - settings.forEachDropChangefeed(builder::addDropChangefeeds); + for (String dropColumn: settings.getDropColumns()) { + builder.addDropColumns(dropColumn); + } - TtlSettings ttlSettings = settings.getTtlSettings(); - if (ttlSettings != null) { - YdbTable.DateTypeColumnModeSettings.Builder dateTypeColumnBuilder = builder.getSetTtlSettingsBuilder() - .getDateTypeColumnBuilder(); - dateTypeColumnBuilder.setColumnName(ttlSettings.getDateTimeColumn()); - dateTypeColumnBuilder.setExpireAfterSeconds(ttlSettings.getExpireAfterSeconds()); + for (String dropChangefeed: settings.getDropChangefeeds()) { + builder.addDropChangefeeds(dropChangefeed); } - applyPartitioningSettings(settings.getPartitioningSettings(), builder::setAlterPartitioningSettings); + for (String dropIndex: settings.getDropIndexes()) { + builder.addDropIndexes(dropIndex); + } final GrpcRequestSettings grpcRequestSettings = makeGrpcRequestSettings(settings.getTimeoutDuration()); return tableRpc.alterTable(builder.build(), grpcRequestSettings); @@ -520,11 +557,7 @@ private static TableDescription mapDescribeTable( compression = ColumnFamily.Compression.COMPRESSION_NONE; } description.addColumnFamily( - new ColumnFamily(family.getName(), - new StoragePool(family.getData().getMedia()), - compression, - family.getKeepInMemory().equals(tech.ydb.proto.common.CommonProtos. - FeatureFlag.Status.ENABLED)) + new ColumnFamily(family.getName(), new StoragePool(family.getData().getMedia()), compression) ); } } diff --git a/table/src/main/java/tech/ydb/table/impl/pool/SessionPool.java b/table/src/main/java/tech/ydb/table/impl/pool/SessionPool.java index 7a7421361..d6168c92a 100644 --- a/table/src/main/java/tech/ydb/table/impl/pool/SessionPool.java +++ b/table/src/main/java/tech/ydb/table/impl/pool/SessionPool.java @@ -110,17 +110,16 @@ private boolean pollNext(CompletableFuture> future) { nextSession.whenComplete((session, th) -> { if (th != null) { - if (future.isDone()) { - logger.warn("can't get session, future is already canceled", th); + Throwable ex = Async.unwrapCompletionException(th); + Result fail = (ex instanceof UnexpectedResultException) + ? Result.fail((UnexpectedResultException) ex) + : Result.error("can't create session", ex); + + if (!future.complete(fail)) { + logger.warn("session acquisition failed with status {}", fail); return; } - Throwable ex = Async.unwrapCompletionException(th); - if (ex instanceof UnexpectedResultException) { - future.complete(Result.fail((UnexpectedResultException) ex)); - } else { - future.complete(Result.error("can't create session", ex)); - } } if (session != null) { validateSession(session, future); @@ -185,9 +184,9 @@ public CompletableFuture create() { .thenApply(response -> { if (!response.isSuccess()) { stats.failed.increment(); + throw new UnexpectedResultException("create session problem", response.getStatus()); } - String id = response.getValue(); - return new ClosableSession(id, tableRpc, keepQueryText); + return new ClosableSession(response.getValue(), tableRpc, keepQueryText); }); } @@ -196,13 +195,13 @@ public void destroy(ClosableSession session) { stats.deleted.increment(); session.delete(new DeleteSessionSettings()).whenComplete((status, th) -> { if (th != null) { - logger.warn("session {} removed with exception {}", session.getId(), th.getMessage()); + logger.warn("session {} destoryed with exception {}", session.getId(), th.getMessage()); } if (status != null) { if (status.isSuccess()) { - logger.debug("session {} successful removed", session.getId()); + logger.debug("session {} successful destoryed", session.getId()); } else { - logger.warn("session {} removed with status {}", session.getId(), status.toString()); + logger.warn("session {} destoryed with status {}", session.getId(), status.toString()); } } }); diff --git a/table/src/main/java/tech/ydb/table/settings/AlterTableSettings.java b/table/src/main/java/tech/ydb/table/settings/AlterTableSettings.java index 9c1f5d520..ebedc4291 100644 --- a/table/src/main/java/tech/ydb/table/settings/AlterTableSettings.java +++ b/table/src/main/java/tech/ydb/table/settings/AlterTableSettings.java @@ -1,14 +1,19 @@ package tech.ydb.table.settings; -import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Consumer; import javax.annotation.Nullable; +import tech.ydb.table.description.TableColumn; +import tech.ydb.table.description.TableIndex; +import tech.ydb.table.values.OptionalType; import tech.ydb.table.values.Type; @@ -17,21 +22,46 @@ */ public class AlterTableSettings extends RequestSettings { - private final LinkedHashMap addColumns = new LinkedHashMap<>(); - private final HashSet dropColumns = new HashSet<>(); + private final Map addColumns = new HashMap<>(); + private final Map addChangefeeds = new HashMap<>(); + private final Map addIndexes = new HashMap<>(); + + private final Set dropColumns = new HashSet<>(); + private final Set dropChangefeeds = new HashSet<>(); + private final Set dropIndexes = new HashSet<>(); + @Nullable private TtlSettings ttlSettings; @Nullable private PartitioningSettings partitioningSettings; - private final List addChangefeeds = new ArrayList<>(); - private final List dropChangefeeds = new ArrayList<>(); public AlterTableSettings() { } + @Deprecated public AlterTableSettings addColumn(String name, Type type) { - addColumns.put(name, type); + addColumns.put(name, new TableColumn(name, type)); + return this; + } + + public AlterTableSettings addNonnullColumn(String name, Type type) { + addColumns.put(name, new TableColumn(name, type)); + return this; + } + + public AlterTableSettings addNonnullColumn(String name, Type type, String family) { + addColumns.put(name, new TableColumn(name, type, family)); + return this; + } + + public AlterTableSettings addNullableColumn(String name, Type type) { + addColumns.put(name, new TableColumn(name, OptionalType.of(type))); + return this; + } + + public AlterTableSettings addNullableColumn(String name, Type type, String family) { + addColumns.put(name, new TableColumn(name, OptionalType.of(type), family)); return this; } @@ -41,7 +71,7 @@ public AlterTableSettings dropColumn(String name) { } public AlterTableSettings addChangefeed(Changefeed changefeed) { - addChangefeeds.add(changefeed); + addChangefeeds.put(changefeed.getName(), changefeed); return this; } @@ -50,20 +80,63 @@ public AlterTableSettings dropChangefeed(String changefeed) { return this; } - public void forEachAddColumn(BiConsumer fn) { - addColumns.forEach(fn); + public AlterTableSettings addGlobalIndex(String name, List columns) { + addIndexes.put(name, new TableIndex(name, columns, TableIndex.Type.GLOBAL)); + return this; } - public void forEachDropColumn(Consumer fn) { - dropColumns.forEach(fn); + public AlterTableSettings addGlobalIndex(String name, List columns, List dataColumns) { + addIndexes.put(name, new TableIndex(name, columns, dataColumns, TableIndex.Type.GLOBAL)); + return this; } - public void forEachAddChangefeed(Consumer fn) { - addChangefeeds.forEach(fn); + public AlterTableSettings addGlobalAsyncIndex(String name, List columns) { + addIndexes.put(name, new TableIndex(name, columns, TableIndex.Type.GLOBAL_ASYNC)); + return this; } - public void forEachDropChangefeed(Consumer fn) { - dropChangefeeds.forEach(fn); + public AlterTableSettings addGlobalAsyncIndex(String name, List columns, List dataColumns) { + addIndexes.put(name, new TableIndex(name, columns, dataColumns, TableIndex.Type.GLOBAL_ASYNC)); + return this; + } + + public AlterTableSettings dropIndex(String index) { + dropIndexes.add(index); + return this; + } + + public AlterTableSettings setTtlSettings(@Nullable TtlSettings ttlSettings) { + this.ttlSettings = ttlSettings; + return this; + } + + public AlterTableSettings setPartitioningSettings(@Nullable PartitioningSettings partitioningSettings) { + this.partitioningSettings = partitioningSettings; + return this; + } + + public Collection getAddColumns() { + return addColumns.values(); + } + + public Collection getAddChangefeeds() { + return addChangefeeds.values(); + } + + public Collection getAddIndexes() { + return addIndexes.values(); + } + + public Collection getDropColumns() { + return dropColumns; + } + + public Collection getDropChangefeeds() { + return dropChangefeeds; + } + + public Collection getDropIndexes() { + return dropIndexes; } @Nullable @@ -76,11 +149,23 @@ public PartitioningSettings getPartitioningSettings() { return partitioningSettings; } - public void setTtlSettings(@Nullable TtlSettings ttlSettings) { - this.ttlSettings = ttlSettings; + @Deprecated + public void forEachAddColumn(BiConsumer fn) { + addColumns.values().forEach(c -> fn.accept(c.getName(), c.getType())); } - public void setPartitioningSettings(@Nullable PartitioningSettings partitioningSettings) { - this.partitioningSettings = partitioningSettings; + @Deprecated + public void forEachDropColumn(Consumer fn) { + dropColumns.forEach(fn); + } + + @Deprecated + public void forEachAddChangefeed(Consumer fn) { + addChangefeeds.values().forEach(fn); + } + + @Deprecated + public void forEachDropChangefeed(Consumer fn) { + dropChangefeeds.forEach(fn); } } diff --git a/table/src/main/java/tech/ydb/table/settings/Changefeed.java b/table/src/main/java/tech/ydb/table/settings/Changefeed.java index 4c17e8f64..eb7335bd5 100644 --- a/table/src/main/java/tech/ydb/table/settings/Changefeed.java +++ b/table/src/main/java/tech/ydb/table/settings/Changefeed.java @@ -22,7 +22,7 @@ public enum Mode { this.proto = proto; } - YdbTable.ChangefeedMode.Mode toPb() { + public YdbTable.ChangefeedMode.Mode toPb() { return proto; } } @@ -36,7 +36,7 @@ public enum Format { this.proto = proto; } - YdbTable.ChangefeedFormat.Format toProto() { + public YdbTable.ChangefeedFormat.Format toProto() { return proto; } } @@ -81,6 +81,7 @@ public Duration getRetentionPeriod() { return retentionPeriod; } + @Deprecated public YdbTable.Changefeed toProto() { YdbTable.Changefeed.Builder builder = YdbTable.Changefeed.newBuilder() .setName(name) diff --git a/table/src/test/java/tech/ydb/table/integration/AlterTableTest.java b/table/src/test/java/tech/ydb/table/integration/AlterTableTest.java new file mode 100644 index 000000000..5e9426b00 --- /dev/null +++ b/table/src/test/java/tech/ydb/table/integration/AlterTableTest.java @@ -0,0 +1,173 @@ +package tech.ydb.table.integration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.After; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; + +import tech.ydb.core.Result; +import tech.ydb.core.Status; +import tech.ydb.table.SessionRetryContext; +import tech.ydb.table.description.TableColumn; +import tech.ydb.table.description.TableDescription; +import tech.ydb.table.description.TableIndex; +import tech.ydb.table.impl.SimpleTableClient; +import tech.ydb.table.rpc.grpc.GrpcTableRpc; +import tech.ydb.table.settings.AlterTableSettings; +import tech.ydb.table.settings.CreateTableSettings; +import tech.ydb.table.values.PrimitiveType; +import tech.ydb.table.values.Type; +import tech.ydb.test.junit4.GrpcTransportRule; + +/** + * + * @author Aleksandr Gorshenin + */ +public class AlterTableTest { + private final static String DEFAULT_FAMILY = "default"; + private final static String EMPTY_FAMILY = ""; + + @ClassRule + public final static GrpcTransportRule ydbTransport = new GrpcTransportRule(); + + private final String TABLE_NAME = "alter_table_test"; + + private final SimpleTableClient tableClient = SimpleTableClient.newClient( + GrpcTableRpc.useTransport(ydbTransport) + ).build(); + + private final SessionRetryContext ctx = SessionRetryContext.create(tableClient).build(); + + private final String tablePath = ydbTransport.getDatabase() + "/" + TABLE_NAME; + + @After + public void dropTable() { + ctx.supplyStatus(session -> session.dropTable(tablePath)).join(); + } + + @Test + public void alterTableTest() { + // --------------------- craete table ----------------------------- + TableDescription createTableDesc = TableDescription.newBuilder() + .addNonnullColumn("id", PrimitiveType.Uint64) + .addNullableColumn("code", PrimitiveType.Text) + .addNullableColumn("size", PrimitiveType.Float) + .addNullableColumn("created", PrimitiveType.Timestamp) + .addNullableColumn("data", PrimitiveType.Text) + .setPrimaryKey("id") + .addGlobalIndex("idx1", Arrays.asList("id", "code")) + .addGlobalAsyncIndex("idx2", Arrays.asList("data"), Arrays.asList("code")) + .build(); + + Status createStatus = ctx.supplyStatus( + session -> session.createTable(tablePath, createTableDesc, new CreateTableSettings()) + ).join(); + Assert.assertTrue("Create table with indexes " + createStatus, createStatus.isSuccess()); + + // --------------------- describe table after creating ----------------------------- + Result describeResult = ctx.supplyResult(session ->session.describeTable(tablePath)).join(); + Assert.assertTrue("Describe table with indexes " + describeResult.getStatus(), describeResult.isSuccess()); + + TableDescription description = describeResult.getValue(); + + Assert.assertEquals(1, description.getColumnFamilies().size()); + Assert.assertEquals(DEFAULT_FAMILY, description.getColumnFamilies().get(0).getName()); + + Assert.assertEquals(5, description.getColumns().size()); + assertColumn(description.getColumns().get(0), "id", PrimitiveType.Uint64); + assertColumn(description.getColumns().get(1), "code", PrimitiveType.Text.makeOptional()); + assertColumn(description.getColumns().get(2), "size", PrimitiveType.Float.makeOptional()); + assertColumn(description.getColumns().get(3), "created", PrimitiveType.Timestamp.makeOptional()); + assertColumn(description.getColumns().get(4), "data", PrimitiveType.Text.makeOptional()); + + Assert.assertEquals(2, description.getIndexes().size()); + assertIndexSync(description.getIndexes().get(0), "idx1", Arrays.asList("id", "code"), Collections.emptyList()); + assertIndexAsync(description.getIndexes().get(1), "idx2", Arrays.asList("data"), Arrays.asList("code")); + + // --------------------- alter table with changing columns ----------------------------- + + Status alterStatus = ctx.supplyStatus( + session -> session.alterTable(tablePath, new AlterTableSettings() + .addNullableColumn("data2", PrimitiveType.Bytes) + .dropColumn("created")) + ).join(); + Assert.assertTrue("Alter table with column " + alterStatus, alterStatus.isSuccess()); + + // --------------------- describe table after first altering ----------------------------- + describeResult = ctx.supplyResult(session ->session.describeTable(tablePath)).join(); + Assert.assertTrue("Describe table after altering " + describeResult.getStatus(), describeResult.isSuccess()); + + description = describeResult.getValue(); + + Assert.assertEquals(1, description.getColumnFamilies().size()); + Assert.assertEquals(DEFAULT_FAMILY, description.getColumnFamilies().get(0).getName()); + + Assert.assertEquals(5, description.getColumns().size()); + assertColumn(description.getColumns().get(0), "id", PrimitiveType.Uint64); + assertColumn(description.getColumns().get(1), "code", PrimitiveType.Text.makeOptional()); + assertColumn(description.getColumns().get(2), "size", PrimitiveType.Float.makeOptional()); + assertColumn(description.getColumns().get(3), "data", PrimitiveType.Text.makeOptional()); + assertColumn(description.getColumns().get(4), "data2", PrimitiveType.Bytes.makeOptional()); + + Assert.assertEquals(2, description.getIndexes().size()); + assertIndexSync(description.getIndexes().get(0), "idx1", Arrays.asList("id", "code"), Collections.emptyList()); + assertIndexAsync(description.getIndexes().get(1), "idx2", Arrays.asList("data"), Arrays.asList("code")); + + // --------------------- alter table with changing indexes ----------------------------- + alterStatus = ctx.supplyStatus( + session -> session.alterTable(tablePath, new AlterTableSettings() + .dropIndex("idx1")) + ).join(); + Assert.assertTrue("Alter table with indexes " + alterStatus, alterStatus.isSuccess()); + + // --------------------- describe table after first altering ----------------------------- + describeResult = ctx.supplyResult(session ->session.describeTable(tablePath)).join(); + Assert.assertTrue("Describe table after altering " + describeResult.getStatus(), describeResult.isSuccess()); + + description = describeResult.getValue(); + + Assert.assertEquals(1, description.getColumnFamilies().size()); + Assert.assertEquals(DEFAULT_FAMILY, description.getColumnFamilies().get(0).getName()); + + Assert.assertEquals(5, description.getColumns().size()); + assertColumn(description.getColumns().get(0), "id", PrimitiveType.Uint64); + assertColumn(description.getColumns().get(1), "code", PrimitiveType.Text.makeOptional()); + assertColumn(description.getColumns().get(2), "size", PrimitiveType.Float.makeOptional()); + assertColumn(description.getColumns().get(3), "data", PrimitiveType.Text.makeOptional()); + assertColumn(description.getColumns().get(4), "data2", PrimitiveType.Bytes.makeOptional()); + + Assert.assertEquals(1, description.getIndexes().size()); + assertIndexAsync(description.getIndexes().get(0), "idx2", Arrays.asList("data"), Arrays.asList("code")); + } + + private void assertColumn(TableColumn column, String name, Type type) { + Assert.assertEquals(name, column.getName()); + Assert.assertEquals(type, column.getType()); + Assert.assertEquals(EMPTY_FAMILY, column.getFamily()); + } + + private void assertIndexSync(TableIndex index, String name, List columns, List dataColumns) { + Assert.assertEquals(name, index.getName()); + Assert.assertEquals(TableIndex.Type.GLOBAL, index.getType()); + assertListEquals(columns, index.getColumns()); + assertListEquals(dataColumns, index.getDataColumns()); + } + + private void assertIndexAsync(TableIndex index, String name, List columns, List dataColumns) { + Assert.assertEquals(name, index.getName()); + Assert.assertEquals(TableIndex.Type.GLOBAL_ASYNC, index.getType()); + assertListEquals(columns, index.getColumns()); + assertListEquals(dataColumns, index.getDataColumns()); + } + + private void assertListEquals(List expected, List values) { + Assert.assertEquals(expected.size(), values.size()); + for (int idx = 0; idx < expected.size(); idx += 1) { + Assert.assertEquals(expected.get(idx), values.get(idx)); + } + } +} diff --git a/tests/common/pom.xml b/tests/common/pom.xml index 8a53fa24a..eaf3306b2 100644 --- a/tests/common/pom.xml +++ b/tests/common/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ../../pom.xml diff --git a/tests/junit4-support/pom.xml b/tests/junit4-support/pom.xml index 4f0f0df0c..93176bba3 100644 --- a/tests/junit4-support/pom.xml +++ b/tests/junit4-support/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ../../pom.xml diff --git a/tests/junit5-support/pom.xml b/tests/junit5-support/pom.xml index 0f53f1ff7..cef52f8e2 100644 --- a/tests/junit5-support/pom.xml +++ b/tests/junit5-support/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ../../pom.xml diff --git a/topic/pom.xml b/topic/pom.xml index add4211ad..f7bfa3cf4 100644 --- a/topic/pom.xml +++ b/topic/pom.xml @@ -8,7 +8,7 @@ tech.ydb ydb-sdk-parent - 2.1.10 + 2.1.11 ydb-sdk-topic