From 5323868030bc817e3a56d444f709f4c36b0aed13 Mon Sep 17 00:00:00 2001 From: yibole Date: Thu, 15 Oct 2026 16:44:55 -0700 Subject: [PATCH] feat(codegen): Translate paginators on the Smithy path The Smithy builder passed an empty paginator map and never set isPaginated, so no paginator classes or xxxPaginator() client methods were generated from a Smithy model. AddSmithyPaginators builds the map from each operation's @paginated trait, with the service's @paginated filling the members the operation leaves out, as Smithy's PaginatedIndex does. A new codegen trait, paginatorOverride, replaces that translation for paginators @paginated cannot express (several tokens, moreResults, several result keys), or with skip, drops the paginator. The builder sets isPaginated with the rule C2J's AddOperations uses, so the client methods and paginator classes agree, and removes paginators of deprecatedOperations as the C2J builder does. --- .../codegen/smithy/AddSmithyPaginators.java | 81 ++++++++++ .../SmithyIntermediateModelBuilder.java | 11 +- .../smithy/traits/PaginatorOverrideTrait.java | 140 ++++++++++++++++++ ...re.amazon.smithy.model.traits.TraitService | 1 + .../smithy/aws-sdk-java-codegen-traits.smithy | 18 +++ .../smithy/AddSmithyOperationsTest.java | 15 -- .../smithy/AddSmithyPaginatorsTest.java | 140 ++++++++++++++++++ .../SmithyIntermediateModelBuilderTest.java | 77 +++++++++- .../build/AwsSdkJavaCodegenPluginTest.java | 71 +++++++++ .../traits/SmithyCodegenTraitsTest.java | 77 +++++++++- 10 files changed, 609 insertions(+), 22 deletions(-) create mode 100644 codegen/src/main/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginators.java create mode 100644 codegen/src/main/java/software/amazon/awssdk/codegen/smithy/traits/PaginatorOverrideTrait.java create mode 100644 codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginatorsTest.java diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginators.java b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginators.java new file mode 100644 index 000000000000..a3df74f2dcd5 --- /dev/null +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginators.java @@ -0,0 +1,81 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.smithy; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import software.amazon.awssdk.codegen.model.service.PaginatorDefinition; +import software.amazon.awssdk.codegen.smithy.traits.PaginatorOverrideTrait; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.traits.PaginatedTrait; + +/** + * Builds a service's paginator definitions, keyed by operation name as in {@code paginators-1.json}. Smithy + * counterpart to loading that file. + * + *

An operation's {@link PaginatorOverrideTrait} wins. Otherwise its {@code @paginated} trait is translated, with the + * service's {@code @paginated} filling the members the operation leaves out, as Smithy's {@code PaginatedIndex} does. + */ +final class AddSmithyPaginators { + + private AddSmithyPaginators() { + } + + static Map paginators(Model model, ServiceShape service) { + PaginatedTrait serviceDefaults = service.getTrait(PaginatedTrait.class).orElse(null); + Map paginators = new TreeMap<>(); + for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) { + String name = operation.getId().getName(); + Optional override = operation.getTrait(PaginatorOverrideTrait.class); + if (override.isPresent()) { + if (!override.get().isSkip()) { + paginators.put(name, fromOverride(override.get())); + } + continue; + } + operation.getTrait(PaginatedTrait.class) + .ifPresent(trait -> paginators.put(name, fromPaginated(trait.merge(serviceDefaults)))); + } + return paginators; + } + + /** + * {@code @paginated} has no {@code moreResults} and one member per field, so each list has at most one element. + */ + private static PaginatorDefinition fromPaginated(PaginatedTrait trait) { + PaginatorDefinition definition = new PaginatorDefinition(); + definition.setInputToken(trait.getInputToken().map(Collections::singletonList).orElse(null)); + definition.setOutputToken(trait.getOutputToken().map(Collections::singletonList).orElse(null)); + definition.setResultKey(trait.getItems().map(Collections::singletonList).orElse(null)); + definition.setLimitKey(trait.getPageSize().orElse(null)); + return definition; + } + + private static PaginatorDefinition fromOverride(PaginatorOverrideTrait override) { + PaginatorDefinition definition = new PaginatorDefinition(); + definition.setInputToken(override.getInputToken()); + definition.setOutputToken(override.getOutputToken()); + definition.setResultKey(override.getResultKey().orElse(null)); + definition.setMoreResults(override.getMoreResults().orElse(null)); + definition.setLimitKey(override.getLimitKey().orElse(null)); + return definition; + } +} diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilder.java b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilder.java index 9112905c09fd..9c5782dbbdf1 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilder.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilder.java @@ -40,6 +40,7 @@ import software.amazon.awssdk.codegen.model.service.AuthType; import software.amazon.awssdk.codegen.model.service.ClientContextParam; import software.amazon.awssdk.codegen.model.service.EndpointRuleSetModel; +import software.amazon.awssdk.codegen.model.service.PaginatorDefinition; import software.amazon.awssdk.codegen.naming.DefaultSmithyNamingStrategy; import software.amazon.awssdk.codegen.naming.NamingStrategy; import software.amazon.awssdk.codegen.utils.ProtocolUtils; @@ -64,7 +65,7 @@ * {@link software.amazon.awssdk.codegen.IntermediateModelBuilder}. * *

Source-model customizations are expected to have been applied to the model as smithy-build - * transforms. Paginators and waiters are not applied yet. + * transforms. Paginators come from {@code @paginated} and codegen overrides; waiters are not applied yet. */ public final class SmithyIntermediateModelBuilder { @@ -143,6 +144,11 @@ public IntermediateModel build() { new TreeMap<>(new AddSmithyOperations(model, service, namingStrategy, customConfig, protocol) .constructOperations()); + Map paginators = AddSmithyPaginators.paginators(model, service); + // The client methods follow isPaginated and the paginator classes follow the map, so the two must agree. + operations.forEach((name, operation) -> operation.setPaginated( + paginators.containsKey(name) && paginators.get(name).hasAllRequiredFields())); + OperationModel endpointOperation = null; boolean endpointCacheRequired = false; for (OperationModel o : operations.values()) { @@ -164,6 +170,7 @@ public IntermediateModel build() { } operations.entrySet().removeIf(e -> customConfig.getDeprecatedOperations().contains(e.getKey())); + paginators.entrySet().removeIf(e -> customConfig.getDeprecatedOperations().contains(e.getKey())); log.info("{} shapes found in total.", shapes.size()); @@ -172,7 +179,7 @@ public IntermediateModel build() { IntermediateModel fullModel = new IntermediateModel( AddSmithyMetadata.constructMetadata(model, service, serviceIndex, namingStrategy, customConfig), operations, shapes, customConfig, endpointOperation, - Collections.emptyMap(), namingStrategy, Collections.emptyMap(), + paginators, namingStrategy, Collections.emptyMap(), endpointRuleSet, endpointTestSuiteModel, clientContextParams); SmithyIntermediateModelPostprocessor.create(customConfig).postprocess(fullModel); diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/traits/PaginatorOverrideTrait.java b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/traits/PaginatorOverrideTrait.java new file mode 100644 index 000000000000..f75558aaf239 --- /dev/null +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/smithy/traits/PaginatorOverrideTrait.java @@ -0,0 +1,140 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.smithy.traits; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.smithy.model.node.ExpectationNotMetException; +import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.node.ObjectNode; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.model.traits.AbstractTrait; +import software.amazon.smithy.model.traits.Trait; + +/** + * {@code software.amazon.awssdk.codegen#paginatorOverride}: the paginator to generate for an operation instead of the + * one {@code @paginated} produces, for a {@code paginators-1.json} entry that {@code @paginated} cannot express, or + * {@code skip} to generate none. Each member means what the {@code paginators-1.json} field of the same name means. + */ +@SdkInternalApi +public final class PaginatorOverrideTrait extends AbstractTrait { + public static final ShapeId ID = ShapeId.from("software.amazon.awssdk.codegen#paginatorOverride"); + + private static final String SKIP = "skip"; + private static final String INPUT_TOKEN = "inputToken"; + private static final String OUTPUT_TOKEN = "outputToken"; + private static final String RESULT_KEY = "resultKey"; + private static final String MORE_RESULTS = "moreResults"; + private static final String LIMIT_KEY = "limitKey"; + + private final ObjectNode node; + private final boolean skip; + private final List inputToken; + private final List outputToken; + private final List resultKey; + private final String moreResults; + private final String limitKey; + + public PaginatorOverrideTrait(ObjectNode node) { + super(ID, node); + node.expectNoAdditionalProperties(Arrays.asList(SKIP, INPUT_TOKEN, OUTPUT_TOKEN, RESULT_KEY, MORE_RESULTS, + LIMIT_KEY)); + this.node = node; + this.skip = node.getMember(SKIP).map(n -> n.expectBooleanNode().getValue()).orElse(false); + this.inputToken = strings(node, INPUT_TOKEN); + this.outputToken = strings(node, OUTPUT_TOKEN); + this.resultKey = strings(node, RESULT_KEY); + this.moreResults = node.getMember(MORE_RESULTS).map(n -> n.expectStringNode().getValue()).orElse(null); + this.limitKey = node.getMember(LIMIT_KEY).map(n -> n.expectStringNode().getValue()).orElse(null); + + if (skip && node.getMembers().size() > 1) { + throw new ExpectationNotMetException(ID + ": skip cannot be combined with other members", node); + } + if (!skip && (isNullOrEmpty(inputToken) || isNullOrEmpty(outputToken))) { + throw new ExpectationNotMetException( + ID + " needs inputToken and outputToken unless skip is true. Remove the trait to use @paginated.", node); + } + } + + /** + * True to generate no paginator for the operation. The other members are then absent. + */ + public boolean isSkip() { + return skip; + } + + /** + * Empty only when {@link #isSkip()}. + */ + public List getInputToken() { + return inputToken == null ? Collections.emptyList() : inputToken; + } + + /** + * Empty only when {@link #isSkip()}. + */ + public List getOutputToken() { + return outputToken == null ? Collections.emptyList() : outputToken; + } + + public Optional> getResultKey() { + return Optional.ofNullable(resultKey); + } + + public Optional getMoreResults() { + return Optional.ofNullable(moreResults); + } + + public Optional getLimitKey() { + return Optional.ofNullable(limitKey); + } + + @Override + protected Node createNode() { + return node; + } + + private static List strings(ObjectNode node, String member) { + return node.getMember(member) + .map(n -> n.expectArrayNode().getElements().stream() + .map(element -> element.expectStringNode().getValue()) + .collect(Collectors.toList())) + .map(Collections::unmodifiableList) + .orElse(null); + } + + private static boolean isNullOrEmpty(List list) { + return list == null || list.isEmpty(); + } + + /** + * Registered in {@code META-INF/services} so Smithy builds this class when it loads the trait. + */ + public static final class Provider extends AbstractTrait.Provider { + public Provider() { + super(ID); + } + + @Override + public Trait createTrait(ShapeId target, Node value) { + return new PaginatorOverrideTrait(value.expectObjectNode()); + } + } +} diff --git a/codegen/src/main/resources/META-INF/services/software.amazon.smithy.model.traits.TraitService b/codegen/src/main/resources/META-INF/services/software.amazon.smithy.model.traits.TraitService index 5625c077c6d3..5a56354d1312 100644 --- a/codegen/src/main/resources/META-INF/services/software.amazon.smithy.model.traits.TraitService +++ b/codegen/src/main/resources/META-INF/services/software.amazon.smithy.model.traits.TraitService @@ -3,3 +3,4 @@ software.amazon.awssdk.codegen.smithy.traits.DeprecatedNameTrait$Provider software.amazon.awssdk.codegen.smithy.traits.EnumConstantNameTrait$Provider software.amazon.awssdk.codegen.smithy.traits.IgnoreDataTypeConversionFailuresTrait$Provider software.amazon.awssdk.codegen.smithy.traits.SyntheticTrait$Provider +software.amazon.awssdk.codegen.smithy.traits.PaginatorOverrideTrait$Provider diff --git a/codegen/src/main/resources/META-INF/smithy/aws-sdk-java-codegen-traits.smithy b/codegen/src/main/resources/META-INF/smithy/aws-sdk-java-codegen-traits.smithy index a96327abca61..fadcffde781a 100644 --- a/codegen/src/main/resources/META-INF/smithy/aws-sdk-java-codegen-traits.smithy +++ b/codegen/src/main/resources/META-INF/smithy/aws-sdk-java-codegen-traits.smithy @@ -26,3 +26,21 @@ string enumConstantName @trait(selector: ":is(simpleType, list, map, structure, union)") structure synthetic {} + +/// Replaces the paginator that @paginated produces for this operation, for a paginator @paginated cannot express. +/// Each member means what the paginators-1.json field of the same name means, and names members by their Smithy +/// member names. With skip, no paginator is generated. +@trait(selector: "operation") +structure paginatorOverride { + skip: Boolean + inputToken: PaginatorMemberPaths + outputToken: PaginatorMemberPaths + resultKey: PaginatorMemberPaths + moreResults: String + limitKey: String +} + +@private +list PaginatorMemberPaths { + member: String +} diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyOperationsTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyOperationsTest.java index 65c47e90983c..506fea771233 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyOperationsTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyOperationsTest.java @@ -390,21 +390,6 @@ void unsignedPayload_setsFlag() { assertThat(op(model, "rest-json").isUnsignedPayload()).isTrue(); } - // ---- Pagination (deferred to PR 15) ----------------------------------- - - @Test - void paginatedTrait_doesNotSetPaginated() { - Model model = modelOf( - "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"items\")\n" - + "@http(method: \"POST\", uri: \"/op\")\n" - + "operation Op { input: OpRequest, output: OpResponse }\n" - + "structure OpRequest { nextToken: String }\n" - + "structure OpResponse { nextToken: String, items: StringList }\n" - + "list StringList { member: String }\n"); - // Pagination is deferred to PR 15; isPaginated stays false here. - assertThat(op(model, "rest-json").isPaginated()).isFalse(); - } - // ---- Endpoint discovery ----------------------------------------------- @Test diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginatorsTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginatorsTest.java new file mode 100644 index 000000000000..40f5b6edfeef --- /dev/null +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/AddSmithyPaginatorsTest.java @@ -0,0 +1,140 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.smithy; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.codegen.model.service.PaginatorDefinition; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; + +class AddSmithyPaginatorsTest { + + private static final ClassLoader LOADER = AddSmithyPaginatorsTest.class.getClassLoader(); + + private static Map paginators(String serviceTraits, String operationTraits) { + String src = "$version: \"2.0\"\nnamespace demo\n\n" + + "use software.amazon.awssdk.codegen#paginatorOverride\n\n" + + serviceTraits + + "service DemoService { version: \"2024-01-01\", operations: [Op, Other] }\n\n" + + operationTraits + + "operation Op { input: OpRequest, output: OpResponse }\n" + + "operation Other { input: OpRequest, output: OpResponse }\n" + + "structure OpRequest { nextToken: String, maxResults: Integer }\n" + + "structure OpResponse { nextToken: String, items: StringList, result: Result }\n" + + "structure Result { next: String, items: StringList }\n" + + "list StringList { member: String }\n"; + Model model = Model.assembler(LOADER).discoverModels(LOADER).addUnparsedModel("test.smithy", src) + .assemble().unwrap(); + return AddSmithyPaginators.paginators(model, model.expectShape(ShapeId.from("demo#DemoService"), + ServiceShape.class)); + } + + @Test + void paginatedTrait_translatesEachMember() { + Map paginators = paginators( + "", "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"items\", " + + "pageSize: \"maxResults\")\n"); + + assertThat(paginators).containsOnlyKeys("Op"); + PaginatorDefinition definition = paginators.get("Op"); + assertThat(definition.getInputToken()).containsExactly("nextToken"); + assertThat(definition.getOutputToken()).containsExactly("nextToken"); + assertThat(definition.getResultKey()).containsExactly("items"); + assertThat(definition.getLimitKey()).isEqualTo("maxResults"); + assertThat(definition.getMoreResults()).isNull(); + } + + @Test + void absentItemsAndPageSize_leaveResultKeyAndLimitKeyNull() { + PaginatorDefinition definition = + paginators("", "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\")\n").get("Op"); + + assertThat(definition.getResultKey()).isNull(); + assertThat(definition.getLimitKey()).isNull(); + } + + @Test + void nestedOutputTokenAndItems_keepTheirPaths() { + PaginatorDefinition definition = paginators( + "", "@paginated(inputToken: \"nextToken\", outputToken: \"result.next\", items: \"result.items\")\n") + .get("Op"); + + assertThat(definition.getOutputToken()).containsExactly("result.next"); + assertThat(definition.getResultKey()).containsExactly("result.items"); + } + + @Test + void servicePaginatedTrait_fillsTheMembersTheOperationLeavesOut() { + PaginatorDefinition definition = paginators( + "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", pageSize: \"maxResults\")\n", + "@paginated(items: \"items\")\n").get("Op"); + + assertThat(definition.getInputToken()).containsExactly("nextToken"); + assertThat(definition.getOutputToken()).containsExactly("nextToken"); + assertThat(definition.getResultKey()).containsExactly("items"); + assertThat(definition.getLimitKey()).isEqualTo("maxResults"); + } + + @Test + void override_replacesThePaginatedTrait() { + PaginatorDefinition definition = paginators( + "", "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"items\")\n" + + "@paginatorOverride(inputToken: [\"nextToken\", \"maxResults\"], outputToken: [\"nextToken\", " + + "\"result.next\"], resultKey: [\"items\", \"result.items\"], moreResults: \"truncated\", " + + "limitKey: \"maxResults\")\n").get("Op"); + + assertThat(definition.getInputToken()).containsExactly("nextToken", "maxResults"); + assertThat(definition.getOutputToken()).containsExactly("nextToken", "result.next"); + assertThat(definition.getResultKey()).containsExactly("items", "result.items"); + assertThat(definition.getMoreResults()).isEqualTo("truncated"); + assertThat(definition.getLimitKey()).isEqualTo("maxResults"); + } + + @Test + void overrideWithoutPaginatedTrait_isUsed() { + Map paginators = + paginators("", "@paginatorOverride(inputToken: [\"nextToken\"], outputToken: [\"nextToken\"])\n"); + + assertThat(paginators).containsOnlyKeys("Op"); + assertThat(paginators.get("Op").getInputToken()).isEqualTo(Collections.singletonList("nextToken")); + assertThat(paginators.get("Op").getResultKey()).isNull(); + } + + @Test + void skipOverride_dropsThePaginatedTrait() { + Map paginators = paginators( + "", "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\")\n" + + "@paginatorOverride(skip: true)\n"); + + assertThat(paginators).isEmpty(); + } + + @Test + void overrideWithAnExpressionToken_isKeptAsWritten() { + PaginatorDefinition definition = paginators( + "", "@paginatorOverride(inputToken: [\"nextToken\"], outputToken: [\"nextToken || items[-1]\"])\n") + .get("Op"); + + assertThat(definition.getOutputToken()).isEqualTo(Arrays.asList("nextToken || items[-1]")); + assertThat(definition.hasAllRequiredFields()).isFalse(); + } +} diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilderTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilderTest.java index 140c6c2636a7..aad154d03e50 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilderTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/SmithyIntermediateModelBuilderTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import java.util.Collections; import org.junit.jupiter.api.Test; import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig; import software.amazon.awssdk.codegen.model.intermediate.EnumModel; @@ -27,12 +28,14 @@ import software.amazon.smithy.model.Model; /** - * Covers the endpoint models reaching the intermediate model from the service's traits. The Smithy - * path takes no sidecar input, so a service without the traits lands on the same defaults the C2J - * path uses when the sidecar files are absent. + * Covers what the builder takes from the service model instead of C2J sidecar files: the endpoint models, and the + * paginators with the {@code isPaginated} flag that must agree with them. A service without the endpoint traits + * lands on the same defaults the C2J path uses when the sidecar files are absent. */ class SmithyIntermediateModelBuilderTest { + private static final ClassLoader LOADER = SmithyIntermediateModelBuilderTest.class.getClassLoader(); + /** * {@code clientContextParams} binds the rule-set parameter to the service model, which the * rules-engine validator requires for any parameter that is not a registered built-in. @@ -114,6 +117,74 @@ void endpointTraitsAbsent_fallBackToTheDefaultRules() { assertThat(model.getEndpointTestSuiteModel().getTestCases()).isEmpty(); } + private static IntermediateModel buildPaginated(CustomizationConfig config) { + String src = + "$version: \"2.0\"\nnamespace demo\n\n" + + "use aws.api#service\n" + + "use aws.auth#sigv4\n" + + "use aws.protocols#restJson1\n" + + "use software.amazon.awssdk.codegen#paginatorOverride\n" + + "@service(sdkId: \"Demo\", arnNamespace: \"demo\")\n" + + "@sigv4(name: \"demo\")\n" + + "@restJson1\n" + + "service DemoService { version: \"2024-01-01\", operations: [ListThings, ListOld, ListExpr, Get] }\n\n" + + "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"things\")\n" + + "@readonly @http(method: \"GET\", uri: \"/things\")\n" + + "operation ListThings { input: ListRequest, output: ListResponse }\n" + + "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"things\")\n" + + "@readonly @http(method: \"GET\", uri: \"/old\")\n" + + "operation ListOld { input: ListRequest, output: ListResponse }\n" + + "@paginatorOverride(inputToken: [\"nextToken\"], outputToken: [\"nextToken || things[-1]\"])\n" + + "@readonly @http(method: \"GET\", uri: \"/expr\")\n" + + "operation ListExpr { input: ListRequest, output: ListResponse }\n" + + "@readonly @http(method: \"GET\", uri: \"/get\")\n" + + "operation Get { input: ListRequest, output: ListResponse }\n" + + "structure ListRequest { @httpQuery(\"nextToken\") nextToken: String }\n" + + "structure ListResponse { nextToken: String, things: Things }\n" + + "list Things { member: String }\n"; + Model smithyModel = Model.assembler(LOADER) + .discoverModels(LOADER) + .addUnparsedModel("test.smithy", src) + .assemble() + .unwrap(); + return new SmithyIntermediateModelBuilder( + SmithyModels.builder().model(smithyModel).customizationConfig(config).build()).build(); + } + + @Test + void paginatedOperations_areMarkedPaginatedAndHaveDefinitions() { + IntermediateModel model = buildPaginated(CustomizationConfig.create()); + + assertThat(model.getPaginators()).containsOnlyKeys("ListThings", "ListOld", "ListExpr"); + assertThat(model.getPaginators().get("ListThings").getResultKey()).containsExactly("things"); + assertThat(model.getOperation("ListThings").isPaginated()).isTrue(); + assertThat(model.getOperation("ListOld").isPaginated()).isTrue(); + assertThat(model.getOperation("Get").isPaginated()).isFalse(); + } + + /** + * C2J keeps such a definition in the map but generates nothing for it, so {@code isPaginated} must be false too: + * the client interface would otherwise reference a paginator class that does not exist. + */ + @Test + void definitionCodegenCannotGenerate_isNotMarkedPaginated() { + IntermediateModel model = buildPaginated(CustomizationConfig.create()); + + assertThat(model.getPaginators().get("ListExpr").hasAllRequiredFields()).isFalse(); + assertThat(model.getOperation("ListExpr").isPaginated()).isFalse(); + } + + @Test + void deprecatedOperation_losesItsPaginator() { + CustomizationConfig config = CustomizationConfig.create(); + config.setDeprecatedOperations(Collections.singletonList("ListOld")); + + IntermediateModel model = buildPaginated(config); + + assertThat(model.getOperations()).doesNotContainKey("ListOld"); + assertThat(model.getPaginators()).containsOnlyKeys("ListThings", "ListExpr"); + } + @Test void stringWithEnumTrait_isTranslatedAsEnum() { String src = diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/build/AwsSdkJavaCodegenPluginTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/build/AwsSdkJavaCodegenPluginTest.java index 1ad59ad6e4fb..de089e3fb418 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/build/AwsSdkJavaCodegenPluginTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/build/AwsSdkJavaCodegenPluginTest.java @@ -17,9 +17,13 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -100,4 +104,71 @@ void smithyBuild_drivesCodegen_andReportsGeneratedFiles(@TempDir Path outputDir) assertThat(generated).as("generated a client from the projected model") .anyMatch(f -> f.endsWith("Client.java")); } + + @Test + void paginatedAndOverriddenOperations_generatePaginators(@TempDir Path outputDir) throws IOException { + String paginatedModel = + "$version: \"2.0\"\nnamespace demo\n\n" + + "use aws.api#service\n" + + "use aws.auth#sigv4\n" + + "use aws.protocols#restJson1\n" + + "use software.amazon.awssdk.codegen#paginatorOverride\n" + + "@service(sdkId: \"Demo\", arnNamespace: \"demo\")\n" + + "@sigv4(name: \"demo\")\n" + + "@restJson1\n" + + "service DemoService { version: \"2024-01-01\", operations: [ListThings, ListParts] }\n\n" + + "@paginated(inputToken: \"nextToken\", outputToken: \"nextToken\", items: \"things\", " + + "pageSize: \"maxResults\")\n" + + "@readonly @http(method: \"GET\", uri: \"/things\")\n" + + "operation ListThings { input: ListThingsRequest, output: ListThingsResponse }\n" + + "structure ListThingsRequest {\n" + + " @httpQuery(\"nextToken\") nextToken: String\n" + + " @httpQuery(\"maxResults\") maxResults: Integer\n" + + "}\n" + + "structure ListThingsResponse { nextToken: String, things: Names }\n" + + "@paginatorOverride(inputToken: [\"marker\"], outputToken: [\"nextMarker\"], moreResults: \"isTruncated\", " + + "resultKey: [\"parts\", \"prefixes\"], limitKey: \"maxParts\")\n" + + "@readonly @http(method: \"GET\", uri: \"/parts\")\n" + + "operation ListParts { input: ListPartsRequest, output: ListPartsResponse }\n" + + "structure ListPartsRequest {\n" + + " @httpQuery(\"marker\") marker: String\n" + + " @httpQuery(\"maxParts\") maxParts: Integer\n" + + "}\n" + + "structure ListPartsResponse { nextMarker: String, isTruncated: Boolean, parts: Names, prefixes: Names }\n" + + "list Names { member: String }\n"; + Model model = Model.assembler(LOADER) + .discoverModels(LOADER) + .addUnparsedModel("test.smithy", paginatedModel) + .assemble() + .unwrap(); + ProjectionConfig sdk = ProjectionConfig.builder() + .plugins(Collections.singletonMap(AwsSdkJavaCodegenPlugin.NAME, + Node.objectNode())) + .build(); + SmithyBuildResult result = SmithyBuild.create(LOADER) + .config(SmithyBuildConfig.builder() + .version("1.0") + .projections(Collections.singletonMap("sdk", sdk)) + .build()) + .model(model) + .outputDirectory(outputDir) + .build(); + + assertThat(result.anyBroken()).as("smithy-build should not be broken").isFalse(); + Map generated = result.getProjectionResult("sdk").get() + .getPluginManifest(AwsSdkJavaCodegenPlugin.NAME).get() + .getFiles().stream() + .collect(Collectors.toMap(p -> p.getFileName().toString(), p -> p, + (first, second) -> first)); + assertThat(generated).containsKeys("ListThingsIterable.java", "ListThingsPublisher.java", + "ListPartsIterable.java", "ListPartsPublisher.java"); + // The override's moreResults decides when to stop, and each result key gets its own item method. + String listParts = new String(Files.readAllBytes(generated.get("ListPartsIterable.java")), + StandardCharsets.UTF_8); + assertThat(listParts).contains("previousPage.isTruncated() != null && previousPage.isTruncated().booleanValue()") + .contains("public final SdkIterable parts()") + .contains("public final SdkIterable prefixes()"); + String client = new String(Files.readAllBytes(generated.get("DemoClient.java")), StandardCharsets.UTF_8); + assertThat(client).contains("listThingsPaginator(").contains("listPartsPaginator("); + } } diff --git a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/traits/SmithyCodegenTraitsTest.java b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/traits/SmithyCodegenTraitsTest.java index eb8c08cc79c7..be321389654d 100644 --- a/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/traits/SmithyCodegenTraitsTest.java +++ b/codegen/src/test/java/software/amazon/awssdk/codegen/smithy/traits/SmithyCodegenTraitsTest.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.codegen.smithy.traits; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.tuple; import java.util.Arrays; @@ -27,7 +28,10 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.ArrayNode; +import software.amazon.smithy.model.node.ExpectationNotMetException; import software.amazon.smithy.model.node.Node; +import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.model.traits.Trait; @@ -40,6 +44,7 @@ class SmithyCodegenTraitsTest { private static final ClassLoader LOADER = SmithyCodegenTraitsTest.class.getClassLoader(); private static final ShapeId TARGET = ShapeId.from("demo#Shape$member"); + private static final ShapeId OPERATION = ShapeId.from("demo#Op"); static Stream traits() { return Stream.of(Arguments.of(DeprecatedNameTrait.ID, Node.from("Bucket"), DeprecatedNameTrait.class), @@ -63,13 +68,77 @@ void traitService_createsTheTraitClass(ShapeId id, Node value, Class invalidPaginatorOverrides() { + return Stream.of( + Arguments.of("skip with tokens", "{\"skip\": true, \"inputToken\": [\"a\"], \"outputToken\": [\"a\"]}", + "skip cannot be combined"), + Arguments.of("no outputToken", "{\"inputToken\": [\"a\"]}", "needs inputToken and outputToken"), + Arguments.of("skip false alone", "{\"skip\": false}", "Remove the trait to use @paginated"), + Arguments.of("empty inputToken", "{\"inputToken\": [], \"outputToken\": [\"a\"]}", + "needs inputToken and outputToken"), + Arguments.of("unknown member", "{\"inputToken\": [\"a\"], \"outputToken\": [\"a\"], \"items\": \"b\"}", + "items")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidPaginatorOverrides") + void invalidPaginatorOverride_isRejected(String description, String json, String message) { + ObjectNode node = Node.parse(json).expectObjectNode(); + + assertThatThrownBy(() -> new PaginatorOverrideTrait(node)) + .isInstanceOf(ExpectationNotMetException.class) + .hasMessageContaining(message); + } + + @Test + void paginatorOverrideOnStructure_isATraitTargetError() { + assertThat(errors("@paginatorOverride(skip: true) structure Shape { member: String }")) + .extracting(ValidationEvent::getId, e -> e.getShapeId().get()) + .containsExactly(tuple("TraitTarget", ShapeId.from("demo#Shape"))); + } + @Test void discoverModels_loadsTheTraitDefinitions() { Model model = Model.assembler(LOADER).discoverModels(LOADER).assemble().unwrap(); for (ShapeId id : Arrays.asList(DeprecatedNameTrait.ID, AlternateBeanPropertyNameTrait.ID, EnumConstantNameTrait.ID, IgnoreDataTypeConversionFailuresTrait.ID, - SyntheticTrait.ID)) { + SyntheticTrait.ID, PaginatorOverrideTrait.ID)) { assertThat(model.getShape(id)).as("definition of %s", id).isPresent(); assertThat(model.expectShape(id).hasTrait(TraitDefinition.class)).as("%s is a trait", id).isTrue(); } @@ -112,7 +181,10 @@ void traitsOnTheirSelectors_validate() { + "}\n" + "@synthetic enum Kind {\n @enumConstantName(\"FIVE_ONE\")\n A = \"5.1\"\n}\n" + "@synthetic string Name\n" - + "@synthetic structure Payload { member: String }\n")) + + "@synthetic structure Payload { member: String }\n" + + "@paginatorOverride(inputToken: [\"a\", \"b\"], outputToken: [\"c || d\"], " + + "resultKey: [\"e\"], moreResults: \"f\", limitKey: \"g\")\n" + + "operation Op {}\n")) .isEmpty(); } @@ -122,6 +194,7 @@ private static List errors(String shapes) { + "use software.amazon.awssdk.codegen#deprecatedName\n" + "use software.amazon.awssdk.codegen#enumConstantName\n" + "use software.amazon.awssdk.codegen#ignoreDataTypeConversionFailures\n" + + "use software.amazon.awssdk.codegen#paginatorOverride\n" + "use software.amazon.awssdk.codegen#synthetic\n\n" + shapes + "\n"; return Model.assembler(LOADER)