From 44ee820af72f19dde79745bace46b3544fd52388 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 18 Oct 2024 16:40:34 -0700 Subject: [PATCH 01/72] Start with model --- examples/tutorial/model/weather.smithy | 136 +++++++++++++++++++++++++ examples/tutorial/smithy-build.json | 10 ++ 2 files changed, 146 insertions(+) create mode 100644 examples/tutorial/model/weather.smithy create mode 100644 examples/tutorial/smithy-build.json diff --git a/examples/tutorial/model/weather.smithy b/examples/tutorial/model/weather.smithy new file mode 100644 index 000000000..df1cf637e --- /dev/null +++ b/examples/tutorial/model/weather.smithy @@ -0,0 +1,136 @@ +$version: "2" + +namespace example.weather + +/// Provides weather forecasts. +@paginated(inputToken: "nextToken", outputToken: "nextToken", pageSize: "pageSize") +service Weather { + version: "2006-03-01" + resources: [ + City + ] + operations: [ + GetCurrentTime + ] +} + +resource City { + identifiers: { cityId: CityId } + properties: { coordinates: CityCoordinates } + read: GetCity + list: ListCities + resources: [ + Forecast + ] +} + +resource Forecast { + identifiers: { cityId: CityId } + properties: { chanceOfRain: Float } + read: GetForecast +} + +// "pattern" is a trait. +@pattern("^[A-Za-z0-9 ]+$") +string CityId + +@readonly +operation GetCity { + input := for City { + // "cityId" provides the identifier for the resource and + // has to be marked as required. + @required + $cityId + } + + output := for City { + // "required" is used on output to indicate if the service + // will always provide a value for the member. + @required + @notProperty + name: String + + @required + $coordinates + } + + errors: [ + NoSuchResource + ] +} + +// This structure is nested within GetCityOutput. +structure CityCoordinates { + @required + latitude: Float + + @required + longitude: Float +} + +// "error" is a trait that is used to specialize +// a structure as an error. +@error("client") +structure NoSuchResource { + @required + resourceType: String +} + +// The paginated trait indicates that the operation may +// return truncated results. +@readonly +@paginated(items: "items") +operation ListCities { + input := { + nextToken: String + pageSize: Integer + } + + output := { + nextToken: String + + @required + items: CitySummaries + } +} + +// CitySummaries is a list of CitySummary structures. +list CitySummaries { + member: CitySummary +} + +// CitySummary contains a reference to a City. +@references([ + { + resource: City + } +]) +structure CitySummary { + @required + cityId: CityId + + @required + name: String +} + +@readonly +operation GetCurrentTime { + output := { + @required + time: Timestamp + } +} + +@readonly +operation GetForecast { + input := for Forecast { + // "cityId" provides the only identifier for the resource since + // a Forecast doesn't have its own. + @required + $cityId + } + + output := for Forecast { + $chanceOfRain + } +} diff --git a/examples/tutorial/smithy-build.json b/examples/tutorial/smithy-build.json new file mode 100644 index 000000000..942839164 --- /dev/null +++ b/examples/tutorial/smithy-build.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "sources": ["model"], + "maven": { + "dependencies": [ + ], + "repositories": [ + ] + } +} \ No newline at end of file From 609b3b8acb4e2e32a1f8711d2e16e2dea433c32d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 18 Oct 2024 16:50:18 -0700 Subject: [PATCH 02/72] m --- smithy-templates.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 smithy-templates.json diff --git a/smithy-templates.json b/smithy-templates.json new file mode 100644 index 000000000..28eb51218 --- /dev/null +++ b/smithy-templates.json @@ -0,0 +1,11 @@ +{ + "name": "smithy-dafny", + "templates": { + "tutorial": { + "documentation": "Add additional files to the main smithy source set.", + "path": "examples/tutorial", + "include": [ + ] + } + } +} \ No newline at end of file From d2e3de258cee77c00731bbbe5fbc07318eca5b2c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 18 Oct 2024 16:54:38 -0700 Subject: [PATCH 03/72] Rename --- smithy-templates.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/smithy-templates.json b/smithy-templates.json index 28eb51218..87f09bc56 100644 --- a/smithy-templates.json +++ b/smithy-templates.json @@ -1,8 +1,8 @@ { "name": "smithy-dafny", "templates": { - "tutorial": { - "documentation": "Add additional files to the main smithy source set.", + "polymorph-tutorial": { + "documentation": "Basic Polymorphing tutorial.", "path": "examples/tutorial", "include": [ ] From 968c352a68db4cb27b63ec628c4df30e727e238f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 18 Oct 2024 16:56:40 -0700 Subject: [PATCH 04/72] Add circular submodule - cause why not? --- .gitmodules | 3 +++ examples/tutorial/Makefile | 0 examples/tutorial/smithy-dafny | 1 + 3 files changed, 4 insertions(+) create mode 100644 examples/tutorial/Makefile create mode 160000 examples/tutorial/smithy-dafny diff --git a/.gitmodules b/.gitmodules index 03f9a1246..6e25de1c1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "smithy-dafny-codegen-modules/smithy-rs"] path = smithy-dafny-codegen-modules/smithy-rs url = https://github.com/smithy-lang/smithy-rs.git +[submodule "examples/tutorial/smithy-dafny"] + path = examples/tutorial/smithy-dafny + url = git@github.com:smithy-lang/smithy-dafny.git diff --git a/examples/tutorial/Makefile b/examples/tutorial/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/examples/tutorial/smithy-dafny b/examples/tutorial/smithy-dafny new file mode 160000 index 000000000..21ecddbb4 --- /dev/null +++ b/examples/tutorial/smithy-dafny @@ -0,0 +1 @@ +Subproject commit 21ecddbb4eaf719d1720e650bdc8738bdb9e78c6 From 6f23d26d3f88a179811422ee075c64602998b59c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 2 Apr 2025 16:30:40 -0700 Subject: [PATCH 05/72] Recover and update *DataStream types --- ...StandardLibraries-smithy-dafny-subset.toml | 9 +- .../StandardLibrary/src/Streams.dfy | 302 ++++++++++++++++++ .../StandardLibrary/src/dfyconfig.toml | 10 + TestModels/dafny-dependencies/dafny | 2 +- 4 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy create mode 100644 TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml index b3ea6e4ac..a8d9c8c64 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml +++ b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml @@ -1,15 +1,18 @@ includes = [ "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Actions.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Aggregators.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Enumerators.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Producers.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Consumers.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/GenericAction.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/BoundedInts.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Seq.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Set.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Multiset.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/DynamicArray.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Frames.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Functions.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Math.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Ordinal.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Relations.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Streams/Streams.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Termination.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Wrappers.dfy" ] diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy new file mode 100644 index 000000000..0ad5e982c --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -0,0 +1,302 @@ +/******************************************************************************* + * Copyright by the contributors to the Dafny Project + * SPDX-License-Identifier: MIT + *******************************************************************************/ + +// TODO: Relocate under Actions/ instead, I don't think Streams has to be a separate library? +module {:options "--function-syntax:4"} Std.Streams { + + import opened Wrappers + import opened Actions + import opened Producers + import opened BoundedInts + import opened Collections.Seq + import opened Termination + + // + // A data stream, i.e. a fallable producer of batches of values. + // + // Allows for streams that can only be read once, + // but see RewindableDataStream for a more specific trait + // that requires the ability to replay the enumeration, + // or seek to an arbitrary position (although this may take linear time). + // That requirement is not in this trait + // because there are lots of ways to implement a stream + // where having to replay forces buffering all previous values in memory, + // which often defeats the purpose of streaming in the first place. + // In particular, boto3 currently (quite implicitly) + // requires file-like streams with the ability to seek, + // but we don't want to force the same requirements on all streams. + // + @AssumeCrossModuleTermination + trait DataStream extends Producer, E>> { + + ghost const data: seq + + // The total length of all produced batches + function ContentLength(): (res: Option) + requires Valid() + reads this, Repr + + ghost predicate ValidOutputs(outputs: seq, E>>>) + requires Seq.Partitioned(outputs, IsSome) + ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + decreases Repr + + ghost predicate ValidDataSoFar(outputs: seq, E>>>) + requires Partitioned(outputs, IsSome) + { + var produced := ProducedOf(outputs); + && var dataSoFar := DataSoFar(produced); + && (dataSoFar.Some? ==> + && dataSoFar.value <= data + && (!Seq.All(outputs, IsSome) ==> dataSoFar.value == data)) + } + + ghost function DataSoFar(produced: seq, E>>): Option> + { + if exists o <- produced :: o.Failure? then + None + else + Some(Flatten(MapPartialFunction((o: Result, E>) requires o.Success? => o.value, produced))) + } + } + + trait RewindableDataStream extends DataStream { + + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + ensures Valid() ==> |data| <= UINT64_MAX as int + decreases Repr, 0 + + function ContentLength(): (res: Option) + requires Valid() + reads this, Repr + ensures res == Some(|data| as uint64) + + function Position(): (res: uint64) + requires Valid() + reads this, Repr + ensures res as int <= |data| + + method Seek(newPosition: uint64) + requires Valid() + requires newPosition as int <= |data| + modifies Repr + ensures Valid() + ensures Position() == newPosition + } + + /* + * Wraps an Producer up as a non-rewindable DataStream that cannot error. + */ + class ProducerDataStream extends DataStream { + + const wrapped: Producer> + const length: uint64 + + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + decreases Repr, 0 + { + && this in Repr + && ValidComponent(wrapped) + && ValidHistory(history) + } + + ghost predicate ValidOutputs(outputs: seq, ()>>>) + requires Seq.Partitioned(outputs, IsSome) + ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + decreases Repr + { + ValidDataSoFar(outputs) + } + + ghost function RemainingMetric(): TerminationMetric + requires Valid() + reads this, Repr + decreases Repr, 3 + { + TMSucc(wrapped.RemainingMetric()) + } + + constructor(wrapped: Producer>, length: uint64) + requires wrapped.Valid() + requires wrapped.history == [] + ensures Valid() + ensures fresh(Repr - wrapped.Repr) + { + this.wrapped := wrapped; + this.length := length; + + this.history := []; + this.Repr := {this} + wrapped.Repr; + } + + function ContentLength(): (res: Option) + requires Valid() + reads this, Repr + { + Some(length) + } + + method Invoke(i: ()) returns (r: Option, ()>>) + requires Requires(i) + reads Reads(i) + modifies Modifies(i) + decreases Decreases(i), 0 + ensures Ensures(i, r) + ensures RemainingDecreasedBy(r) + { + assert Requires(i); + + assert Valid(); + var result := wrapped.Next(); + + r := match result + case None => None + case Some(value) => Some(Success(value)); + UpdateHistory(i, r); + + // TODO: work to do + assume {:axiom} Valid(); + if r.Some? { + old(RemainingMetric()).SuccDecreasesToSucc(RemainingMetric()); + } else { + old(RemainingMetric()).SuccNonIncreasesToSucc(RemainingMetric()); + } + } + } + + /* + * Rewindable stream of a sequence with a configured chunk size. + */ + class SeqDataStream extends RewindableDataStream { + + const s: seq + const chunkSize: uint64 + var position: uint64 + + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + ensures Valid() ==> |data| <= UINT64_MAX as int + decreases Repr, 0 + { + && this in Repr + && ValidHistory(history) + && s == data + && |s| <= UINT64_MAX as int + && position as int <= |s| + && 0 < chunkSize + } + + ghost predicate ValidOutputs(outputs: seq, ()>>>) + requires Seq.Partitioned(outputs, IsSome) + ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + decreases Repr + { + ValidDataSoFar(outputs) + } + + ghost function RemainingMetric(): TerminationMetric + requires Valid() + reads this, Repr + decreases Repr, 3 + { + TMNat(|s| - position as int) + } + + constructor(s: seq, chunkSize: uint64) + requires |s| <= UINT64_MAX as int + requires 0 < chunkSize + ensures Valid() + { + this.data := s; + this.s := s; + this.position := 0; + this.chunkSize := chunkSize; + + this.history := []; + this.Repr := {this}; + } + + function ContentLength(): (res: Option) + requires Valid() + reads this, Repr + ensures res == Some(|data| as uint64) + { + Some(|s| as uint64) + } + + function Position(): (res: uint64) + requires Valid() + reads this, Repr + ensures res as int <= |data| + { + position + } + + method Seek(newPosition: uint64) + requires Valid() + requires newPosition as int <= |data| + modifies Repr + ensures Valid() + ensures Position() == newPosition + { + position := newPosition; + if position == 0 { + history := []; + } else { + history := [((), Some(Success(s[..position])))]; + } + + var outputs := Outputs(); + var produced := ProducedOf(outputs); + var dataSoFar := DataSoFar(produced); + if outputs == [] { + assert produced == []; + assert (forall o <- produced :: o.Success? ==> 0 < |o.value|); + } else { + assert outputs == [Some(Success(s[..position]))]; + assert produced == [Success(s[..position])]; + assert MapPartialFunction((o: Result, ()>) requires o.Success? => o.value, produced) == [s[..position]]; + assert Flatten([s[..position]]) == s[..position]; + assert dataSoFar == Some(s[..position]); + } + } + + method Invoke(t: ()) returns (r: Option, ()>>) + requires Requires(t) + reads Reads(t) + modifies Modifies(t) + decreases Decreases(t), 0 + ensures Ensures(t, r) + ensures RemainingDecreasedBy(r) + { + assert Requires(t); + + assert Valid(); + if position == |s| as uint64 { + r := None; + } else { + // Warning: unbounded integers + var size := Math.Min(chunkSize as int, |s| - position as int) as uint64; + var newPosition := position + size; + r := Some(Success(s[position..newPosition])); + position := newPosition; + } + UpdateHistory(t, r); + + reveal TerminationMetric.Ordinal(); + + // TODO: Work to do + assume {:axiom} Ensures(t, r); + } + } +} \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml b/TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml new file mode 100644 index 000000000..da8b8f871 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml @@ -0,0 +1,10 @@ +# Project file for easier development in VS Code. +# Not yet used in the actual Makefile build. + +[options] +unicode-char = false +function-syntax = 3 + +library = [ + "../bin/DafnyStandardLibraries-smithy-dafny-subset.doo" +] \ No newline at end of file diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 70831a833..685ef6436 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 70831a8332ccbca883ac54b273db26d98cc06e48 +Subproject commit 685ef64365feb5e6b76794fabf03bed8ab805002 From b36d8aece5da39d835a422add0e40c1d003fc927 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 7 Apr 2025 16:21:30 -0700 Subject: [PATCH 06/72] Add Read() (not yet verifying) --- .../StandardLibrary/src/Streams.dfy | 210 +++++++++++++----- 1 file changed, 150 insertions(+), 60 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 0ad5e982c..19840a02a 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -3,7 +3,6 @@ * SPDX-License-Identifier: MIT *******************************************************************************/ -// TODO: Relocate under Actions/ instead, I don't think Streams has to be a separate library? module {:options "--function-syntax:4"} Std.Streams { import opened Wrappers @@ -18,7 +17,7 @@ module {:options "--function-syntax:4"} Std.Streams { // // Allows for streams that can only be read once, // but see RewindableDataStream for a more specific trait - // that requires the ability to replay the enumeration, + // that requires the ability to replay the data, // or seek to an arbitrary position (although this may take linear time). // That requirement is not in this trait // because there are lots of ways to implement a stream @@ -31,50 +30,51 @@ module {:options "--function-syntax:4"} Std.Streams { @AssumeCrossModuleTermination trait DataStream extends Producer, E>> { - ghost const data: seq - // The total length of all produced batches - function ContentLength(): (res: Option) - requires Valid() + const contentLength: Option + + ghost predicate Valid() reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + decreases Repr, 0 ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) decreases Repr - ghost predicate ValidDataSoFar(outputs: seq, E>>>) - requires Partitioned(outputs, IsSome) - { - var produced := ProducedOf(outputs); - && var dataSoFar := DataSoFar(produced); - && (dataSoFar.Some? ==> - && dataSoFar.value <= data - && (!Seq.All(outputs, IsSome) ==> dataSoFar.value == data)) - } + method Read(max: uint64) returns (r: Option, E>>) + requires Requires(()) + reads Reads(()) + modifies Modifies(()) + decreases Decreases(()) + ensures Ensures((), r) + ensures 0 < max ==> RemainingDecreasedBy(r) + ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int + } - ghost function DataSoFar(produced: seq, E>>): Option> - { - if exists o <- produced :: o.Failure? then - None - else - Some(Flatten(MapPartialFunction((o: Result, E>) requires o.Success? => o.value, produced))) - } + ghost predicate ValidDataSoFar(outputs: seq, E>>>, length: int) + requires Partitioned(outputs, IsSome) + { + var produced := ProducedOf(outputs); + && var dataSoFar := DataSoFar(produced); + && (dataSoFar.Some? ==> + && |dataSoFar.value| <= length + && (!Seq.All(outputs, IsSome) ==> |dataSoFar.value| == length)) } + ghost function DataSoFar(produced: seq, E>>): Option> + { + if exists o <- produced :: o.Failure? then + None + else + Some(Flatten(MapPartialFunction((o: Result, E>) requires o.Success? => o.value, produced))) + } + trait RewindableDataStream extends DataStream { - ghost predicate Valid() - reads this, Repr - ensures Valid() ==> this in Repr - ensures Valid() ==> ValidHistory(history) - ensures Valid() ==> |data| <= UINT64_MAX as int - decreases Repr, 0 - - function ContentLength(): (res: Option) - requires Valid() - reads this, Repr - ensures res == Some(|data| as uint64) + ghost const data: seq function Position(): (res: uint64) requires Valid() @@ -89,13 +89,37 @@ module {:options "--function-syntax:4"} Std.Streams { ensures Position() == newPosition } + ghost predicate ValidDataLengthSoFar(outputs: seq>>, length: int) + requires Partitioned(outputs, IsSome) + { + var dataSoFar := Flatten(ProducedOf(outputs)); + && |dataSoFar| <= length + && (!Seq.All(outputs, IsSome) ==> |dataSoFar| == length) + } + + trait ProducesTotalLengthProof { + + const producer: Producer> + const length: int + + lemma ProducesTotalLength(history: seq<((), Option>)>) + requires producer.ValidHistory(history) + ensures ValidDataLengthSoFar(OutputsOf(history), length) + } + /* * Wraps an Producer up as a non-rewindable DataStream that cannot error. + * It implements Read() using Next(), and buffers extra data + * as needed. */ class ProducerDataStream extends DataStream { const wrapped: Producer> const length: uint64 + var buffer: seq + + ghost const producesTotalLengthProof: ProducesTotalLengthProof + ghost const maxWrappedRemaining: TerminationMetric ghost predicate Valid() reads this, Repr @@ -106,14 +130,17 @@ module {:options "--function-syntax:4"} Std.Streams { && this in Repr && ValidComponent(wrapped) && ValidHistory(history) + && producesTotalLengthProof.producer == wrapped + && producesTotalLengthProof.length == length as int + && |buffer| <= length as int } ghost predicate ValidOutputs(outputs: seq, ()>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) decreases Repr { - ValidDataSoFar(outputs) + contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -121,27 +148,24 @@ module {:options "--function-syntax:4"} Std.Streams { reads this, Repr decreases Repr, 3 { - TMSucc(wrapped.RemainingMetric()) + TMTuple(maxWrappedRemaining, wrapped.RemainingMetric(), TMNat(|buffer|)) } - constructor(wrapped: Producer>, length: uint64) + constructor(wrapped: Producer>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) requires wrapped.Valid() requires wrapped.history == [] + requires producesTotalLengthProof.producer == wrapped + requires producesTotalLengthProof.length == length as int ensures Valid() ensures fresh(Repr - wrapped.Repr) { this.wrapped := wrapped; this.length := length; + this.buffer := []; this.history := []; this.Repr := {this} + wrapped.Repr; - } - - function ContentLength(): (res: Option) - requires Valid() - reads this, Repr - { - Some(length) + this.producesTotalLengthProof := producesTotalLengthProof; } method Invoke(i: ()) returns (r: Option, ()>>) @@ -165,15 +189,61 @@ module {:options "--function-syntax:4"} Std.Streams { // TODO: work to do assume {:axiom} Valid(); if r.Some? { - old(RemainingMetric()).SuccDecreasesToSucc(RemainingMetric()); + old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); + } else { + old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); + } + } + + method Read(max: uint64) returns (r: Option, ()>>) + requires Requires(()) + reads Reads(()) + modifies Modifies(()) + decreases Decreases(()) + ensures Ensures((), r) + ensures 0 < max ==> RemainingDecreasedBy(r) + ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int + { + assert Requires(()); + + assert Valid(); + var next; + if 0 < |buffer| { + next := Some(buffer); + buffer := []; + } else { + next := wrapped.Next(); + } + assert next.Some? ==> |next.value| <= length as int; + + if next.None? { + r := None; + + OutputsPartitionedAfterOutputtingNone(); + ProduceNone(); } else { - old(RemainingMetric()).SuccNonIncreasesToSucc(RemainingMetric()); + var size := if max <= |next.value| as uint64 then max else |next.value| as uint64; + var value := Success(next.value[..size]); + r := Some(value); + buffer := next.value[size..]; + + OutputsPartitionedAfterOutputtingSome(value); + ProduceSome(value); + } + + if 0 < max { + if r.Some? { + old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); + } else { + old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); + } } + assert 0 < max ==> RemainingDecreasedBy(r); } } /* - * Rewindable stream of a sequence with a configured chunk size. + * Rewindable stream of a sequence with a configured default chunk size. */ class SeqDataStream extends RewindableDataStream { @@ -192,16 +262,18 @@ module {:options "--function-syntax:4"} Std.Streams { && ValidHistory(history) && s == data && |s| <= UINT64_MAX as int + && contentLength == Some(|s| as uint64) && position as int <= |s| && 0 < chunkSize } ghost predicate ValidOutputs(outputs: seq, ()>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) ==> ValidDataSoFar(outputs) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) decreases Repr { - ValidDataSoFar(outputs) + && contentLength.Some? + && ValidDataSoFar(outputs, contentLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -221,6 +293,7 @@ module {:options "--function-syntax:4"} Std.Streams { this.s := s; this.position := 0; this.chunkSize := chunkSize; + this.contentLength := Some(|s| as uint64); this.history := []; this.Repr := {this}; @@ -242,6 +315,7 @@ module {:options "--function-syntax:4"} Std.Streams { position } + @ResourceLimit("0") method Seek(newPosition: uint64) requires Valid() requires newPosition as int <= |data| @@ -261,10 +335,9 @@ module {:options "--function-syntax:4"} Std.Streams { var dataSoFar := DataSoFar(produced); if outputs == [] { assert produced == []; - assert (forall o <- produced :: o.Success? ==> 0 < |o.value|); } else { assert outputs == [Some(Success(s[..position]))]; - assert produced == [Success(s[..position])]; + assert produced == [Success(s[..position])] + ProducedOf([]); assert MapPartialFunction((o: Result, ()>) requires o.Success? => o.value, produced) == [s[..position]]; assert Flatten([s[..position]]) == s[..position]; assert dataSoFar == Some(s[..position]); @@ -273,30 +346,47 @@ module {:options "--function-syntax:4"} Std.Streams { method Invoke(t: ()) returns (r: Option, ()>>) requires Requires(t) - reads Reads(t) + reads this, Repr modifies Modifies(t) decreases Decreases(t), 0 ensures Ensures(t, r) ensures RemainingDecreasedBy(r) { - assert Requires(t); + assert Valid(); + r := Read(chunkSize); + } + + method Read(max: uint64) returns (r: Option, ()>>) + requires Requires(()) + reads this, Repr + modifies Modifies(()) + decreases Decreases(()) + ensures Ensures((), r) + ensures 0 < max ==> RemainingDecreasedBy(r) + ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int + { + assert Requires(()); assert Valid(); if position == |s| as uint64 { r := None; + + OutputsPartitionedAfterOutputtingNone(); + ProduceNone(); } else { // Warning: unbounded integers - var size := Math.Min(chunkSize as int, |s| - position as int) as uint64; + var remaining := |s| as uint64 - position; + var size := if max <= remaining then max else remaining; var newPosition := position + size; - r := Some(Success(s[position..newPosition])); + var chunk := Success(s[position..newPosition]); + r := Some(chunk); position := newPosition; + + OutputsPartitionedAfterOutputtingSome(chunk); + ProduceSome(chunk); } - UpdateHistory(t, r); reveal TerminationMetric.Ordinal(); - - // TODO: Work to do - assume {:axiom} Ensures(t, r); } } } \ No newline at end of file From 0fad32d2d694505a6262a10523308300be309934 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 8 Apr 2025 13:38:13 -0700 Subject: [PATCH 07/72] SeqDataStream verifies --- .../StandardLibrary/src/Streams.dfy | 160 ++++++++++++++++-- 1 file changed, 145 insertions(+), 15 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 19840a02a..2341f0cfa 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -57,21 +57,131 @@ module {:options "--function-syntax:4"} Std.Streams { ghost predicate ValidDataSoFar(outputs: seq, E>>>, length: int) requires Partitioned(outputs, IsSome) { - var produced := ProducedOf(outputs); - && var dataSoFar := DataSoFar(produced); + && var dataSoFar := DataSoFar(outputs); && (dataSoFar.Some? ==> && |dataSoFar.value| <= length && (!Seq.All(outputs, IsSome) ==> |dataSoFar.value| == length)) } - ghost function DataSoFar(produced: seq, E>>): Option> + // TODO: Move next to ProducedOf? + lemma AboutProducedOf(outputs: seq>, x: T) + requires Partitioned(outputs, IsSome) + ensures x in ProducedOf(outputs) <==> Some(x) in outputs + {} + + ghost function ValueOfSuccess(o: Result): T + requires o.Success? + { + o.value + } + + ghost function DataSoFar(outputs: seq, E>>>): Option> + requires Partitioned(outputs, IsSome) { - if exists o <- produced :: o.Failure? then + if exists o <- outputs :: o.Some? && o.value.Failure? then None else - Some(Flatten(MapPartialFunction((o: Result, E>) requires o.Success? => o.value, produced))) + forall o <- ProducedOf(outputs) ensures o.Success? { + AboutProducedOf(outputs, o); + } + Some(Flatten(MapPartialFunction(ValueOfSuccess, ProducedOf(outputs)))) + } + + lemma DataSoFarErrorIndex(outputs: seq, E>>>) returns (index: nat) + requires Partitioned(outputs, IsSome) + requires DataSoFar(outputs).None? + ensures index < |outputs| + ensures outputs[index].Some? && outputs[index].value.Failure? + { + assert 0 < |outputs|; + if outputs[0].Some? && outputs[0].value.Failure? { + return 0; + } else { + var index' := DataSoFarErrorIndex(outputs[1..]); + return index' + 1; + } + } + + + ghost function OptionConcat(left: Option>, right: Option>): Option> + { + if left.Some? && right.Some? then Some(left.value + right.value) else None + } + + lemma DataSoFarComposition(left: seq, E>>>, right: seq, E>>>) + requires Partitioned(left, IsSome) + requires Partitioned(right, IsSome) + requires Partitioned(left + right, IsSome) + ensures DataSoFar(left + right) == OptionConcat(DataSoFar(left), DataSoFar(right)) + { + if DataSoFar(left).None? { + var errorIndex := DataSoFarErrorIndex(left); + var error := left[errorIndex]; + assert (left + right)[errorIndex] == error; + assert DataSoFar(left + right) == None; + } else if DataSoFar(right).None? { + var errorIndex := DataSoFarErrorIndex(right); + var error := right[errorIndex]; + assert (left + right)[|left| + errorIndex] == error; + assert DataSoFar(left + right) == None; + } else { + ProducedComposition(left, right); + forall o <- ProducedOf(left) ensures o.Success? { + AboutProducedOf(left, o); + } + forall o <- ProducedOf(right) ensures o.Success? { + AboutProducedOf(right, o); + } + LemmaMapPartialFunctionDistributesOverConcat(ValueOfSuccess, ProducedOf(left), ProducedOf(right)); + LemmaFlattenConcat(MapPartialFunction(ValueOfSuccess, ProducedOf(left)), + MapPartialFunction(ValueOfSuccess, ProducedOf(right))); + } } - + + lemma ValidDataSoFarAfterNone(outputs: seq, E>>>, length: int) + requires Partitioned(outputs, IsSome) + requires + var dataSoFar := DataSoFar(outputs); + dataSoFar.Some? ==> |dataSoFar.value| == length + ensures Partitioned(outputs + [None], IsSome) + ensures ValidDataSoFar(outputs + [None], length) + { + var right: seq, E>>> := [None]; + PartitionedCompositionRight(outputs, [None], IsSome); + assert Partitioned(outputs + [None], IsSome); + var dataSoFarAfter := DataSoFar(outputs + [None]); + assert ProducedOf(right) == []; + DataSoFarComposition(outputs, right); + } + + lemma ValidDataSoFarAfterMoreData(outputs: seq, E>>>, value: seq, length: int) + requires All(outputs, IsSome) + requires + var dataSoFar := (AllImpliesPartitioned(outputs, IsSome); DataSoFar(outputs)); + dataSoFar.Some? ==> |dataSoFar.value| + |value| <= length + requires ValidDataSoFar(outputs, length) + ensures Partitioned(outputs + [Some(Success(value))], IsSome) + ensures ValidDataSoFar(outputs + [Some(Success(value))], length) + ensures DataSoFar(outputs).Some? ==> DataSoFar(outputs + [Some(Success(value))]).value == DataSoFar(outputs).value + value + { + AllImpliesPartitioned(outputs, IsSome); + var right: seq, E>>> := [Some(Success(value))]; + assert All(right, IsSome); + PartitionedCompositionLeft(outputs, right, IsSome); + if DataSoFar(outputs).None? { + assert DataSoFar(outputs + [Some(Success(value))]).None?; + } else { + assert ProducedOf(right) == [Success(value)]; + reveal Seq.Map(); + assert DataSoFar(right).value == Flatten([value]); + reveal Seq.Flatten(); + assert DataSoFar(right).value == value + Flatten([]); + assert DataSoFar(right).value == value; + DataSoFarComposition(outputs, right); + } + } + + trait RewindableDataStream extends DataStream { ghost const data: seq @@ -148,7 +258,7 @@ module {:options "--function-syntax:4"} Std.Streams { reads this, Repr decreases Repr, 3 { - TMTuple(maxWrappedRemaining, wrapped.RemainingMetric(), TMNat(|buffer|)) + TMTuple(TMTop, wrapped.RemainingMetric(), TMNat(|buffer|)) } constructor(wrapped: Producer>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) @@ -179,9 +289,15 @@ module {:options "--function-syntax:4"} Std.Streams { assert Requires(i); assert Valid(); - var result := wrapped.Next(); + var next; + if 0 < |buffer| { + next := Some(buffer); + buffer := []; + } else { + next := wrapped.Next(); + } - r := match result + r := match next case None => None case Some(value) => Some(Success(value)); UpdateHistory(i, r); @@ -265,6 +381,7 @@ module {:options "--function-syntax:4"} Std.Streams { && contentLength == Some(|s| as uint64) && position as int <= |s| && 0 < chunkSize + && DataSoFar(Outputs()) == Some(s[..position]) } ghost predicate ValidOutputs(outputs: seq, ()>>>) @@ -332,13 +449,13 @@ module {:options "--function-syntax:4"} Std.Streams { var outputs := Outputs(); var produced := ProducedOf(outputs); - var dataSoFar := DataSoFar(produced); + var dataSoFar := DataSoFar(outputs); if outputs == [] { assert produced == []; } else { - assert outputs == [Some(Success(s[..position]))]; + assert outputs == [Some(Success(s[..position]))] + []; assert produced == [Success(s[..position])] + ProducedOf([]); - assert MapPartialFunction((o: Result, ()>) requires o.Success? => o.value, produced) == [s[..position]]; + assert MapPartialFunction(ValueOfSuccess, produced) == [s[..position]]; assert Flatten([s[..position]]) == s[..position]; assert dataSoFar == Some(s[..position]); } @@ -371,10 +488,11 @@ module {:options "--function-syntax:4"} Std.Streams { if position == |s| as uint64 { r := None; - OutputsPartitionedAfterOutputtingNone(); + ValidDataSoFarAfterNone(Outputs(), contentLength.value as int); + assert OutputsOf(history + [((), None)]) == Outputs() + [None]; + assert ValidHistory(history + [((), None)]); ProduceNone(); } else { - // Warning: unbounded integers var remaining := |s| as uint64 - position; var size := if max <= remaining then max else remaining; var newPosition := position + size; @@ -382,11 +500,23 @@ module {:options "--function-syntax:4"} Std.Streams { r := Some(chunk); position := newPosition; - OutputsPartitionedAfterOutputtingSome(chunk); + ValidDataSoFarAfterMoreData(Outputs(), chunk.value, contentLength.value as int); + assert OutputsOf(history + [((), r)]) == Outputs() + [r]; + assert ValidHistory(history + [((), r)]); ProduceSome(chunk); + + calc { + DataSoFar(Outputs()).value; + old(DataSoFar(Outputs()).value) + chunk.value; + s[..old(position)] + chunk.value; + s[..old(position)] + s[old(position)..position]; + s[..position]; + } + assert DataSoFar(Outputs()) == Some(s[..position]); } reveal TerminationMetric.Ordinal(); + assert Valid(); } } } \ No newline at end of file From c619d8279dad9db9b19008489587e5ef819e2cfb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 9 Apr 2025 11:30:50 -0700 Subject: [PATCH 08/72] Progress --- .../StandardLibrary/src/Streams.dfy | 307 +++++++++++------- 1 file changed, 187 insertions(+), 120 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 2341f0cfa..1f9d479b5 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -12,6 +12,10 @@ module {:options "--function-syntax:4"} Std.Streams { import opened Collections.Seq import opened Termination + // TODO: Consider a more generic term for Streamed[Of], + // especially if we can decouple the filtering out Failures + // from the concatenation of batches. + // // A data stream, i.e. a fallable producer of batches of values. // @@ -41,9 +45,15 @@ module {:options "--function-syntax:4"} Std.Streams { ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) decreases Repr + ghost function Streamed(): seq + requires Valid() + { + StreamedOf(Outputs()) + } + method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) reads Reads(()) @@ -54,13 +64,18 @@ module {:options "--function-syntax:4"} Std.Streams { ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int } - ghost predicate ValidDataSoFar(outputs: seq, E>>>, length: int) + ghost function StreamedOf(outputs: seq, E>>>): seq requires Partitioned(outputs, IsSome) { - && var dataSoFar := DataSoFar(outputs); - && (dataSoFar.Some? ==> - && |dataSoFar.value| <= length - && (!Seq.All(outputs, IsSome) ==> |dataSoFar.value| == length)) + Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, ProducedOf(outputs)))) + } + + ghost predicate ValidStreamed(outputs: seq, E>>>, length: int) + requires Partitioned(outputs, IsSome) + { + && var streamed := StreamedOf(outputs); + && |streamed| <= length + && (!Seq.All(outputs, IsSome) ==> |streamed| == length) } // TODO: Move next to ProducedOf? @@ -69,62 +84,51 @@ module {:options "--function-syntax:4"} Std.Streams { ensures x in ProducedOf(outputs) <==> Some(x) in outputs {} - ghost function ValueOfSuccess(o: Result): T - requires o.Success? + lemma StreamedOfSingleton(s: seq) + ensures + var singleton: Option, E>> := Some(Success(s)); + StreamedOf([singleton]) == s { - o.value - } - - ghost function DataSoFar(outputs: seq, E>>>): Option> - requires Partitioned(outputs, IsSome) - { - if exists o <- outputs :: o.Some? && o.value.Failure? then - None - else - forall o <- ProducedOf(outputs) ensures o.Success? { - AboutProducedOf(outputs, o); - } - Some(Flatten(MapPartialFunction(ValueOfSuccess, ProducedOf(outputs)))) + var singleton: Option, E>> := Some(Success(s)); + calc { + StreamedOf([singleton]); + Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, ProducedOf([singleton])))); + Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, [singleton.value]))); + { reveal Filter(); } + Flatten(MapPartialFunction(ValueOfSuccess, [singleton.value])); + Flatten([s]); + s; + } } - lemma DataSoFarErrorIndex(outputs: seq, E>>>) returns (index: nat) - requires Partitioned(outputs, IsSome) - requires DataSoFar(outputs).None? - ensures index < |outputs| - ensures outputs[index].Some? && outputs[index].value.Failure? + ghost predicate IsSuccess(o: Result) { - assert 0 < |outputs|; - if outputs[0].Some? && outputs[0].value.Failure? { - return 0; - } else { - var index' := DataSoFarErrorIndex(outputs[1..]); - return index' + 1; - } + o.Success? } - - ghost function OptionConcat(left: Option>, right: Option>): Option> + ghost function ValueOfSuccess(o: Result): T + requires o.Success? { - if left.Some? && right.Some? then Some(left.value + right.value) else None + o.value } - lemma DataSoFarComposition(left: seq, E>>>, right: seq, E>>>) + lemma StreamedOfComposition(left: seq, E>>>, right: seq, E>>>) requires Partitioned(left, IsSome) requires Partitioned(right, IsSome) requires Partitioned(left + right, IsSome) - ensures DataSoFar(left + right) == OptionConcat(DataSoFar(left), DataSoFar(right)) + ensures StreamedOf(left + right) == StreamedOf(left) + StreamedOf(right) { - if DataSoFar(left).None? { - var errorIndex := DataSoFarErrorIndex(left); - var error := left[errorIndex]; - assert (left + right)[errorIndex] == error; - assert DataSoFar(left + right) == None; - } else if DataSoFar(right).None? { - var errorIndex := DataSoFarErrorIndex(right); - var error := right[errorIndex]; - assert (left + right)[|left| + errorIndex] == error; - assert DataSoFar(left + right) == None; - } else { + // if StreamedOf(left).None? { + // var errorIndex := StreamedOfErrorIndex(left); + // var error := left[errorIndex]; + // assert (left + right)[errorIndex] == error; + // assert StreamedOf(left + right) == None; + // } else if StreamedOf(right).None? { + // var errorIndex := StreamedOfErrorIndex(right); + // var error := right[errorIndex]; + // assert (left + right)[|left| + errorIndex] == error; + // assert StreamedOf(left + right) == None; + // } else { ProducedComposition(left, right); forall o <- ProducedOf(left) ensures o.Success? { AboutProducedOf(left, o); @@ -135,50 +139,44 @@ module {:options "--function-syntax:4"} Std.Streams { LemmaMapPartialFunctionDistributesOverConcat(ValueOfSuccess, ProducedOf(left), ProducedOf(right)); LemmaFlattenConcat(MapPartialFunction(ValueOfSuccess, ProducedOf(left)), MapPartialFunction(ValueOfSuccess, ProducedOf(right))); - } + // } } - lemma ValidDataSoFarAfterNone(outputs: seq, E>>>, length: int) + lemma ValidStreamedAfterNone(outputs: seq, E>>>, length: int) requires Partitioned(outputs, IsSome) - requires - var dataSoFar := DataSoFar(outputs); - dataSoFar.Some? ==> |dataSoFar.value| == length + requires |StreamedOf(outputs)| == length ensures Partitioned(outputs + [None], IsSome) - ensures ValidDataSoFar(outputs + [None], length) + ensures ValidStreamed(outputs + [None], length) { var right: seq, E>>> := [None]; PartitionedCompositionRight(outputs, [None], IsSome); assert Partitioned(outputs + [None], IsSome); - var dataSoFarAfter := DataSoFar(outputs + [None]); + var streamedAfter := StreamedOf(outputs + [None]); assert ProducedOf(right) == []; - DataSoFarComposition(outputs, right); + StreamedOfComposition(outputs, right); } - lemma ValidDataSoFarAfterMoreData(outputs: seq, E>>>, value: seq, length: int) + lemma ValidStreamedAfterMoreData(outputs: seq, E>>>, value: seq, length: int) requires All(outputs, IsSome) requires - var dataSoFar := (AllImpliesPartitioned(outputs, IsSome); DataSoFar(outputs)); - dataSoFar.Some? ==> |dataSoFar.value| + |value| <= length - requires ValidDataSoFar(outputs, length) + (AllImpliesPartitioned(outputs, IsSome); + |StreamedOf(outputs)| + |value| <= length) + requires ValidStreamed(outputs, length) ensures Partitioned(outputs + [Some(Success(value))], IsSome) - ensures ValidDataSoFar(outputs + [Some(Success(value))], length) - ensures DataSoFar(outputs).Some? ==> DataSoFar(outputs + [Some(Success(value))]).value == DataSoFar(outputs).value + value + ensures ValidStreamed(outputs + [Some(Success(value))], length) + ensures StreamedOf(outputs + [Some(Success(value))]) == StreamedOf(outputs) + value { AllImpliesPartitioned(outputs, IsSome); var right: seq, E>>> := [Some(Success(value))]; assert All(right, IsSome); PartitionedCompositionLeft(outputs, right, IsSome); - if DataSoFar(outputs).None? { - assert DataSoFar(outputs + [Some(Success(value))]).None?; - } else { - assert ProducedOf(right) == [Success(value)]; - reveal Seq.Map(); - assert DataSoFar(right).value == Flatten([value]); - reveal Seq.Flatten(); - assert DataSoFar(right).value == value + Flatten([]); - assert DataSoFar(right).value == value; - DataSoFarComposition(outputs, right); - } + assert ProducedOf(right) == [Success(value)]; + reveal Seq.Map(); + assert StreamedOf(right) == Flatten([value]); + reveal Seq.Flatten(); + assert StreamedOf(right) == value + Flatten([]); + assert StreamedOf(right) == value; + StreamedOfComposition(outputs, right); } @@ -199,36 +197,27 @@ module {:options "--function-syntax:4"} Std.Streams { ensures Position() == newPosition } - ghost predicate ValidDataLengthSoFar(outputs: seq>>, length: int) - requires Partitioned(outputs, IsSome) - { - var dataSoFar := Flatten(ProducedOf(outputs)); - && |dataSoFar| <= length - && (!Seq.All(outputs, IsSome) ==> |dataSoFar| == length) - } - - trait ProducesTotalLengthProof { + trait ProducesTotalLengthProof { - const producer: Producer> + const producer: Producer, E>> const length: int - lemma ProducesTotalLength(history: seq<((), Option>)>) + lemma ProducesTotalLength(history: seq<((), Option, E>>)>) requires producer.ValidHistory(history) - ensures ValidDataLengthSoFar(OutputsOf(history), length) + ensures ValidStreamed(OutputsOf(history), length) } /* - * Wraps an Producer up as a non-rewindable DataStream that cannot error. - * It implements Read() using Next(), and buffers extra data - * as needed. + * Wraps an Producer up as a non-rewindable DataStream. + * It implements Read() using Next(), and buffers extra data as needed. */ - class ProducerDataStream extends DataStream { + class ProducerDataStream extends DataStream { - const wrapped: Producer> + const wrapped: Producer, E>> const length: uint64 var buffer: seq - ghost const producesTotalLengthProof: ProducesTotalLengthProof + ghost const producesTotalLengthProof: ProducesTotalLengthProof ghost const maxWrappedRemaining: TerminationMetric ghost predicate Valid() @@ -239,18 +228,23 @@ module {:options "--function-syntax:4"} Std.Streams { { && this in Repr && ValidComponent(wrapped) + && maxWrappedRemaining.NonIncreasesTo(wrapped.RemainingMetric()) && ValidHistory(history) && producesTotalLengthProof.producer == wrapped && producesTotalLengthProof.length == length as int + && contentLength == Some(length) && |buffer| <= length as int + && StreamedOf(Outputs()) + buffer == StreamedOf(wrapped.Outputs()) + && (0 < |buffer| ==> !wrapped.Done()) + && (!Done() <==> !wrapped.Done()) } - ghost predicate ValidOutputs(outputs: seq, ()>>>) + ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) decreases Repr { - contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) + ValidStreamed(outputs, contentLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -261,7 +255,7 @@ module {:options "--function-syntax:4"} Std.Streams { TMTuple(TMTop, wrapped.RemainingMetric(), TMNat(|buffer|)) } - constructor(wrapped: Producer>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) + constructor(wrapped: Producer, E>>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) requires wrapped.Valid() requires wrapped.history == [] requires producesTotalLengthProof.producer == wrapped @@ -273,12 +267,13 @@ module {:options "--function-syntax:4"} Std.Streams { this.length := length; this.buffer := []; + this.contentLength := Some(length); this.history := []; this.Repr := {this} + wrapped.Repr; this.producesTotalLengthProof := producesTotalLengthProof; } - method Invoke(i: ()) returns (r: Option, ()>>) + method Invoke(i: ()) returns (r: Option, E>>) requires Requires(i) reads Reads(i) modifies Modifies(i) @@ -289,17 +284,13 @@ module {:options "--function-syntax:4"} Std.Streams { assert Requires(i); assert Valid(); - var next; + var next: Option, E>>; if 0 < |buffer| { - next := Some(buffer); + r := Some(Success(buffer)); buffer := []; } else { - next := wrapped.Next(); + r := wrapped.Next(); } - - r := match next - case None => None - case Some(value) => Some(Success(value)); UpdateHistory(i, r); // TODO: work to do @@ -311,7 +302,7 @@ module {:options "--function-syntax:4"} Std.Streams { } } - method Read(max: uint64) returns (r: Option, ()>>) + method {:only} Read(max: uint64) returns (r: Option, E>>) requires Requires(()) reads Reads(()) modifies Modifies(()) @@ -325,29 +316,104 @@ module {:options "--function-syntax:4"} Std.Streams { assert Valid(); var next; if 0 < |buffer| { - next := Some(buffer); + assert !wrapped.Done(); + next := Some(Success(buffer)); + StreamedOfSingleton(buffer); buffer := []; + + assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); } else { next := wrapped.Next(); + producesTotalLengthProof.ProducesTotalLength(wrapped.history); + + if next.Some? { + assert !wrapped.Done(); + } else { + assert !IsSome(Last(wrapped.Outputs())); + assert !Seq.All(wrapped.Outputs(), IsSome); + // assert |StreamedOf(wrapped.Outputs())| == length as int; + } + + assert wrapped.Outputs() == old(wrapped.Outputs()) + [next]; + StreamedOfComposition(old(wrapped.Outputs()), [next]); + assert StreamedOf(wrapped.Outputs()) == StreamedOf(old(wrapped.Outputs())) + StreamedOf([next]); + assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); } - assert next.Some? ==> |next.value| <= length as int; + assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); + producesTotalLengthProof.ProducesTotalLength(wrapped.history); + assert |StreamedOf(wrapped.Outputs())| <= length as int; if next.None? { r := None; + ghost var right := [next]; + assert Last(wrapped.Outputs()) == next; + assert wrapped.Outputs() == old(wrapped.Outputs()) + right; + assert StreamedOf(wrapped.Outputs()) == StreamedOf(old(wrapped.Outputs()) + right); + assert StreamedOf(right) == []; + StreamedOfComposition(old(wrapped.Outputs()), right); + assert old(|StreamedOf(wrapped.Outputs())|) == old(|StreamedOf(Outputs())|); + assert StreamedOf(wrapped.Outputs()) == old(StreamedOf(wrapped.Outputs())) + []; + assert old(|StreamedOf(wrapped.Outputs())|) == |StreamedOf(wrapped.Outputs())|; + assert |StreamedOf(wrapped.Outputs())| == length as int; + assert |StreamedOf(Outputs())| == length as int; + ValidStreamedAfterNone(Outputs(), length as int); OutputsPartitionedAfterOutputtingNone(); + + assert ValidHistory(history + [((), None)]); ProduceNone(); + + assert Valid(); + } else if next.value.Failure? { + r := next; + + assert Valid(); } else { - var size := if max <= |next.value| as uint64 then max else |next.value| as uint64; - var value := Success(next.value[..size]); + assert !wrapped.Done(); + wrapped.DoneIsOneWay(); + assert old(!wrapped.Done()); + assert old(!Done()); + assert !Done(); + assert All(Outputs(), IsSome); + + var size := if max <= |next.value.value| as uint64 then max else |next.value.value| as uint64; + var result := next.value.value[..size]; + var value := Success(result); r := Some(value); - buffer := next.value[size..]; + buffer := next.value.value[size..]; + + assert next.value.value == next.value.value[..size] + next.value.value[size..]; + StreamedOfSingleton(r.value.value); + StreamedOfSingleton(next.value.value); + assert StreamedOf([r]) + buffer == StreamedOf([next]); + + assert StreamedOf(Outputs()) + StreamedOf([r]) + buffer == StreamedOf(wrapped.Outputs()); + + assert old(Valid()); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); OutputsPartitionedAfterOutputtingSome(value); + PartitionedCompositionLeft(Outputs(), [r], IsSome); + StreamedOfComposition(Outputs(), [r]); + assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); + + ghost var wrappedStreamed := StreamedOf(wrapped.Outputs()); + assert |wrappedStreamed| <= length as int; + assert |StreamedOf(wrapped.Outputs())| <= length as int; + ValidStreamedAfterMoreData(Outputs(), result, length as int); + + assert ValidOutputs(Outputs() + [Some(value)]); + assert OutputsOf(history + [((), Some(value))]) == Outputs() + [Some(value)]; + assert ValidHistory(history + [((), Some(value))]); ProduceSome(value); + + assert Valid(); } + assert Valid(); if 0 < max { + reveal TerminationMetric.Ordinal(); if r.Some? { old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); } else { @@ -355,6 +421,7 @@ module {:options "--function-syntax:4"} Std.Streams { } } assert 0 < max ==> RemainingDecreasedBy(r); + } } @@ -381,16 +448,16 @@ module {:options "--function-syntax:4"} Std.Streams { && contentLength == Some(|s| as uint64) && position as int <= |s| && 0 < chunkSize - && DataSoFar(Outputs()) == Some(s[..position]) + && StreamedOf(Outputs()) == s[..position] } ghost predicate ValidOutputs(outputs: seq, ()>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidDataSoFar(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) decreases Repr { && contentLength.Some? - && ValidDataSoFar(outputs, contentLength.value as int) + && ValidStreamed(outputs, contentLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -449,7 +516,7 @@ module {:options "--function-syntax:4"} Std.Streams { var outputs := Outputs(); var produced := ProducedOf(outputs); - var dataSoFar := DataSoFar(outputs); + var streamed := StreamedOf(outputs); if outputs == [] { assert produced == []; } else { @@ -457,7 +524,7 @@ module {:options "--function-syntax:4"} Std.Streams { assert produced == [Success(s[..position])] + ProducedOf([]); assert MapPartialFunction(ValueOfSuccess, produced) == [s[..position]]; assert Flatten([s[..position]]) == s[..position]; - assert dataSoFar == Some(s[..position]); + assert streamed == s[..position]; } } @@ -488,7 +555,7 @@ module {:options "--function-syntax:4"} Std.Streams { if position == |s| as uint64 { r := None; - ValidDataSoFarAfterNone(Outputs(), contentLength.value as int); + ValidStreamedAfterNone(Outputs(), contentLength.value as int); assert OutputsOf(history + [((), None)]) == Outputs() + [None]; assert ValidHistory(history + [((), None)]); ProduceNone(); @@ -500,19 +567,19 @@ module {:options "--function-syntax:4"} Std.Streams { r := Some(chunk); position := newPosition; - ValidDataSoFarAfterMoreData(Outputs(), chunk.value, contentLength.value as int); + ValidStreamedAfterMoreData(Outputs(), chunk.value, contentLength.value as int); assert OutputsOf(history + [((), r)]) == Outputs() + [r]; assert ValidHistory(history + [((), r)]); ProduceSome(chunk); calc { - DataSoFar(Outputs()).value; - old(DataSoFar(Outputs()).value) + chunk.value; + StreamedOf(Outputs()); + old(StreamedOf(Outputs())) + chunk.value; s[..old(position)] + chunk.value; s[..old(position)] + s[old(position)..position]; s[..position]; } - assert DataSoFar(Outputs()) == Some(s[..position]); + assert StreamedOf(Outputs()) == s[..position]; } reveal TerminationMetric.Ordinal(); From de7408e7c095fd03dbc57e839f4172aacc69d27c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 9 Apr 2025 16:19:20 -0700 Subject: [PATCH 09/72] Fixed StreamedOf() definition --- .../StandardLibrary/src/Streams.dfy | 159 +++++++++--------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 1f9d479b5..1bfea7ba9 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -50,6 +50,7 @@ module {:options "--function-syntax:4"} Std.Streams { ghost function Streamed(): seq requires Valid() + reads this, Repr { StreamedOf(Outputs()) } @@ -65,9 +66,20 @@ module {:options "--function-syntax:4"} Std.Streams { } ghost function StreamedOf(outputs: seq, E>>>): seq - requires Partitioned(outputs, IsSome) { - Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, ProducedOf(outputs)))) + // This could be defined as something like this instead: + // + // Flatten(MapPartialFunction(x => x.value, Filter(x => x.Success?, ProducedOf(outputs)))) + // + // But then the lack of extensionality in Dafny hits hard, + // and it becomes impossible to expand the definition elsewhere. + + if |outputs| == 0 then + [] + else if outputs[0].Some? && outputs[0].value.Success? then + outputs[0].value.value + StreamedOf(outputs[1..]) + else + StreamedOf(outputs[1..]) } ghost predicate ValidStreamed(outputs: seq, E>>>, length: int) @@ -88,59 +100,29 @@ module {:options "--function-syntax:4"} Std.Streams { ensures var singleton: Option, E>> := Some(Success(s)); StreamedOf([singleton]) == s - { - var singleton: Option, E>> := Some(Success(s)); - calc { - StreamedOf([singleton]); - Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, ProducedOf([singleton])))); - Flatten(MapPartialFunction(ValueOfSuccess, Filter(IsSuccess, [singleton.value]))); - { reveal Filter(); } - Flatten(MapPartialFunction(ValueOfSuccess, [singleton.value])); - Flatten([s]); - s; - } - } - - ghost predicate IsSuccess(o: Result) - { - o.Success? - } + {} - ghost function ValueOfSuccess(o: Result): T - requires o.Success? + // TODO: Correcting a type in the original ensures clause, should be fixed at the source + lemma LemmaFilterImpliesAll(f: (T ~> bool), xs: seq) + requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) + ensures + var result := Filter(f, xs); + forall i: nat :: i < |result| ==> f.requires(result[i]) && f(result[i]) { - o.value + reveal Filter(); } lemma StreamedOfComposition(left: seq, E>>>, right: seq, E>>>) - requires Partitioned(left, IsSome) - requires Partitioned(right, IsSome) - requires Partitioned(left + right, IsSome) ensures StreamedOf(left + right) == StreamedOf(left) + StreamedOf(right) { - // if StreamedOf(left).None? { - // var errorIndex := StreamedOfErrorIndex(left); - // var error := left[errorIndex]; - // assert (left + right)[errorIndex] == error; - // assert StreamedOf(left + right) == None; - // } else if StreamedOf(right).None? { - // var errorIndex := StreamedOfErrorIndex(right); - // var error := right[errorIndex]; - // assert (left + right)[|left| + errorIndex] == error; - // assert StreamedOf(left + right) == None; - // } else { - ProducedComposition(left, right); - forall o <- ProducedOf(left) ensures o.Success? { - AboutProducedOf(left, o); - } - forall o <- ProducedOf(right) ensures o.Success? { - AboutProducedOf(right, o); - } - LemmaMapPartialFunctionDistributesOverConcat(ValueOfSuccess, ProducedOf(left), ProducedOf(right)); - LemmaFlattenConcat(MapPartialFunction(ValueOfSuccess, ProducedOf(left)), - MapPartialFunction(ValueOfSuccess, ProducedOf(right))); - // } - } + if left == [] { + assert left + right == right; + } else { + StreamedOfComposition(left[1..], right); + assert left == [left[0]] + left[1..]; + assert left + right == [left[0]] + (left[1..] + right); + } + } lemma ValidStreamedAfterNone(outputs: seq, E>>>, length: int) requires Partitioned(outputs, IsSome) @@ -236,7 +218,7 @@ module {:options "--function-syntax:4"} Std.Streams { && |buffer| <= length as int && StreamedOf(Outputs()) + buffer == StreamedOf(wrapped.Outputs()) && (0 < |buffer| ==> !wrapped.Done()) - && (!Done() <==> !wrapped.Done()) + && (!Done() ==> !wrapped.Done()) } ghost predicate ValidOutputs(outputs: seq, E>>>) @@ -244,7 +226,7 @@ module {:options "--function-syntax:4"} Std.Streams { ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) decreases Repr { - ValidStreamed(outputs, contentLength.value as int) + contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -255,6 +237,14 @@ module {:options "--function-syntax:4"} Std.Streams { TMTuple(TMTop, wrapped.RemainingMetric(), TMNat(|buffer|)) } + twostate lemma RemainingMetricDoesntReadHistory() + requires old(Valid()) + requires Valid() + requires wrapped.RemainingMetric() == old(wrapped.RemainingMetric()); + requires buffer == old(buffer); + ensures RemainingMetric() == old(RemainingMetric()) + {} + constructor(wrapped: Producer, E>>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) requires wrapped.Valid() requires wrapped.history == [] @@ -271,38 +261,44 @@ module {:options "--function-syntax:4"} Std.Streams { this.history := []; this.Repr := {this} + wrapped.Repr; this.producesTotalLengthProof := producesTotalLengthProof; + this.maxWrappedRemaining := wrapped.RemainingMetric(); } method Invoke(i: ()) returns (r: Option, E>>) requires Requires(i) - reads Reads(i) + reads this, Repr modifies Modifies(i) decreases Decreases(i), 0 ensures Ensures(i, r) ensures RemainingDecreasedBy(r) { - assert Requires(i); - + assert Requires(()); assert Valid(); - var next: Option, E>>; - if 0 < |buffer| { - r := Some(Success(buffer)); - buffer := []; - } else { - r := wrapped.Next(); - } - UpdateHistory(i, r); + if length == 0 { + r := None; + + producesTotalLengthProof.ProducesTotalLength(wrapped.history); + assert |StreamedOf(wrapped.Outputs())| == 0; + PartitionedCompositionRight(Outputs(), [None], IsSome); + StreamedOfComposition(Outputs(), [None]); + assert OutputsOf(history + [((), None)]) == Outputs() + [None]; + + OutputsPartitionedAfterOutputtingNone(); + ProduceNone(); - // TODO: work to do - assume {:axiom} Valid(); - if r.Some? { - old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); + assert Last(Outputs()) == None; + assert Done(); + assert Valid(); + + RemainingMetricDoesntReadHistory(); + assert RemainingDecreasedBy(r); } else { - old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); + r := Read(length); + assert RemainingDecreasedBy(r); } } - method {:only} Read(max: uint64) returns (r: Option, E>>) + method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) reads Reads(()) modifies Modifies(()) @@ -324,15 +320,16 @@ module {:options "--function-syntax:4"} Std.Streams { assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); } else { next := wrapped.Next(); + Repr := {this} + wrapped.Repr; producesTotalLengthProof.ProducesTotalLength(wrapped.history); - if next.Some? { - assert !wrapped.Done(); - } else { - assert !IsSome(Last(wrapped.Outputs())); - assert !Seq.All(wrapped.Outputs(), IsSome); - // assert |StreamedOf(wrapped.Outputs())| == length as int; - } + // if next.Some? { + // assert !wrapped.Done(); + // } else { + // assert !IsSome(Last(wrapped.Outputs())); + // assert !Seq.All(wrapped.Outputs(), IsSome); + // // assert |StreamedOf(wrapped.Outputs())| == length as int; + // } assert wrapped.Outputs() == old(wrapped.Outputs()) + [next]; StreamedOfComposition(old(wrapped.Outputs()), [next]); @@ -363,10 +360,14 @@ module {:options "--function-syntax:4"} Std.Streams { assert ValidHistory(history + [((), None)]); ProduceNone(); + assert Last(wrapped.Outputs()) == None; + assert Last(Outputs()) == None; assert Valid(); } else if next.value.Failure? { r := next; + assert Last(wrapped.Outputs()).Some?; + assert Last(Outputs()).Some?; assert Valid(); } else { assert !wrapped.Done(); @@ -520,11 +521,7 @@ module {:options "--function-syntax:4"} Std.Streams { if outputs == [] { assert produced == []; } else { - assert outputs == [Some(Success(s[..position]))] + []; - assert produced == [Success(s[..position])] + ProducedOf([]); - assert MapPartialFunction(ValueOfSuccess, produced) == [s[..position]]; - assert Flatten([s[..position]]) == s[..position]; - assert streamed == s[..position]; + StreamedOfSingleton(s[..position]); } } @@ -559,6 +556,10 @@ module {:options "--function-syntax:4"} Std.Streams { assert OutputsOf(history + [((), None)]) == Outputs() + [None]; assert ValidHistory(history + [((), None)]); ProduceNone(); + + assert StreamedOf([r]) == []; + StreamedOfComposition(old(Outputs()), [r]); + assert StreamedOf(Outputs()) == s[..position]; } else { var remaining := |s| as uint64 - position; var size := if max <= remaining then max else remaining; From f1699f36a319fdf9c53dc0e5243878acc4847016 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 9 Apr 2025 16:52:35 -0700 Subject: [PATCH 10/72] All but one wrinkle when length == 0 --- .../StandardLibrary/src/Streams.dfy | 72 ++++++++++++++++--- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 1bfea7ba9..bee2fe7d0 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -161,6 +161,26 @@ module {:options "--function-syntax:4"} Std.Streams { StreamedOfComposition(outputs, right); } + lemma ValidStreamedAfterFailure(outputs: seq, E>>>, error: E, length: int) + requires All(outputs, IsSome) + requires + (AllImpliesPartitioned(outputs, IsSome); + |StreamedOf(outputs)| <= length) + requires ValidStreamed(outputs, length) + ensures Partitioned(outputs + [Some(Failure(error))], IsSome) + ensures ValidStreamed(outputs + [Some(Failure(error))], length) + ensures StreamedOf(outputs + [Some(Failure(error))]) == StreamedOf(outputs) + { + AllImpliesPartitioned(outputs, IsSome); + var right: seq, E>>> := [Some(Failure(error))]; + assert All(right, IsSome); + PartitionedCompositionLeft(outputs, right, IsSome); + assert ProducedOf(right) == [Failure(error)]; + reveal Seq.Map(); + assert StreamedOf(right) == []; + StreamedOfComposition(outputs, right); + } + trait RewindableDataStream extends DataStream { @@ -218,7 +238,7 @@ module {:options "--function-syntax:4"} Std.Streams { && |buffer| <= length as int && StreamedOf(Outputs()) + buffer == StreamedOf(wrapped.Outputs()) && (0 < |buffer| ==> !wrapped.Done()) - && (!Done() ==> !wrapped.Done()) + && (!Done() <==> !wrapped.Done()) } ghost predicate ValidOutputs(outputs: seq, E>>>) @@ -298,6 +318,7 @@ module {:options "--function-syntax:4"} Std.Streams { } } + @IsolateAssertions method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) reads Reads(()) @@ -323,14 +344,6 @@ module {:options "--function-syntax:4"} Std.Streams { Repr := {this} + wrapped.Repr; producesTotalLengthProof.ProducesTotalLength(wrapped.history); - // if next.Some? { - // assert !wrapped.Done(); - // } else { - // assert !IsSome(Last(wrapped.Outputs())); - // assert !Seq.All(wrapped.Outputs(), IsSome); - // // assert |StreamedOf(wrapped.Outputs())| == length as int; - // } - assert wrapped.Outputs() == old(wrapped.Outputs()) + [next]; StreamedOfComposition(old(wrapped.Outputs()), [next]); assert StreamedOf(wrapped.Outputs()) == StreamedOf(old(wrapped.Outputs())) + StreamedOf([next]); @@ -357,19 +370,58 @@ module {:options "--function-syntax:4"} Std.Streams { ValidStreamedAfterNone(Outputs(), length as int); OutputsPartitionedAfterOutputtingNone(); + assert OutputsOf(history + [((), None)]) == Outputs() + [None]; assert ValidHistory(history + [((), None)]); ProduceNone(); assert Last(wrapped.Outputs()) == None; assert Last(Outputs()) == None; + + assert StreamedOf([r]) == []; + StreamedOfComposition(old(Outputs()), [r]); + assert Valid(); } else if next.value.Failure? { r := next; + assert Last(wrapped.Outputs()).Some?; + PartitionedLastTrueImpliesAll(wrapped.Outputs(), IsSome); + assert !wrapped.Done(); + wrapped.DoneIsOneWay(); + assert old(!wrapped.Done()); + assert old(!Done()); + assert !Done(); + assert All(Outputs(), IsSome); + + assert old(Valid()); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); + OutputsPartitionedAfterOutputtingSome(r.value); + PartitionedCompositionLeft(Outputs(), [r], IsSome); + StreamedOfComposition(Outputs(), [r]); + assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); + + ghost var wrappedStreamed := StreamedOf(wrapped.Outputs()); + assert |wrappedStreamed| <= length as int; + assert |StreamedOf(wrapped.Outputs())| <= length as int; + ValidStreamedAfterFailure(Outputs(), next.value.error, length as int); + + assert ValidOutputs(Outputs() + [r]); + assert OutputsOf(history + [((), r)]) == Outputs() + [r]; + assert ValidHistory(history + [((), r)]); + ProduceSome(r.value); + assert Last(wrapped.Outputs()).Some?; assert Last(Outputs()).Some?; + + assert StreamedOf([r]) == []; + StreamedOfComposition(old(Outputs()), [r]); + assert Valid(); } else { + assert Last(wrapped.Outputs()).Some?; + PartitionedLastTrueImpliesAll(wrapped.Outputs(), IsSome); + assert !wrapped.Done(); wrapped.DoneIsOneWay(); assert old(!wrapped.Done()); @@ -500,7 +552,6 @@ module {:options "--function-syntax:4"} Std.Streams { position } - @ResourceLimit("0") method Seek(newPosition: uint64) requires Valid() requires newPosition as int <= |data| @@ -521,6 +572,7 @@ module {:options "--function-syntax:4"} Std.Streams { if outputs == [] { assert produced == []; } else { + assert outputs == [Some(Success(s[..position]))]; StreamedOfSingleton(s[..position]); } } From 89ac6a77982f958c1b56f099c625526fc0ad29ce Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 9 Apr 2025 21:57:37 -0700 Subject: [PATCH 11/72] Almost --- .../StandardLibrary/src/Streams.dfy | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index bee2fe7d0..eab270ead 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -181,6 +181,24 @@ module {:options "--function-syntax:4"} Std.Streams { StreamedOfComposition(outputs, right); } + method Drain(p: Producer) + requires p.Valid() + reads p.Repr + modifies p.Repr + ensures p.ValidAndDisjoint() + ensures p.Done() + ensures old(p.RemainingMetric()).NonIncreasesTo(p.RemainingMetric()) + { + while true + invariant fresh(p.Repr - old(p.Repr)) + invariant p.ValidAndDisjoint() + decreases p.Remaining() + { + var n := p.Next(); + if n.None? { break; } + } + assert Last(p.Outputs()) == None; + } trait RewindableDataStream extends DataStream { @@ -284,7 +302,7 @@ module {:options "--function-syntax:4"} Std.Streams { this.maxWrappedRemaining := wrapped.RemainingMetric(); } - method Invoke(i: ()) returns (r: Option, E>>) + method {:only} Invoke(i: ()) returns (r: Option, E>>) requires Requires(i) reads this, Repr modifies Modifies(i) @@ -305,7 +323,12 @@ module {:options "--function-syntax:4"} Std.Streams { OutputsPartitionedAfterOutputtingNone(); ProduceNone(); + label before: + // assert Valid(); + Drain(wrapped); + Repr := {this} + wrapped.Repr; + producesTotalLengthProof.ProducesTotalLength(wrapped.history); assert Last(Outputs()) == None; assert Done(); assert Valid(); @@ -381,11 +404,11 @@ module {:options "--function-syntax:4"} Std.Streams { StreamedOfComposition(old(Outputs()), [r]); assert Valid(); - } else if next.value.Failure? { - r := next; + } else { assert Last(wrapped.Outputs()).Some?; PartitionedLastTrueImpliesAll(wrapped.Outputs(), IsSome); + assert !wrapped.Done(); wrapped.DoneIsOneWay(); assert old(!wrapped.Done()); @@ -393,18 +416,42 @@ module {:options "--function-syntax:4"} Std.Streams { assert !Done(); assert All(Outputs(), IsSome); - assert old(Valid()); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - OutputsPartitionedAfterOutputtingSome(r.value); - PartitionedCompositionLeft(Outputs(), [r], IsSome); - StreamedOfComposition(Outputs(), [r]); - assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); + if next.value.Failure? { + r := next; + + assert old(Valid()); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); + OutputsPartitionedAfterOutputtingSome(r.value); + PartitionedCompositionLeft(Outputs(), [r], IsSome); + StreamedOfComposition(Outputs(), [r]); + assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); - ghost var wrappedStreamed := StreamedOf(wrapped.Outputs()); - assert |wrappedStreamed| <= length as int; - assert |StreamedOf(wrapped.Outputs())| <= length as int; - ValidStreamedAfterFailure(Outputs(), next.value.error, length as int); + assert |StreamedOf(wrapped.Outputs())| <= length as int; + ValidStreamedAfterFailure(Outputs(), next.value.error, length as int); + } else { + var size := if max <= |next.value.value| as uint64 then max else |next.value.value| as uint64; + var result := next.value.value[..size]; + var value := Success(result); + r := Some(value); + buffer := next.value.value[size..]; + + assert next.value.value == next.value.value[..size] + next.value.value[size..]; + StreamedOfSingleton(r.value.value); + StreamedOfSingleton(next.value.value); + assert StreamedOf([r]) + buffer == StreamedOf([next]); + + assert StreamedOf(Outputs()) + StreamedOf([r]) + buffer == StreamedOf(wrapped.Outputs()); + + assert old(Valid()); + assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); + OutputsPartitionedAfterOutputtingSome(value); + PartitionedCompositionLeft(Outputs(), [r], IsSome); + StreamedOfComposition(Outputs(), [r]); + assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); + + assert |StreamedOf(wrapped.Outputs())| <= length as int; + ValidStreamedAfterMoreData(Outputs(), result, length as int); + } assert ValidOutputs(Outputs() + [r]); assert OutputsOf(history + [((), r)]) == Outputs() + [r]; @@ -414,53 +461,6 @@ module {:options "--function-syntax:4"} Std.Streams { assert Last(wrapped.Outputs()).Some?; assert Last(Outputs()).Some?; - assert StreamedOf([r]) == []; - StreamedOfComposition(old(Outputs()), [r]); - - assert Valid(); - } else { - assert Last(wrapped.Outputs()).Some?; - PartitionedLastTrueImpliesAll(wrapped.Outputs(), IsSome); - - assert !wrapped.Done(); - wrapped.DoneIsOneWay(); - assert old(!wrapped.Done()); - assert old(!Done()); - assert !Done(); - assert All(Outputs(), IsSome); - - var size := if max <= |next.value.value| as uint64 then max else |next.value.value| as uint64; - var result := next.value.value[..size]; - var value := Success(result); - r := Some(value); - buffer := next.value.value[size..]; - - assert next.value.value == next.value.value[..size] + next.value.value[size..]; - StreamedOfSingleton(r.value.value); - StreamedOfSingleton(next.value.value); - assert StreamedOf([r]) + buffer == StreamedOf([next]); - - assert StreamedOf(Outputs()) + StreamedOf([r]) + buffer == StreamedOf(wrapped.Outputs()); - - - assert old(Valid()); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - OutputsPartitionedAfterOutputtingSome(value); - PartitionedCompositionLeft(Outputs(), [r], IsSome); - StreamedOfComposition(Outputs(), [r]); - assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); - - ghost var wrappedStreamed := StreamedOf(wrapped.Outputs()); - assert |wrappedStreamed| <= length as int; - assert |StreamedOf(wrapped.Outputs())| <= length as int; - ValidStreamedAfterMoreData(Outputs(), result, length as int); - - assert ValidOutputs(Outputs() + [Some(value)]); - assert OutputsOf(history + [((), Some(value))]) == Outputs() + [Some(value)]; - assert ValidHistory(history + [((), Some(value))]); - ProduceSome(value); - assert Valid(); } From d07ab0f57f41a385860deec105e9e6a9ea5f6472 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 10 Apr 2025 08:07:24 -0700 Subject: [PATCH 12/72] Fixed --- .../StandardLibrary/src/Streams.dfy | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index eab270ead..d2398a3e8 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -238,7 +238,6 @@ module {:options "--function-syntax:4"} Std.Streams { var buffer: seq ghost const producesTotalLengthProof: ProducesTotalLengthProof - ghost const maxWrappedRemaining: TerminationMetric ghost predicate Valid() reads this, Repr @@ -248,7 +247,6 @@ module {:options "--function-syntax:4"} Std.Streams { { && this in Repr && ValidComponent(wrapped) - && maxWrappedRemaining.NonIncreasesTo(wrapped.RemainingMetric()) && ValidHistory(history) && producesTotalLengthProof.producer == wrapped && producesTotalLengthProof.length == length as int @@ -278,7 +276,7 @@ module {:options "--function-syntax:4"} Std.Streams { twostate lemma RemainingMetricDoesntReadHistory() requires old(Valid()) requires Valid() - requires wrapped.RemainingMetric() == old(wrapped.RemainingMetric()); + requires wrapped.RemainingMetric() == old(wrapped.RemainingMetric()) requires buffer == old(buffer); ensures RemainingMetric() == old(RemainingMetric()) {} @@ -299,10 +297,9 @@ module {:options "--function-syntax:4"} Std.Streams { this.history := []; this.Repr := {this} + wrapped.Repr; this.producesTotalLengthProof := producesTotalLengthProof; - this.maxWrappedRemaining := wrapped.RemainingMetric(); } - method {:only} Invoke(i: ()) returns (r: Option, E>>) + method Invoke(i: ()) returns (r: Option, E>>) requires Requires(i) reads this, Repr modifies Modifies(i) @@ -323,9 +320,7 @@ module {:options "--function-syntax:4"} Std.Streams { OutputsPartitionedAfterOutputtingNone(); ProduceNone(); - label before: - // assert Valid(); Drain(wrapped); Repr := {this} + wrapped.Repr; producesTotalLengthProof.ProducesTotalLength(wrapped.history); @@ -333,11 +328,10 @@ module {:options "--function-syntax:4"} Std.Streams { assert Done(); assert Valid(); - RemainingMetricDoesntReadHistory(); - assert RemainingDecreasedBy(r); + reveal TerminationMetric.Ordinal(); + old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); } else { r := Read(length); - assert RemainingDecreasedBy(r); } } From 6acf9c147ffc5c253f94f57fd78b7ab541290cfb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 14 Apr 2025 08:42:11 -0700 Subject: [PATCH 13/72] cleanup --- .../StandardLibrary/src/Streams.dfy | 60 +++++-------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index d2398a3e8..2ca2dd121 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -15,6 +15,8 @@ module {:options "--function-syntax:4"} Std.Streams { // TODO: Consider a more generic term for Streamed[Of], // especially if we can decouple the filtering out Failures // from the concatenation of batches. + // TODO: Add this to the actual Dafny standard library, + // since it's generic enough. // // A data stream, i.e. a fallable producer of batches of values. @@ -35,7 +37,7 @@ module {:options "--function-syntax:4"} Std.Streams { trait DataStream extends Producer, E>> { // The total length of all produced batches - const contentLength: Option + const totalLength: Option ghost predicate Valid() reads this, Repr @@ -45,7 +47,7 @@ module {:options "--function-syntax:4"} Std.Streams { ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) decreases Repr ghost function Streamed(): seq @@ -90,28 +92,12 @@ module {:options "--function-syntax:4"} Std.Streams { && (!Seq.All(outputs, IsSome) ==> |streamed| == length) } - // TODO: Move next to ProducedOf? - lemma AboutProducedOf(outputs: seq>, x: T) - requires Partitioned(outputs, IsSome) - ensures x in ProducedOf(outputs) <==> Some(x) in outputs - {} - lemma StreamedOfSingleton(s: seq) ensures var singleton: Option, E>> := Some(Success(s)); StreamedOf([singleton]) == s {} - // TODO: Correcting a type in the original ensures clause, should be fixed at the source - lemma LemmaFilterImpliesAll(f: (T ~> bool), xs: seq) - requires forall i :: 0 <= i < |xs| ==> f.requires(xs[i]) - ensures - var result := Filter(f, xs); - forall i: nat :: i < |result| ==> f.requires(result[i]) && f(result[i]) - { - reveal Filter(); - } - lemma StreamedOfComposition(left: seq, E>>>, right: seq, E>>>) ensures StreamedOf(left + right) == StreamedOf(left) + StreamedOf(right) { @@ -250,7 +236,7 @@ module {:options "--function-syntax:4"} Std.Streams { && ValidHistory(history) && producesTotalLengthProof.producer == wrapped && producesTotalLengthProof.length == length as int - && contentLength == Some(length) + && totalLength == Some(length) && |buffer| <= length as int && StreamedOf(Outputs()) + buffer == StreamedOf(wrapped.Outputs()) && (0 < |buffer| ==> !wrapped.Done()) @@ -259,10 +245,10 @@ module {:options "--function-syntax:4"} Std.Streams { ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) decreases Repr { - contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) + totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -273,14 +259,6 @@ module {:options "--function-syntax:4"} Std.Streams { TMTuple(TMTop, wrapped.RemainingMetric(), TMNat(|buffer|)) } - twostate lemma RemainingMetricDoesntReadHistory() - requires old(Valid()) - requires Valid() - requires wrapped.RemainingMetric() == old(wrapped.RemainingMetric()) - requires buffer == old(buffer); - ensures RemainingMetric() == old(RemainingMetric()) - {} - constructor(wrapped: Producer, E>>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) requires wrapped.Valid() requires wrapped.history == [] @@ -293,7 +271,7 @@ module {:options "--function-syntax:4"} Std.Streams { this.length := length; this.buffer := []; - this.contentLength := Some(length); + this.totalLength := Some(length); this.history := []; this.Repr := {this} + wrapped.Repr; this.producesTotalLengthProof := producesTotalLengthProof; @@ -492,7 +470,7 @@ module {:options "--function-syntax:4"} Std.Streams { && ValidHistory(history) && s == data && |s| <= UINT64_MAX as int - && contentLength == Some(|s| as uint64) + && totalLength == Some(|s| as uint64) && position as int <= |s| && 0 < chunkSize && StreamedOf(Outputs()) == s[..position] @@ -500,11 +478,11 @@ module {:options "--function-syntax:4"} Std.Streams { ghost predicate ValidOutputs(outputs: seq, ()>>>) requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && contentLength.Some? ==> ValidStreamed(outputs, contentLength.value as int) + ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) decreases Repr { - && contentLength.Some? - && ValidStreamed(outputs, contentLength.value as int) + && totalLength.Some? + && ValidStreamed(outputs, totalLength.value as int) } ghost function RemainingMetric(): TerminationMetric @@ -524,20 +502,12 @@ module {:options "--function-syntax:4"} Std.Streams { this.s := s; this.position := 0; this.chunkSize := chunkSize; - this.contentLength := Some(|s| as uint64); + this.totalLength := Some(|s| as uint64); this.history := []; this.Repr := {this}; } - function ContentLength(): (res: Option) - requires Valid() - reads this, Repr - ensures res == Some(|data| as uint64) - { - Some(|s| as uint64) - } - function Position(): (res: uint64) requires Valid() reads this, Repr @@ -598,7 +568,7 @@ module {:options "--function-syntax:4"} Std.Streams { if position == |s| as uint64 { r := None; - ValidStreamedAfterNone(Outputs(), contentLength.value as int); + ValidStreamedAfterNone(Outputs(), totalLength.value as int); assert OutputsOf(history + [((), None)]) == Outputs() + [None]; assert ValidHistory(history + [((), None)]); ProduceNone(); @@ -614,7 +584,7 @@ module {:options "--function-syntax:4"} Std.Streams { r := Some(chunk); position := newPosition; - ValidStreamedAfterMoreData(Outputs(), chunk.value, contentLength.value as int); + ValidStreamedAfterMoreData(Outputs(), chunk.value, totalLength.value as int); assert OutputsOf(history + [((), r)]) == Outputs() + [r]; assert ValidHistory(history + [((), r)]); ProduceSome(chunk); From 919ed23fd72515885cdb34f2a60aca06f3585c9b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 14 Apr 2025 09:19:14 -0700 Subject: [PATCH 14/72] Update streams.py --- .../internaldafny/extern/streams.py | 51 +++++++++++-------- TestModels/dafny-dependencies/dafny | 2 +- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index 7a6eff36b..d577f7ecd 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -1,7 +1,7 @@ from _dafny import Seq from smithy_python.interfaces.blobs import ByteStream -from smithy_dafny_standard_library.internaldafny.generated.Std_Streams import ByteStream as DafnyByteStream, RewindableByteStream as DafnyRewindableByteStream +from smithy_dafny_standard_library.internaldafny.generated.Std_Streams import DataStream as DafnyDataStream, RewindableDataStream as DafnyRewindableDataStream from smithy_dafny_standard_library.internaldafny.generated.Std_Enumerators import Enumerator from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None @@ -10,54 +10,56 @@ # These are the equivalent of type conversions, # but avoiding having to load all data into memory at once. -class DafnyByteStreamAsByteStream(ByteStream): - """Wrapper class adapting a Dafny ByteStream as a native ByteStream.""" +class DafnyDataStreamAsByteStream(ByteStream): + """Wrapper class adapting a Dafny DataStream as a native ByteStream.""" - def __init__(self, dafny_byte_stream): - self.dafny_byte_stream = dafny_byte_stream + def __init__(self, dafny_data_stream): + self.dafny_data_stream = dafny_data_stream def read(self, size: int = -1) -> bytes: - # TODO: assert size is -1, buffer, - # or define a more specialized Action type for streams. - next = self.dafny_byte_stream.Next() - while next.is_Some and len(next.value) == 0: - next = self.dafny_byte_stream.Next() + next = None + while next is None or (next.is_Some and len(next.value) == 0): + if size == -1: + next = self.dafny_data_stream.Next() + else: + next = self.dafny_byte_stream.Read(size) + # Do NOT return None, because that indicates "no data right now, might be more later" return bytes(next.value) if next.is_Some else bytes() -class RewindableDafnyByteStreamAsByteStream(DafnyByteStreamAsByteStream): - """Wrapper class adapting a Dafny RewindableByteStream as a native ByteStream +class DafnyRewindableDataStreamAsByteStream(DafnyByteStreamAsByteStream): + """Wrapper class adapting a Dafny RewindableDataStream as a native ByteStream that supports tell and seek. """ - def __init__(self, dafny_byte_stream): - if not isinstance(dafny_byte_stream, DafnyRewindableByteStream): + def __init__(self, dafny_data_stream): + if not isinstance(dafny_data_stream, DafnyRewindableDataStream): raise ValueError("Rewindable stream required") - super().__init__(dafny_byte_stream) + super().__init__(dafny_data_stream) def tell(self) -> int: - return self.dafny_byte_stream.Position() + return self.dafny_data_stream.Position() def seek(self, offset, whence=0): match whence: case 0: position = offset case 1: - position = self.dafny_byte_stream.Position() + offset + position = self.dafny_data_stream.Position() + offset case 2: - position = self.dafny_byte_stream.ContentLength() + offset - return self.dafny_byte_stream.Seek(position) + position = self.dafny_data_stream.totalLength.value + offset + return self.dafny_data_stream.Seek(position) -class StreamingBlobAsDafnyDataStream(DafnyByteStream): +class StreamingBlobAsDafnyDataStream(DafnyDataStream): """Wrapper class adapting a native StreamingBlob as a Dafny ByteStream.""" def __init__(self, streaming_blob): self.streaming_blob = streaming_blob def Next(self): - return Enumerator.Next(self) + return Producer.Next(self) def Invoke(self, _) -> Option: next = self.streaming_blob.read() @@ -65,3 +67,10 @@ def Invoke(self, _) -> Option: return Option_Some(Seq(next)) else: return Option_None() + + def Read(self, size) -> Option: + next = self.streaming_blob.read(size) + if next: + return Option_Some(Seq(next)) + else: + return Option_None() diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 685ef6436..48cac11b3 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 685ef64365feb5e6b76794fabf03bed8ab805002 +Subproject commit 48cac11b391a7919102e460420988b17808f1b21 From 8022fff7e3c67930e03a2086cad03b7725168db7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 14 Apr 2025 09:24:20 -0700 Subject: [PATCH 15/72] Renames in codegen --- .../amazon/polymorph/smithydafny/DafnyApiCodegen.java | 5 ++--- .../amazon/polymorph/smithydafny/DafnyNameResolver.java | 2 +- .../awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java | 4 ++-- .../shapevisitor/DafnyToLocalServiceShapeVisitor.java | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index 93e2424d9..e2128d850 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -321,9 +321,8 @@ public TokenTree generateBlobTypeDefinition(final ShapeId blobShapeId) { .map(DafnyApiCodegen::generateLengthConstraint); if (blobShape.hasTrait(StreamingTrait.class)) { // TODO: need to handle @length too, - // something like `forall produced | a.CanProduce(produced) :: min <= |Enumerated(produced)| <= max - // (which should have a simpler helper predicate version, especially when allowing for reference types) - return generateTypeSynonym(blobShapeId, "ByteStream"); + // something like `a.totalLength.Some? && min <= a.contentLength.value <= max + return generateTypeSynonym(blobShapeId, "DataStream, Error>"); } else { return generateSubsetType(blobShapeId, "seq", lengthConstraint); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index 80cce84b3..ea5e2f536 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -90,7 +90,7 @@ public String baseTypeForShape(final ShapeId shapeId) { MAP -> dafnyModulePrefixForShape(shape) + shapeName; case BLOB -> { if (shape.hasTrait(StreamingTrait.class)) { - yield "ByteStream"; + yield "DataStream>"; } else { yield dafnyModulePrefixForShape(shape) + shapeName; } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java index 6a2029a1f..d757d32e2 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java @@ -78,9 +78,9 @@ public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( "smithy_dafny_standard_library.internaldafny.extern.streams", - "RewindableDafnyByteStreamAsByteStream" + "RewindableDafnyDataStreamAsByteStream" ); - return "RewindableDafnyByteStreamAsByteStream(%1$s)".formatted( + return "RewindableDafnyDataStreamAsByteStream(%1$s)".formatted( dataSource ); } else { diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java index 06679418c..b057fb16d 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java @@ -96,9 +96,9 @@ public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( "smithy_dafny_standard_library.internaldafny.extern.streams", - "DafnyByteStreamAsByteStream" + "DafnyDataStreamAsByteStream" ); - return "DafnyByteStreamAsByteStream(%1$s)".formatted(dataSource); + return "DafnyDataStreamAsByteStream(%1$s)".formatted(dataSource); } else { return "bytes(%1$s)".formatted(dataSource); } From dfe02822637e463f28898f6b28a0c40fa8fa83d0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 14 Apr 2025 16:09:57 -0700 Subject: [PATCH 16/72] Partial fixes --- .../Streaming/Model/SimpleStreamingTypes.dfy | 10 +- .../codegen-patches/dafny/dafny-4.9.0.patch | 94 --------------- TestModels/Streaming/src/Chunker.dfy | 110 ++++++++++++------ .../Streaming/src/SimpleStreamingImpl.dfy | 33 ++---- .../src/WrappedSimpleStreamingImpl.dfy | 2 +- .../StandardLibrary/src/Index.dfy | 3 +- .../StandardLibrary/src/Streams.dfy | 12 +- TestModels/dafny-dependencies/dafny | 2 +- .../smithydafny/DafnyApiCodegen.java | 2 +- .../smithydafny/DafnyNameResolver.java | 2 +- 10 files changed, 103 insertions(+), 167 deletions(-) delete mode 100644 TestModels/Streaming/codegen-patches/dafny/dafny-4.9.0.patch diff --git a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy index c047e70e0..e192991f2 100644 --- a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy +++ b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy @@ -17,17 +17,17 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes nameonly number: int32 ) datatype BinaryOfOutput = | BinaryOfOutput ( - nameonly binary: ByteStream + nameonly binary: DataStream ) datatype ChunksInput = | ChunksInput ( - nameonly bytesIn: ByteStream , + nameonly bytesIn: DataStream , nameonly chunkSize: CountingInteger ) datatype ChunksOutput = | ChunksOutput ( - nameonly bytesOut: ByteStream + nameonly bytesOut: DataStream ) datatype CountBitsInput = | CountBitsInput ( - nameonly bits: ByteStream + nameonly bits: DataStream ) datatype CountBitsOutput = | CountBitsOutput ( nameonly sum: int32 @@ -130,7 +130,7 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes datatype SimpleStreamingConfig = | SimpleStreamingConfig ( ) - type StreamingBlob = ByteStream + type StreamingBlob = DataStream datatype Error = // Local Error structures are listed here | OverflowError ( diff --git a/TestModels/Streaming/codegen-patches/dafny/dafny-4.9.0.patch b/TestModels/Streaming/codegen-patches/dafny/dafny-4.9.0.patch deleted file mode 100644 index 4736142a2..000000000 --- a/TestModels/Streaming/codegen-patches/dafny/dafny-4.9.0.patch +++ /dev/null @@ -1,94 +0,0 @@ -diff --git b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy -index af6ad336f..50140cc6c 100644 ---- b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy -+++ a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy -@@ -79,8 +79,12 @@ module SimpleStreamingTypes - returns (output: Result) - requires - && ValidState() -+ // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` part. -+ requires -+ && input.bits.Valid() && History !in input.bits.Repr - modifies Modifies - {History} , -- History`CountBits -+ History`CountBits , -+ input.bits.Repr - // Dafny will skip type parameters when generating a default decreases clause. - decreases Modifies - {History} - ensures -@@ -102,6 +106,8 @@ module SimpleStreamingTypes - && ValidState() - ensures BinaryOfEnsuresPublicly(input, output) - ensures History.BinaryOf == old(History.BinaryOf) + [DafnyCallEvent(input, output)] -+ // TODO: smithy-dafny isn't yet generating this -+ ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) - - predicate ChunksEnsuresPublicly(input: ChunksInput , output: Result) - // The public method to be called by library consumers -@@ -212,8 +218,12 @@ abstract module AbstractSimpleStreamingService - returns (output: Result) - requires - && ValidState() -+ // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` parts. -+ requires -+ && input.bits.Valid() && History !in input.bits.Repr - modifies Modifies - {History} , -- History`CountBits -+ History`CountBits , -+ input.bits.Repr - // Dafny will skip type parameters when generating a default decreases clause. - decreases Modifies - {History} - ensures -@@ -221,6 +231,10 @@ abstract module AbstractSimpleStreamingService - ensures CountBitsEnsuresPublicly(input, output) - ensures History.CountBits == old(History.CountBits) + [DafnyCallEvent(input, output)] - { -+ // TODO: It's not clear how to actually ensure this, -+ // since the internal config is not visible to the trait -+ // so it can't really be a precondition there. -+ assume {:axiom} Operations.ModifiesInternalConfig(config) !! input.bits.Repr; - output := Operations.CountBits(config, input); - History.CountBits := History.CountBits + [DafnyCallEvent(input, output)]; - } -@@ -240,6 +254,8 @@ abstract module AbstractSimpleStreamingService - && ValidState() - ensures BinaryOfEnsuresPublicly(input, output) - ensures History.BinaryOf == old(History.BinaryOf) + [DafnyCallEvent(input, output)] -+ // TODO: smithy-dafny isn't yet generating this -+ ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) - { - output := Operations.BinaryOf(config, input); - History.BinaryOf := History.BinaryOf + [DafnyCallEvent(input, output)]; -@@ -283,8 +299,12 @@ abstract module AbstractSimpleStreamingOperations { - method CountBits ( config: InternalConfig , input: CountBitsInput ) - returns (output: Result) - requires -- && ValidInternalConfig?(config) -- modifies ModifiesInternalConfig(config) -+ // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` part. -+ && ValidInternalConfig?(config) -+ && input.bits.Valid() -+ && ModifiesInternalConfig(config) !! input.bits.Repr -+ // TODO: smithy-dafny isn't yet generating the `input.bits.Repr` part. -+ modifies ModifiesInternalConfig(config), input.bits.Repr - // Dafny will skip type parameters when generating a default decreases clause. - decreases ModifiesInternalConfig(config) - ensures -@@ -306,6 +326,8 @@ abstract module AbstractSimpleStreamingOperations { - ensures - && ValidInternalConfig?(config) - ensures BinaryOfEnsuresPublicly(input, output) -+ // TODO: smithy-dafny isn't yet generating this -+ ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) - - - predicate ChunksEnsuresPublicly(input: ChunksInput , output: Result) -@@ -316,6 +338,8 @@ abstract module AbstractSimpleStreamingOperations { - returns (output: Result) - requires - && ValidInternalConfig?(config) -+ // TODO: smithy-dafny isn't yet generating this -+ && input.bytesIn.Valid() - modifies ModifiesInternalConfig(config) - // Dafny will skip type parameters when generating a default decreases clause. - decreases ModifiesInternalConfig(config) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 8bcdedd95..d7635f8c7 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -1,14 +1,14 @@ include "../Model/SimpleStreamingTypes.dfy" -module Chunker { +module {:options "--function-syntax:4"} Chunker { import Std.BoundedInts import opened Std.Wrappers import opened Types = SimpleStreamingTypes import opened StandardLibrary.UInt - import opened Std.Enumerators - import opened Std.Aggregators + import opened Std.Actions + import opened Std.Producers // An example of a Pipeline, which processes chunks of bytes // as they flow through a stream. @@ -18,55 +18,95 @@ module Chunker { // when a chunk becomes available to the pipeline, // zero or more chunks are made available downstream. @AssumeCrossModuleTermination - class Chunker extends Pipeline { + class Chunker extends Action, Option>> { const chunkSize: CountingInteger var chunkBuffer: BoundedInts.bytes - constructor(upstream: Enumerator, chunkSize: CountingInteger) + constructor(chunkSize: CountingInteger) ensures Valid() ensures history == [] { - this.buffer := new Collector(); - this.upstream := upstream; - this.chunkSize := chunkSize; chunkBuffer := []; history := []; - Repr := {this} + upstream.Repr; - new; - assume {:axiom} Valid(); + Repr := {this}; } - method Process(event: Option, a: Accumulator) - requires Valid() - requires a.Valid() - requires Repr !! a.Repr - modifies Repr, a.Repr - ensures a.ValidAndDisjoint() + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + decreases Repr, 0 { - assert this in Repr; - assert this !in a.Repr; - match event { - case Some(bits) => { - chunkBuffer := chunkBuffer + bits; - } - case None => return; - } + this in Repr + } - while chunkSize as int <= |chunkBuffer| - invariant a.ValidAndDisjoint() - { - a.CanConsumeAll(a.history, chunkBuffer[..chunkSize]); - a.Accept(chunkBuffer[..chunkSize]); - chunkBuffer := chunkBuffer[chunkSize..]; - } - - if event == None { + ghost predicate ValidHistory(history: seq<(Option, Option>)>) + decreases Repr + { + true + } + + ghost predicate ValidInput(history: seq<(Option, Option>)>, next: Option) + requires ValidHistory(history) + decreases Repr + { + true + } + + ghost function Decreases(i: Option): ORDINAL + requires Requires(i) + reads Reads(i) + { + 0 + } + + method Invoke(bits: Option) returns (r: Option>) + requires Requires(bits) + modifies Modifies(bits) + decreases Decreases(bits), 0 + ensures Ensures(bits, r) + { + var outputChunks := []; + if bits.Some? { + chunkBuffer := chunkBuffer + bits.value; + + while chunkSize as int <= |chunkBuffer| + { + outputChunks := outputChunks + [chunkBuffer[..chunkSize]]; + chunkBuffer := chunkBuffer[chunkSize..]; + } + } else { if 0 < |chunkBuffer| { - var _ := a.Invoke(chunkBuffer); + outputChunks := outputChunks + [chunkBuffer]; + } else { + r := None; + return; } } + + var output := new SeqReader(outputChunks); + r := Some(output); } } + + function SumBits(sum: int, maybeChunk: Result, Error>): int { + match maybeChunk + case Success(chunk) => sum + BytesBitCount(chunk) + case Failure(_) => sum + } + + function BytesBitCount(b: seq): int { + Seq.FoldLeft((sum, byte) => sum + BitCount(byte), 0 as int, b) + } + + function BitCount(x: uint8): int { + if x == 0 then + 0 + else if x % 2 == 1 then + 1 + BitCount(x / 2) + else + BitCount(x / 2) + } } \ No newline at end of file diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index a1513822d..e94bf9c82 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -5,8 +5,9 @@ include "../Model/SimpleStreamingTypes.dfy" include "Chunker.dfy" module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpleStreamingOperations { - import Std.Enumerators - import Std.Aggregators + import Std.Actions + import Std.Producers + import Std.Consumers import Std.Collections.Seq import opened Chunker @@ -22,11 +23,12 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl method CountBits ( config: InternalConfig , input: CountBitsInput ) returns (output: Result) { - var counter := new Aggregators.FoldingAccumulator(0, (sum, byte) => sum + BytesBitCount(byte)); + var counter := new Consumers.FoldingConsumer(0, SumBits); + var counterTotalProof := new Consumers.FoldingConsumerTotalActionProof(counter); - Enumerators.ForEach(input.bits, counter); + input.bits.ForEachRemaining(counter, counterTotalProof); - // Should really have the FoldingAccumulator fail instead, + // Should really have the FoldingConsumer fail instead, // but this is a simpler correct approach. if 0 <= counter.value < INT32_MAX_LIMIT { return Success(CountBitsOutput(sum := counter.value as int32)); @@ -35,19 +37,6 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl } } - function BytesBitCount(b: BoundedInts.bytes): int { - Seq.FoldLeft((sum, byte) => sum + BitCount(byte), 0 as int, b) - } - - function BitCount(x: BoundedInts.uint8): int { - if x == 0 then - 0 - else if x % 2 == 1 then - 1 + BitCount(x / 2) - else - BitCount(x / 2) - } - predicate BinaryOfEnsuresPublicly(input: BinaryOfInput , output: Result) {true} @@ -58,9 +47,9 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl { // TODO: Actually compute the binary - var fakeBinary: seq := [[12], [34, 56]]; - var fakeBinaryEnumerator := new Enumerators.SeqEnumerator(fakeBinary); - var fakeBinaryStream := new EnumeratorDataStream(fakeBinaryEnumerator, 3 as BoundedInts.uint64); + var fakeBinary := [Success([12]), Success([34, 56])]; + var fakeBinaryEnumerator := new Producers.SeqReader(fakeBinary); + var fakeBinaryStream := new ProducerDataStream(fakeBinaryEnumerator, 3 as BoundedInts.uint64); return Success(BinaryOfOutput(binary := fakeBinaryStream)); } @@ -74,7 +63,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl { // TODO: for now assume {:axiom} input.bytesIn.history == []; - var chunker := new Chunker(input.bytesIn, input.chunkSize); + var chunker := new Chunker(input.chunkSize); var chunkerStream := new EnumeratorDataStream(chunker, input.bytesIn.ContentLength()); return Success(ChunksOutput(bytesOut := chunkerStream)); diff --git a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy index 58bf8aed5..00827803f 100644 --- a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/SimpleStreamingTypesWrapped.dfy" -module {:extern "simple.streaming.internaldafny.wrapped"} WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { +module WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { import WrappedService = SimpleStreaming function method WrappedDefaultSimpleStreamingConfig(): SimpleStreamingConfig { SimpleStreamingConfig diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy index e9c061a82..55fab1f33 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy @@ -4,4 +4,5 @@ include "./WrappersInterop.dfy" include "./StandardLibrary.dfy" include "./UInt.dfy" -include "./UTF8.dfy" \ No newline at end of file +include "./UTF8.dfy" +include "./Streams.dfy" \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 2ca2dd121..e921f32bc 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -59,7 +59,7 @@ module {:options "--function-syntax:4"} Std.Streams { method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) - reads Reads(()) + // reads Reads(()) modifies Modifies(()) decreases Decreases(()) ensures Ensures((), r) @@ -169,7 +169,7 @@ module {:options "--function-syntax:4"} Std.Streams { method Drain(p: Producer) requires p.Valid() - reads p.Repr + // reads p.Repr modifies p.Repr ensures p.ValidAndDisjoint() ensures p.Done() @@ -279,7 +279,7 @@ module {:options "--function-syntax:4"} Std.Streams { method Invoke(i: ()) returns (r: Option, E>>) requires Requires(i) - reads this, Repr + // reads this, Repr modifies Modifies(i) decreases Decreases(i), 0 ensures Ensures(i, r) @@ -316,7 +316,7 @@ module {:options "--function-syntax:4"} Std.Streams { @IsolateAssertions method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) - reads Reads(()) + // reads Reads(()) modifies Modifies(()) decreases Decreases(()) ensures Ensures((), r) @@ -543,7 +543,7 @@ module {:options "--function-syntax:4"} Std.Streams { method Invoke(t: ()) returns (r: Option, ()>>) requires Requires(t) - reads this, Repr + // reads this, Repr modifies Modifies(t) decreases Decreases(t), 0 ensures Ensures(t, r) @@ -555,7 +555,7 @@ module {:options "--function-syntax:4"} Std.Streams { method Read(max: uint64) returns (r: Option, ()>>) requires Requires(()) - reads this, Repr + // reads this, Repr modifies Modifies(()) decreases Decreases(()) ensures Ensures((), r) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 48cac11b3..4285f1057 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 48cac11b391a7919102e460420988b17808f1b21 +Subproject commit 4285f10572ecf5e6a023cf26ae218b44554eb298 diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index e2128d850..99aa9d6c6 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -322,7 +322,7 @@ public TokenTree generateBlobTypeDefinition(final ShapeId blobShapeId) { if (blobShape.hasTrait(StreamingTrait.class)) { // TODO: need to handle @length too, // something like `a.totalLength.Some? && min <= a.contentLength.value <= max - return generateTypeSynonym(blobShapeId, "DataStream, Error>"); + return generateTypeSynonym(blobShapeId, "DataStream"); } else { return generateSubsetType(blobShapeId, "seq", lengthConstraint); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index ea5e2f536..e3f6bebe8 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -90,7 +90,7 @@ public String baseTypeForShape(final ShapeId shapeId) { MAP -> dafnyModulePrefixForShape(shape) + shapeName; case BLOB -> { if (shape.hasTrait(StreamingTrait.class)) { - yield "DataStream>"; + yield "DataStream"; } else { yield dafnyModulePrefixForShape(shape) + shapeName; } From 75602554c7fe4c97cdb192d1b59e2403fda56d61 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 14 Apr 2025 16:39:05 -0700 Subject: [PATCH 17/72] Fix BinaryOf --- TestModels/Streaming/src/Chunker.dfy | 12 +++++------- TestModels/Streaming/src/SimpleStreamingImpl.dfy | 10 ++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index d7635f8c7..b26c2798e 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -10,13 +10,6 @@ module {:options "--function-syntax:4"} Chunker { import opened Std.Actions import opened Std.Producers - // An example of a Pipeline, which processes chunks of bytes - // as they flow through a stream. - // Pipelines let you define a stream transformation once - // in a way that allows external code to apply it to either - // push or pull-based streams: - // when a chunk becomes available to the pipeline, - // zero or more chunks are made available downstream. @AssumeCrossModuleTermination class Chunker extends Action, Option>> { @@ -109,4 +102,9 @@ module {:options "--function-syntax:4"} Chunker { else BitCount(x / 2) } + + function BinaryOfNumber(x: int32): seq { + // TODO: Actually compute the binary + [12 as uint8, 34, 56] + } } \ No newline at end of file diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index e94bf9c82..437e2e25d 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -46,12 +46,10 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl returns (output: Result) { - // TODO: Actually compute the binary - var fakeBinary := [Success([12]), Success([34, 56])]; - var fakeBinaryEnumerator := new Producers.SeqReader(fakeBinary); - var fakeBinaryStream := new ProducerDataStream(fakeBinaryEnumerator, 3 as BoundedInts.uint64); + var binary := BinaryOfNumber(input.number); + var binaryStream := new SeqDataStream(binary, 3 as BoundedInts.uint64); - return Success(BinaryOfOutput(binary := fakeBinaryStream)); + return Success(BinaryOfOutput(binary := binaryStream)); } @@ -64,7 +62,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl // TODO: for now assume {:axiom} input.bytesIn.history == []; var chunker := new Chunker(input.chunkSize); - var chunkerStream := new EnumeratorDataStream(chunker, input.bytesIn.ContentLength()); + var chunkerStream := new ProducerDataStream(chunker, input.bytesIn.totalLength.value); return Success(ChunksOutput(bytesOut := chunkerStream)); } From 12fab4c59c76b0348b16a9192f48980fef37577a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 23 Apr 2025 14:24:12 -0700 Subject: [PATCH 18/72] Partial work --- TestModels/Streaming/src/Chunker.dfy | 104 ++++++++++++--- .../Streaming/src/SimpleStreamingImpl.dfy | 2 +- TestModels/Streaming/src/dfyconfig.toml | 7 + ...StandardLibraries-smithy-dafny-subset.toml | 1 + .../StandardLibrary/src/Streams.dfy | 126 +++++++++++++++++- 5 files changed, 217 insertions(+), 23 deletions(-) create mode 100644 TestModels/Streaming/src/dfyconfig.toml diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index b26c2798e..2440fca52 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -8,16 +8,20 @@ module {:options "--function-syntax:4"} Chunker { import opened Types = SimpleStreamingTypes import opened StandardLibrary.UInt import opened Std.Actions + import opened Std.BulkActions import opened Std.Producers + import opened Std.Consumers + import opened Std.Streams @AssumeCrossModuleTermination - class Chunker extends Action, Option>> { + class Chunker extends BulkAction>, Option>>> { const chunkSize: CountingInteger var chunkBuffer: BoundedInts.bytes constructor(chunkSize: CountingInteger) ensures Valid() + ensures fresh(Repr) ensures history == [] { this.chunkSize := chunkSize; @@ -35,55 +39,119 @@ module {:options "--function-syntax:4"} Chunker { this in Repr } - ghost predicate ValidHistory(history: seq<(Option, Option>)>) + ghost predicate ValidHistory(history: seq<(Option>, Option>>)>) decreases Repr { true } - ghost predicate ValidInput(history: seq<(Option, Option>)>, next: Option) + ghost predicate ValidInput(history: seq<(Option>, Option>>)>, next: Option>) requires ValidHistory(history) decreases Repr { true } - ghost function Decreases(i: Option): ORDINAL + ghost function Decreases(i: Option>): ORDINAL requires Requires(i) reads Reads(i) { 0 } - method Invoke(bits: Option) returns (r: Option>) - requires Requires(bits) - modifies Modifies(bits) - decreases Decreases(bits), 0 - ensures Ensures(bits, r) + // Ideally could use this in a MappedConsumer as well + + method Single(input: Option>) returns (r: Option>>) { + + } + + method Bulk(input: Producer>, output: Consumer>) { + + } + + method BulkInvoke(input: Producer>, output: IConsumer>) + requires Requires(input) + modifies Modifies(input) + decreases Decreases(input), 0 + ensures Ensures(input, r) + { + assert Valid(); var outputChunks := []; - if bits.Some? { - chunkBuffer := chunkBuffer + bits.value; - - while chunkSize as int <= |chunkBuffer| - { - outputChunks := outputChunks + [chunkBuffer[..chunkSize]]; - chunkBuffer := chunkBuffer[chunkSize..]; + if input.Some? { + if input.value.Failure? { + outputChunks := outputChunks + [input.value]; + } else { + + chunkBuffer := chunkBuffer + input.value.value; + + while chunkSize as int <= |chunkBuffer| + invariant ValidAndDisjoint() + invariant history == old(history) + { + outputChunks := outputChunks + [Success(chunkBuffer[..chunkSize])]; + chunkBuffer := chunkBuffer[chunkSize..]; + } } } else { if 0 < |chunkBuffer| { - outputChunks := outputChunks + [chunkBuffer]; + outputChunks := outputChunks + [Success(chunkBuffer)]; } else { r := None; + UpdateHistory(input, r); return; } } - var output := new SeqReader(outputChunks); r := Some(output); + UpdateHistory(input, r); } } + @AssumeCrossModuleTermination + class ChunkerTotalProof extends TotalActionProof>, Option>>> { + + ghost const chunker: Chunker + + ghost constructor(chunker: Chunker) + requires chunker.Valid() + ensures this.chunker == chunker + ensures Valid() + ensures fresh(Repr) + { + this.chunker := chunker; + Repr := {this}; + } + + ghost function Action(): Action>, Option>>> { + chunker + } + + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + decreases Repr, 0 + { + this in Repr + } + + lemma AnyInputIsValid(history: seq<(Option>, Option>>)>, next: Option>) + requires Valid() + requires Action().ValidHistory(history) + ensures Action().ValidInput(history, next) + {} + + } + + method ChunkingStream(chunkSize: CountingInteger, s: DataStream) + requires s.Valid() + requires s.history == [] + { + var chunker := new Chunker(chunkSize); + ghost var chunkerTotalProof := new ChunkerTotalProof(chunker); + var chunkerStream := new OptionMappedProducer(s, chunker, chunkerTotalProof); + } + function SumBits(sum: int, maybeChunk: Result, Error>): int { match maybeChunk case Success(chunk) => sum + BytesBitCount(chunk) diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index 437e2e25d..647c17b5a 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -62,7 +62,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl // TODO: for now assume {:axiom} input.bytesIn.history == []; var chunker := new Chunker(input.chunkSize); - var chunkerStream := new ProducerDataStream(chunker, input.bytesIn.totalLength.value); + var chunkerStream := new MappedDataStream(input.bytesIn, chunker); return Success(ChunksOutput(bytesOut := chunkerStream)); } diff --git a/TestModels/Streaming/src/dfyconfig.toml b/TestModels/Streaming/src/dfyconfig.toml new file mode 100644 index 000000000..5ab3faf93 --- /dev/null +++ b/TestModels/Streaming/src/dfyconfig.toml @@ -0,0 +1,7 @@ +[options] +function-syntax = 3 +unicode-char = false + +library = [ + "/Users/salkeldr/Documents/GitHub/smithy-dafny/TestModels/dafny-dependencies/StandardLibrary/bin/DafnyStandardLibraries-smithy-dafny-subset.doo" +] \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml index a8d9c8c64..ead8c694b 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml +++ b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml @@ -1,5 +1,6 @@ includes = [ "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Actions.dfy", + "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/BulkActions.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Producers.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Consumers.dfy", "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/GenericAction.dfy", diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index e921f32bc..e7b8278c1 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -453,7 +453,7 @@ module {:options "--function-syntax:4"} Std.Streams { /* * Rewindable stream of a sequence with a configured default chunk size. */ - class SeqDataStream extends RewindableDataStream { + class SeqDataStream extends RewindableDataStream { const s: seq const chunkSize: uint64 @@ -476,7 +476,7 @@ module {:options "--function-syntax:4"} Std.Streams { && StreamedOf(Outputs()) == s[..position] } - ghost predicate ValidOutputs(outputs: seq, ()>>>) + ghost predicate ValidOutputs(outputs: seq, E>>>) requires Seq.Partitioned(outputs, IsSome) ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) decreases Repr @@ -538,10 +538,11 @@ module {:options "--function-syntax:4"} Std.Streams { } else { assert outputs == [Some(Success(s[..position]))]; StreamedOfSingleton(s[..position]); + assert ValidHistory(history); } } - method Invoke(t: ()) returns (r: Option, ()>>) + method Invoke(t: ()) returns (r: Option, E>>) requires Requires(t) // reads this, Repr modifies Modifies(t) @@ -553,7 +554,7 @@ module {:options "--function-syntax:4"} Std.Streams { r := Read(chunkSize); } - method Read(max: uint64) returns (r: Option, ()>>) + method Read(max: uint64) returns (r: Option, E>>) requires Requires(()) // reads this, Repr modifies Modifies(()) @@ -603,4 +604,121 @@ module {:options "--function-syntax:4"} Std.Streams { assert Valid(); } } + + // TODO: better name? + // TODO: Document the fact that this doesn't have control + // over the parameter to original.Read(), + // but also that this would be low value since some languages + // will be forced to buffer to obey that parameter anyway (e.g. Java Publishers) + @AssumeCrossModuleTermination + class OptionMappedProducer extends Producer { + + const original: Producer + const mapping: Action, Option> + + ghost const mappingTotalProof: TotalActionProof, Option> + ghost const base: TerminationMetric + + constructor (original: Producer, + mapping: Action, Option>, + ghost mappingTotalProof: TotalActionProof, Option>) + requires original.Valid() + requires original.history == [] + requires mapping.Valid() + requires mapping.history == [] + requires mappingTotalProof.Valid() + requires mappingTotalProof.Action() == mapping + requires original.Repr !! mapping.Repr !! mappingTotalProof.Repr + ensures Valid() + ensures history == [] + ensures fresh(Repr - original.Repr - mapping.Repr - mappingTotalProof.Repr) + { + this.original := original; + this.mapping := mapping; + this.mappingTotalProof := mappingTotalProof; + this.base := TMSucc(original.RemainingMetric()); + + Repr := {this} + original.Repr + mapping.Repr + mappingTotalProof.Repr; + history := []; + + new; + this.base.SuccDecreasesToOriginal(); + } + + ghost predicate Valid() + reads this, Repr + ensures Valid() ==> this in Repr + ensures Valid() ==> ValidHistory(history) + decreases Repr, 0 + { + && this in Repr + && ValidComponent(original) + && ValidComponent(mapping) + && ValidComponent(mappingTotalProof) + && mappingTotalProof.Action() == mapping + && original.Repr !! mapping.Repr !! mappingTotalProof.Repr + && ValidHistory(history) + && base.DecreasesTo(original.RemainingMetric()) + && (!original.Done() <==> !Done()) + } + + ghost predicate ValidOutputs(outputs: seq>) + requires Seq.Partitioned(outputs, IsSome) + decreases Repr + { + true + } + + ghost function RemainingMetric(): TerminationMetric + requires Valid() + reads this, Repr + decreases Repr, 3 + { + TMTuple(base, TMNat(0), original.RemainingMetric()) + } + + @ResourceLimit("1e7") + method Invoke(t: ()) returns (result: Option) + requires Requires(t) + // reads Reads(t) + modifies Modifies(t) + decreases Decreases(t), 0 + ensures Ensures(t, result) + ensures RemainingDecreasedBy(result) + { + assert Requires(t); + assert Valid(); + RemainingMetric().TupleDecreasesToSecond(); + assert Remaining() > original.Remaining(); + var next := original.Next(); + + mappingTotalProof.AnyInputIsValid(mapping.history, next); + var mappedNext := mapping.Invoke(next); + result := mappedNext; + + if mappedNext.Some? { + mappingTotalProof.AnyInputIsValid(mapping.history, next); + var nextValue := mapping.Invoke(next); + result := nextValue; + + OutputsPartitionedAfterOutputtingSome(mappedNext.value); + ProduceSome(mappedNext.value); + } else { + original.OutputtingNoneMeansNotAllSome(); + OutputsPartitionedAfterOutputtingNone(); + ProduceNone(); + + assert !IsSome(Seq.Last(Outputs())); + } + + Repr := {this} + original.Repr + mapping.Repr + mappingTotalProof.Repr; + assert Valid(); + assert original.RemainingDecreasedBy(next); + if next.Some? { + old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); + } else { + old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); + } + } + } } \ No newline at end of file From 70b40cdacef523940e38143bd52099a428d66b70 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 25 Apr 2025 09:17:59 -0700 Subject: [PATCH 19/72] Mostly there --- .../Streaming/Model/SimpleStreamingTypes.dfy | 48 +- TestModels/Streaming/src/Chunker.dfy | 4 +- .../Streaming/src/SimpleStreamingImpl.dfy | 3 +- TestModels/aws-sdks/s3/dfyconfig.toml | 10 + .../aws-sdks/s3/test/TestComAmazonawsS3.dfy | 31 +- ...StandardLibraries-smithy-dafny-subset.toml | 8 +- .../internaldafny/extern/streams.py | 80 +- .../StandardLibrary/src/StandardLibrary.dfy | 2 +- .../StandardLibrary/src/Streams.dfy | 728 +----------------- .../StandardLibrary/src/UTF8.dfy | 4 +- TestModels/dafny-dependencies/dafny | 2 +- .../smithydafny/DafnyApiCodegen.java | 4 +- .../smithydafny/DafnyNameResolver.java | 4 +- 13 files changed, 129 insertions(+), 799 deletions(-) create mode 100644 TestModels/aws-sdks/s3/dfyconfig.toml diff --git a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy index e192991f2..dfaf96c7f 100644 --- a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy +++ b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy @@ -7,7 +7,7 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes import opened Wrappers import opened StandardLibrary.UInt import opened UTF8 - import opened Std.Streams + import opened StandardLibrary.Streams // Generic helpers for verification of mock/unit tests. datatype DafnyCallEvent = DafnyCallEvent(input: I, output: O) @@ -17,17 +17,17 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes nameonly number: int32 ) datatype BinaryOfOutput = | BinaryOfOutput ( - nameonly binary: DataStream + nameonly binary: DataStream ) datatype ChunksInput = | ChunksInput ( - nameonly bytesIn: DataStream , + nameonly bytesIn: DataStream , nameonly chunkSize: CountingInteger ) datatype ChunksOutput = | ChunksOutput ( - nameonly bytesOut: DataStream + nameonly bytesOut: DataStream ) datatype CountBitsInput = | CountBitsInput ( - nameonly bits: DataStream + nameonly bits: DataStream ) datatype CountBitsOutput = | CountBitsOutput ( nameonly sum: int32 @@ -79,12 +79,8 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes returns (output: Result) requires && ValidState() - // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` part. - requires - && input.bits.Valid() && History !in input.bits.Repr modifies Modifies - {History} , - History`CountBits , - input.bits.Repr + History`CountBits // Dafny will skip type parameters when generating a default decreases clause. decreases Modifies - {History} ensures @@ -106,8 +102,6 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes && ValidState() ensures BinaryOfEnsuresPublicly(input, output) ensures History.BinaryOf == old(History.BinaryOf) + [DafnyCallEvent(input, output)] - // TODO: smithy-dafny isn't yet generating this - ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) predicate ChunksEnsuresPublicly(input: ChunksInput , output: Result) // The public method to be called by library consumers @@ -123,14 +117,12 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes && ValidState() ensures ChunksEnsuresPublicly(input, output) ensures History.Chunks == old(History.Chunks) + [DafnyCallEvent(input, output)] - // TODO: smithy-dafny isn't yet generating this - ensures output.Success? ==> output.value.bytesOut.Valid() && fresh(output.value.bytesOut.Repr) } datatype SimpleStreamingConfig = | SimpleStreamingConfig ( ) - type StreamingBlob = DataStream + type StreamingBlob = DataStream datatype Error = // Local Error structures are listed here | OverflowError ( @@ -180,7 +172,7 @@ abstract module AbstractSimpleStreamingService import opened Wrappers import opened StandardLibrary.UInt import opened UTF8 - import opened Std.Streams + import opened StandardLibrary.Streams import opened Types = SimpleStreamingTypes import Operations : AbstractSimpleStreamingOperations function method DefaultSimpleStreamingConfig(): SimpleStreamingConfig @@ -220,12 +212,8 @@ abstract module AbstractSimpleStreamingService returns (output: Result) requires && ValidState() - // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` parts. - requires - && input.bits.Valid() && History !in input.bits.Repr modifies Modifies - {History} , - History`CountBits , - input.bits.Repr + History`CountBits // Dafny will skip type parameters when generating a default decreases clause. decreases Modifies - {History} ensures @@ -233,10 +221,6 @@ abstract module AbstractSimpleStreamingService ensures CountBitsEnsuresPublicly(input, output) ensures History.CountBits == old(History.CountBits) + [DafnyCallEvent(input, output)] { - // TODO: It's not clear how to actually ensure this, - // since the internal config is not visible to the trait - // so it can't really be a precondition there. - assume {:axiom} Operations.ModifiesInternalConfig(config) !! input.bits.Repr; output := Operations.CountBits(config, input); History.CountBits := History.CountBits + [DafnyCallEvent(input, output)]; } @@ -256,8 +240,6 @@ abstract module AbstractSimpleStreamingService && ValidState() ensures BinaryOfEnsuresPublicly(input, output) ensures History.BinaryOf == old(History.BinaryOf) + [DafnyCallEvent(input, output)] - // TODO: smithy-dafny isn't yet generating this - ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) { output := Operations.BinaryOf(config, input); History.BinaryOf := History.BinaryOf + [DafnyCallEvent(input, output)]; @@ -289,7 +271,7 @@ abstract module AbstractSimpleStreamingOperations { import opened Wrappers import opened StandardLibrary.UInt import opened UTF8 - import opened Std.Streams + import opened StandardLibrary.Streams import opened Types = SimpleStreamingTypes type InternalConfig predicate ValidInternalConfig?(config: InternalConfig) @@ -301,12 +283,8 @@ abstract module AbstractSimpleStreamingOperations { method CountBits ( config: InternalConfig , input: CountBitsInput ) returns (output: Result) requires - // TODO: smithy-dafny isn't yet generating the `input.bits.Valid()` part. && ValidInternalConfig?(config) - && input.bits.Valid() - && ModifiesInternalConfig(config) !! input.bits.Repr - // TODO: smithy-dafny isn't yet generating the `input.bits.Repr` part. - modifies ModifiesInternalConfig(config), input.bits.Repr + modifies ModifiesInternalConfig(config) // Dafny will skip type parameters when generating a default decreases clause. decreases ModifiesInternalConfig(config) ensures @@ -328,8 +306,6 @@ abstract module AbstractSimpleStreamingOperations { ensures && ValidInternalConfig?(config) ensures BinaryOfEnsuresPublicly(input, output) - // TODO: smithy-dafny isn't yet generating this - ensures output.Success? ==> output.value.binary.Valid() && fresh(output.value.binary.Repr) predicate ChunksEnsuresPublicly(input: ChunksInput , output: Result) @@ -340,8 +316,6 @@ abstract module AbstractSimpleStreamingOperations { returns (output: Result) requires && ValidInternalConfig?(config) - // TODO: smithy-dafny isn't yet generating this - && input.bytesIn.Valid() modifies ModifiesInternalConfig(config) // Dafny will skip type parameters when generating a default decreases clause. decreases ModifiesInternalConfig(config) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 2440fca52..73483dbec 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -11,7 +11,7 @@ module {:options "--function-syntax:4"} Chunker { import opened Std.BulkActions import opened Std.Producers import opened Std.Consumers - import opened Std.Streams + import opened StandardLibrary.Streams @AssumeCrossModuleTermination class Chunker extends BulkAction>, Option>>> { @@ -143,7 +143,7 @@ module {:options "--function-syntax:4"} Chunker { } - method ChunkingStream(chunkSize: CountingInteger, s: DataStream) + method ChunkingStream(chunkSize: CountingInteger, s: DataStream) requires s.Valid() requires s.history == [] { diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index 647c17b5a..6d4ec6287 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -62,7 +62,8 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl // TODO: for now assume {:axiom} input.bytesIn.history == []; var chunker := new Chunker(input.chunkSize); - var chunkerStream := new MappedDataStream(input.bytesIn, chunker); + ghost var chunkerTotalProof := new ChunkerTotalProof(chunker); + var chunkerStream := new MappedDataStream(input.bytesIn, chunker, chunkerTotalProof); return Success(ChunksOutput(bytesOut := chunkerStream)); } diff --git a/TestModels/aws-sdks/s3/dfyconfig.toml b/TestModels/aws-sdks/s3/dfyconfig.toml new file mode 100644 index 000000000..6c9348c16 --- /dev/null +++ b/TestModels/aws-sdks/s3/dfyconfig.toml @@ -0,0 +1,10 @@ +# Project file for easier development in VS Code. +# Not yet used in the actual Makefile build. + +[options] +unicode-char = false +function-syntax = 3 + +library = [ + "../../dafny-dependencies/StandardLibrary/bin/DafnyStandardLibraries-smithy-dafny-subset.doo" +] diff --git a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy index 7663aae4e..1192a5115 100644 --- a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy +++ b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy @@ -5,11 +5,14 @@ include "../src/Index.dfy" module TestComAmazonawsS3 { import Com.Amazonaws.S3 + import opened ComAmazonawsS3Types import opened StandardLibrary.UInt + import opened StandardLibrary.Streams import opened Wrappers - import opened Std.Enumerators - import opened Std.Aggregators - import opened Std.Streams + import opened Std.BulkActions + import opened Std.Producers + import opened Std.Consumers + const testBucket := "s3-dafny-test-bucket" const testObjectKey := "smithy-dafny-test-model-object-key" @@ -23,8 +26,7 @@ module TestComAmazonawsS3 { ); // Note the chunk size has to ensure all but the last chunk is >= 8192 bytes. // For a small stream like this that means just one chunk. - var s: ByteStream := new SeqByteStream([ 97, 115, 100, 102 ], 10); - expect s is RewindableByteStream; + var s: DataStream := new SeqDataStream([ 97, 115, 100, 102 ]); PutObjectTest( input := S3.Types.PutObjectRequest( Bucket := testBucket, @@ -68,11 +70,7 @@ module TestComAmazonawsS3 { var MyBody := ret.value.Body; expect MyBody.Some?; - // TODO: These need to be generated as postconditions on GetObject instead - assume {:axiom} fresh(MyBody.value.Repr); - assume {:axiom} MyBody.value.Valid(); - - var bodyValue := Collect(MyBody.value); + var bodyValue :- expect Collect(MyBody.value); expect bodyValue == expectedBody; } @@ -115,12 +113,13 @@ module TestComAmazonawsS3 { expect(ret.Success?); } - method Collect(e: ByteStream) returns (s: BoundedInts.bytes) - requires e.Valid() - modifies e.Repr + method Collect(e: DataStream) returns (s: Result) + { - var a := new Collector(); - ForEach(e, a); - return Seq.Flatten(a.values); + var reader := e.Reader(); + var a: BatchSeqWriter := new BatchSeqWriter(); + var aTotalProof := new BatchSeqWriterTotalProof(a); + reader.ForEach(a, aTotalProof); + return Success(a.elements); } } diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml index ead8c694b..e9186821f 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml +++ b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml @@ -24,13 +24,19 @@ includes = [ track-print-effects = true enforce-determinism = false reads-clauses-on-methods = true +type-system-refresh = true +general-traits = "datatype" +general-newtypes = true # Options that differ from Dafny's unicode-char = false -# TODO +# TODO: Required because this file isn't named dfyconfig.toml :P allow-warnings = true + +hidden-no-verify = true + # Options important for sustainable development # of the libraries themselves. resource-limit = 1000000 diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index d577f7ecd..c348e1846 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -1,8 +1,8 @@ from _dafny import Seq from smithy_python.interfaces.blobs import ByteStream -from smithy_dafny_standard_library.internaldafny.generated.Std_Streams import DataStream as DafnyDataStream, RewindableDataStream as DafnyRewindableDataStream -from smithy_dafny_standard_library.internaldafny.generated.Std_Enumerators import Enumerator +from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream +from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None # Adaptor classes for wrapping up Python-native types as their @@ -13,47 +13,42 @@ class DafnyDataStreamAsByteStream(ByteStream): """Wrapper class adapting a Dafny DataStream as a native ByteStream.""" - def __init__(self, dafny_data_stream): - self.dafny_data_stream = dafny_data_stream + def __init__(self, data_stream): + self.data_stream = data_stream + self.reader = data_stream.Reader() def read(self, size: int = -1) -> bytes: - next = None - while next is None or (next.is_Some and len(next.value) == 0): - if size == -1: - next = self.dafny_data_stream.Next() - else: - next = self.dafny_byte_stream.Read(size) - - # Do NOT return None, because that indicates "no data right now, might be more later" - return bytes(next.value) if next.is_Some else bytes() - - -class DafnyRewindableDataStreamAsByteStream(DafnyByteStreamAsByteStream): - """Wrapper class adapting a Dafny RewindableDataStream as a native ByteStream - that supports tell and seek. - """ - - def __init__(self, dafny_data_stream): - if not isinstance(dafny_data_stream, DafnyRewindableDataStream): - raise ValueError("Rewindable stream required") - super().__init__(dafny_data_stream) + if size == -1: + writer = new BatchSeqWriter() + self.reader.ForEach(writer) + else: + writer = new BatchArrayWriter(size) + self.reader.ForEachToCapacity(writer) + # TODO: Check for errors. Fine to ignore EOI though. + return bytes(writer.elements) def tell(self) -> int: - return self.dafny_data_stream.Position() + return self.reader.ProducedCount() def seek(self, offset, whence=0): + # TODO: check whether invalid offsets must raise errors match whence: case 0: - position = offset + new_position = offset case 1: - position = self.dafny_data_stream.Position() + offset + new_position = self.reader.Position() + offset case 2: - position = self.dafny_data_stream.totalLength.value + offset - return self.dafny_data_stream.Seek(position) + new_position = self.data_stream.ContentLength().value + offset + + if position > self.reader.Position(): + self.reader.ForEach(new IgnoreNConsumer(position - self.reader.Position())) + else if position < self.reader.Position(): + self.reader = data_stream.Reader() + self.reader.ForEach(new IgnoreNConsumer(position)) class StreamingBlobAsDafnyDataStream(DafnyDataStream): - """Wrapper class adapting a native StreamingBlob as a Dafny ByteStream.""" + """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" def __init__(self, streaming_blob): self.streaming_blob = streaming_blob @@ -62,15 +57,26 @@ def Next(self): return Producer.Next(self) def Invoke(self, _) -> Option: - next = self.streaming_blob.read() + # TODO: error handling + next = self.streaming_blob.read(1) if next: - return Option_Some(Seq(next)) + return Option_Some(Option_Some(Result_Success(next))) else: return Option_None() - def Read(self, size) -> Option: + def ProducedCount(self): + return self.streaming_blob.position + + def ForEach(self, consumer): + # TODO: error handling + while next = self.streaming_blob.read(4096): + batch = new BatchReader(Seq(next)) + batch.ForEach(consumer) + batch.Accept(Option_None) + + def ForEachToCapacity(self, consumer): + # TODO: error handling + size = consumer.Capacity() next = self.streaming_blob.read(size) - if next: - return Option_Some(Seq(next)) - else: - return Option_None() + batch = new BatchReader(Seq(next)) + batch.ForEachToCapacity(consumer) diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/StandardLibrary.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/StandardLibrary.dfy index 75119a391..8e9efdabd 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/StandardLibrary.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/StandardLibrary.dfy @@ -341,7 +341,7 @@ module StandardLibrary { ensures forall i :: 0 <= i < |q| ==> q[i] in s ensures forall k :: k in s ==> k in q ensures forall i :: 0 < i < |q| ==> LexicographicLessOrEqual(q[i-1], q[i], less) - ensures forall i, j | 0 <= i < j < |q| :: q[i] != q[j]; + ensures forall i, j | 0 <= i < j < |q| :: q[i] != q[j] { if s == {} then [] diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index e7b8278c1..2baf94425 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -3,722 +3,56 @@ * SPDX-License-Identifier: MIT *******************************************************************************/ -module {:options "--function-syntax:4"} Std.Streams { +module {:options "--function-syntax:4"} StandardLibrary.Streams { - import opened Wrappers - import opened Actions - import opened Producers - import opened BoundedInts - import opened Collections.Seq - import opened Termination - - // TODO: Consider a more generic term for Streamed[Of], - // especially if we can decouple the filtering out Failures - // from the concatenation of batches. - // TODO: Add this to the actual Dafny standard library, - // since it's generic enough. + import opened Std.Wrappers + import opened Std.Actions + import opened Std.BulkActions + import opened Std.Producers + import opened Std.BoundedInts + import opened Std.Collections.Seq + import opened Std.Termination // // A data stream, i.e. a fallable producer of batches of values. // - // Allows for streams that can only be read once, - // but see RewindableDataStream for a more specific trait - // that requires the ability to replay the data, - // or seek to an arbitrary position (although this may take linear time). - // That requirement is not in this trait - // because there are lots of ways to implement a stream - // where having to replay forces buffering all previous values in memory, - // which often defeats the purpose of streaming in the first place. - // In particular, boto3 currently (quite implicitly) - // requires file-like streams with the ability to seek, - // but we don't want to force the same requirements on all streams. - // - @AssumeCrossModuleTermination - trait DataStream extends Producer, E>> { - - // The total length of all produced batches - const totalLength: Option + trait DataStream { - ghost predicate Valid() - reads this, Repr - ensures Valid() ==> this in Repr - ensures Valid() ==> ValidHistory(history) - decreases Repr, 0 + function ContentLength(): Option - ghost predicate ValidOutputs(outputs: seq, E>>>) - requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) - decreases Repr - - ghost function Streamed(): seq - requires Valid() - reads this, Repr - { - StreamedOf(Outputs()) - } - - method Read(max: uint64) returns (r: Option, E>>) - requires Requires(()) - // reads Reads(()) - modifies Modifies(()) - decreases Decreases(()) - ensures Ensures((), r) - ensures 0 < max ==> RemainingDecreasedBy(r) - ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int + method Reader() returns (p: Producer>) + ensures + && p.Valid() + && fresh(p.Repr) + && p.history == [] + && (ContentLength().Some? ==> p.Remaining() == Some(ContentLength().value as int + 1)) } - ghost function StreamedOf(outputs: seq, E>>>): seq - { - // This could be defined as something like this instead: - // - // Flatten(MapPartialFunction(x => x.value, Filter(x => x.Success?, ProducedOf(outputs)))) - // - // But then the lack of extensionality in Dafny hits hard, - // and it becomes impossible to expand the definition elsewhere. - - if |outputs| == 0 then - [] - else if outputs[0].Some? && outputs[0].value.Success? then - outputs[0].value.value + StreamedOf(outputs[1..]) - else - StreamedOf(outputs[1..]) - } + class SeqDataStream extends DataStream { - ghost predicate ValidStreamed(outputs: seq, E>>>, length: int) - requires Partitioned(outputs, IsSome) - { - && var streamed := StreamedOf(outputs); - && |streamed| <= length - && (!Seq.All(outputs, IsSome) ==> |streamed| == length) - } - - lemma StreamedOfSingleton(s: seq) - ensures - var singleton: Option, E>> := Some(Success(s)); - StreamedOf([singleton]) == s - {} - - lemma StreamedOfComposition(left: seq, E>>>, right: seq, E>>>) - ensures StreamedOf(left + right) == StreamedOf(left) + StreamedOf(right) - { - if left == [] { - assert left + right == right; - } else { - StreamedOfComposition(left[1..], right); - assert left == [left[0]] + left[1..]; - assert left + right == [left[0]] + (left[1..] + right); - } - } - - lemma ValidStreamedAfterNone(outputs: seq, E>>>, length: int) - requires Partitioned(outputs, IsSome) - requires |StreamedOf(outputs)| == length - ensures Partitioned(outputs + [None], IsSome) - ensures ValidStreamed(outputs + [None], length) - { - var right: seq, E>>> := [None]; - PartitionedCompositionRight(outputs, [None], IsSome); - assert Partitioned(outputs + [None], IsSome); - var streamedAfter := StreamedOf(outputs + [None]); - assert ProducedOf(right) == []; - StreamedOfComposition(outputs, right); - } - - lemma ValidStreamedAfterMoreData(outputs: seq, E>>>, value: seq, length: int) - requires All(outputs, IsSome) - requires - (AllImpliesPartitioned(outputs, IsSome); - |StreamedOf(outputs)| + |value| <= length) - requires ValidStreamed(outputs, length) - ensures Partitioned(outputs + [Some(Success(value))], IsSome) - ensures ValidStreamed(outputs + [Some(Success(value))], length) - ensures StreamedOf(outputs + [Some(Success(value))]) == StreamedOf(outputs) + value - { - AllImpliesPartitioned(outputs, IsSome); - var right: seq, E>>> := [Some(Success(value))]; - assert All(right, IsSome); - PartitionedCompositionLeft(outputs, right, IsSome); - assert ProducedOf(right) == [Success(value)]; - reveal Seq.Map(); - assert StreamedOf(right) == Flatten([value]); - reveal Seq.Flatten(); - assert StreamedOf(right) == value + Flatten([]); - assert StreamedOf(right) == value; - StreamedOfComposition(outputs, right); - } + const s: seq - lemma ValidStreamedAfterFailure(outputs: seq, E>>>, error: E, length: int) - requires All(outputs, IsSome) - requires - (AllImpliesPartitioned(outputs, IsSome); - |StreamedOf(outputs)| <= length) - requires ValidStreamed(outputs, length) - ensures Partitioned(outputs + [Some(Failure(error))], IsSome) - ensures ValidStreamed(outputs + [Some(Failure(error))], length) - ensures StreamedOf(outputs + [Some(Failure(error))]) == StreamedOf(outputs) - { - AllImpliesPartitioned(outputs, IsSome); - var right: seq, E>>> := [Some(Failure(error))]; - assert All(right, IsSome); - PartitionedCompositionLeft(outputs, right, IsSome); - assert ProducedOf(right) == [Failure(error)]; - reveal Seq.Map(); - assert StreamedOf(right) == []; - StreamedOfComposition(outputs, right); - } - - method Drain(p: Producer) - requires p.Valid() - // reads p.Repr - modifies p.Repr - ensures p.ValidAndDisjoint() - ensures p.Done() - ensures old(p.RemainingMetric()).NonIncreasesTo(p.RemainingMetric()) - { - while true - invariant fresh(p.Repr - old(p.Repr)) - invariant p.ValidAndDisjoint() - decreases p.Remaining() + constructor (s: seq) + ensures this.s == s { - var n := p.Next(); - if n.None? { break; } - } - assert Last(p.Outputs()) == None; - } - - trait RewindableDataStream extends DataStream { - - ghost const data: seq - - function Position(): (res: uint64) - requires Valid() - reads this, Repr - ensures res as int <= |data| - - method Seek(newPosition: uint64) - requires Valid() - requires newPosition as int <= |data| - modifies Repr - ensures Valid() - ensures Position() == newPosition - } - - trait ProducesTotalLengthProof { - - const producer: Producer, E>> - const length: int - - lemma ProducesTotalLength(history: seq<((), Option, E>>)>) - requires producer.ValidHistory(history) - ensures ValidStreamed(OutputsOf(history), length) - } - - /* - * Wraps an Producer up as a non-rewindable DataStream. - * It implements Read() using Next(), and buffers extra data as needed. - */ - class ProducerDataStream extends DataStream { - - const wrapped: Producer, E>> - const length: uint64 - var buffer: seq - - ghost const producesTotalLengthProof: ProducesTotalLengthProof - - ghost predicate Valid() - reads this, Repr - ensures Valid() ==> this in Repr - ensures Valid() ==> ValidHistory(history) - decreases Repr, 0 - { - && this in Repr - && ValidComponent(wrapped) - && ValidHistory(history) - && producesTotalLengthProof.producer == wrapped - && producesTotalLengthProof.length == length as int - && totalLength == Some(length) - && |buffer| <= length as int - && StreamedOf(Outputs()) + buffer == StreamedOf(wrapped.Outputs()) - && (0 < |buffer| ==> !wrapped.Done()) - && (!Done() <==> !wrapped.Done()) - } - - ghost predicate ValidOutputs(outputs: seq, E>>>) - requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) - decreases Repr - { - totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) - } - - ghost function RemainingMetric(): TerminationMetric - requires Valid() - reads this, Repr - decreases Repr, 3 - { - TMTuple(TMTop, wrapped.RemainingMetric(), TMNat(|buffer|)) - } - - constructor(wrapped: Producer, E>>, length: uint64, ghost producesTotalLengthProof: ProducesTotalLengthProof) - requires wrapped.Valid() - requires wrapped.history == [] - requires producesTotalLengthProof.producer == wrapped - requires producesTotalLengthProof.length == length as int - ensures Valid() - ensures fresh(Repr - wrapped.Repr) - { - this.wrapped := wrapped; - this.length := length; - this.buffer := []; - - this.totalLength := Some(length); - this.history := []; - this.Repr := {this} + wrapped.Repr; - this.producesTotalLengthProof := producesTotalLengthProof; - } - - method Invoke(i: ()) returns (r: Option, E>>) - requires Requires(i) - // reads this, Repr - modifies Modifies(i) - decreases Decreases(i), 0 - ensures Ensures(i, r) - ensures RemainingDecreasedBy(r) - { - assert Requires(()); - assert Valid(); - if length == 0 { - r := None; - - producesTotalLengthProof.ProducesTotalLength(wrapped.history); - assert |StreamedOf(wrapped.Outputs())| == 0; - PartitionedCompositionRight(Outputs(), [None], IsSome); - StreamedOfComposition(Outputs(), [None]); - assert OutputsOf(history + [((), None)]) == Outputs() + [None]; - - OutputsPartitionedAfterOutputtingNone(); - ProduceNone(); - - Drain(wrapped); - Repr := {this} + wrapped.Repr; - producesTotalLengthProof.ProducesTotalLength(wrapped.history); - assert Last(Outputs()) == None; - assert Done(); - assert Valid(); - - reveal TerminationMetric.Ordinal(); - old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); - } else { - r := Read(length); - } - } - - @IsolateAssertions - method Read(max: uint64) returns (r: Option, E>>) - requires Requires(()) - // reads Reads(()) - modifies Modifies(()) - decreases Decreases(()) - ensures Ensures((), r) - ensures 0 < max ==> RemainingDecreasedBy(r) - ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int - { - assert Requires(()); - - assert Valid(); - var next; - if 0 < |buffer| { - assert !wrapped.Done(); - next := Some(Success(buffer)); - StreamedOfSingleton(buffer); - buffer := []; - - assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); - } else { - next := wrapped.Next(); - Repr := {this} + wrapped.Repr; - producesTotalLengthProof.ProducesTotalLength(wrapped.history); - - assert wrapped.Outputs() == old(wrapped.Outputs()) + [next]; - StreamedOfComposition(old(wrapped.Outputs()), [next]); - assert StreamedOf(wrapped.Outputs()) == StreamedOf(old(wrapped.Outputs())) + StreamedOf([next]); - assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); - } - assert StreamedOf(Outputs()) + StreamedOf([next]) == StreamedOf(wrapped.Outputs()); - producesTotalLengthProof.ProducesTotalLength(wrapped.history); - assert |StreamedOf(wrapped.Outputs())| <= length as int; - - if next.None? { - r := None; - - ghost var right := [next]; - assert Last(wrapped.Outputs()) == next; - assert wrapped.Outputs() == old(wrapped.Outputs()) + right; - assert StreamedOf(wrapped.Outputs()) == StreamedOf(old(wrapped.Outputs()) + right); - assert StreamedOf(right) == []; - StreamedOfComposition(old(wrapped.Outputs()), right); - assert old(|StreamedOf(wrapped.Outputs())|) == old(|StreamedOf(Outputs())|); - assert StreamedOf(wrapped.Outputs()) == old(StreamedOf(wrapped.Outputs())) + []; - assert old(|StreamedOf(wrapped.Outputs())|) == |StreamedOf(wrapped.Outputs())|; - assert |StreamedOf(wrapped.Outputs())| == length as int; - assert |StreamedOf(Outputs())| == length as int; - ValidStreamedAfterNone(Outputs(), length as int); - OutputsPartitionedAfterOutputtingNone(); - - assert OutputsOf(history + [((), None)]) == Outputs() + [None]; - assert ValidHistory(history + [((), None)]); - ProduceNone(); - - assert Last(wrapped.Outputs()) == None; - assert Last(Outputs()) == None; - - assert StreamedOf([r]) == []; - StreamedOfComposition(old(Outputs()), [r]); - - assert Valid(); - } else { - - assert Last(wrapped.Outputs()).Some?; - PartitionedLastTrueImpliesAll(wrapped.Outputs(), IsSome); - - assert !wrapped.Done(); - wrapped.DoneIsOneWay(); - assert old(!wrapped.Done()); - assert old(!Done()); - assert !Done(); - assert All(Outputs(), IsSome); - - if next.value.Failure? { - r := next; - - assert old(Valid()); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - OutputsPartitionedAfterOutputtingSome(r.value); - PartitionedCompositionLeft(Outputs(), [r], IsSome); - StreamedOfComposition(Outputs(), [r]); - assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); - - assert |StreamedOf(wrapped.Outputs())| <= length as int; - ValidStreamedAfterFailure(Outputs(), next.value.error, length as int); - } else { - var size := if max <= |next.value.value| as uint64 then max else |next.value.value| as uint64; - var result := next.value.value[..size]; - var value := Success(result); - r := Some(value); - buffer := next.value.value[size..]; - - assert next.value.value == next.value.value[..size] + next.value.value[size..]; - StreamedOfSingleton(r.value.value); - StreamedOfSingleton(next.value.value); - assert StreamedOf([r]) + buffer == StreamedOf([next]); - - assert StreamedOf(Outputs()) + StreamedOf([r]) + buffer == StreamedOf(wrapped.Outputs()); - - assert old(Valid()); - assert StreamedOf(old(Outputs())) + old(buffer) == old(StreamedOf(wrapped.Outputs())); - OutputsPartitionedAfterOutputtingSome(value); - PartitionedCompositionLeft(Outputs(), [r], IsSome); - StreamedOfComposition(Outputs(), [r]); - assert StreamedOf(Outputs() + [r]) + buffer == StreamedOf(wrapped.Outputs()); - - assert |StreamedOf(wrapped.Outputs())| <= length as int; - ValidStreamedAfterMoreData(Outputs(), result, length as int); - } - - assert ValidOutputs(Outputs() + [r]); - assert OutputsOf(history + [((), r)]) == Outputs() + [r]; - assert ValidHistory(history + [((), r)]); - ProduceSome(r.value); - - assert Last(wrapped.Outputs()).Some?; - assert Last(Outputs()).Some?; - - assert Valid(); - } - - assert Valid(); - if 0 < max { - reveal TerminationMetric.Ordinal(); - if r.Some? { - old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); - } else { - old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); - } - } - assert 0 < max ==> RemainingDecreasedBy(r); - - } - } - - /* - * Rewindable stream of a sequence with a configured default chunk size. - */ - class SeqDataStream extends RewindableDataStream { - - const s: seq - const chunkSize: uint64 - var position: uint64 - - ghost predicate Valid() - reads this, Repr - ensures Valid() ==> this in Repr - ensures Valid() ==> ValidHistory(history) - ensures Valid() ==> |data| <= UINT64_MAX as int - decreases Repr, 0 - { - && this in Repr - && ValidHistory(history) - && s == data - && |s| <= UINT64_MAX as int - && totalLength == Some(|s| as uint64) - && position as int <= |s| - && 0 < chunkSize - && StreamedOf(Outputs()) == s[..position] - } - - ghost predicate ValidOutputs(outputs: seq, E>>>) - requires Seq.Partitioned(outputs, IsSome) - ensures ValidOutputs(outputs) && totalLength.Some? ==> ValidStreamed(outputs, totalLength.value as int) - decreases Repr - { - && totalLength.Some? - && ValidStreamed(outputs, totalLength.value as int) - } - - ghost function RemainingMetric(): TerminationMetric - requires Valid() - reads this, Repr - decreases Repr, 3 - { - TMNat(|s| - position as int) - } - - constructor(s: seq, chunkSize: uint64) - requires |s| <= UINT64_MAX as int - requires 0 < chunkSize - ensures Valid() - { - this.data := s; this.s := s; - this.position := 0; - this.chunkSize := chunkSize; - this.totalLength := Some(|s| as uint64); - - this.history := []; - this.Repr := {this}; - } - - function Position(): (res: uint64) - requires Valid() - reads this, Repr - ensures res as int <= |data| - { - position - } - - method Seek(newPosition: uint64) - requires Valid() - requires newPosition as int <= |data| - modifies Repr - ensures Valid() - ensures Position() == newPosition - { - position := newPosition; - if position == 0 { - history := []; - } else { - history := [((), Some(Success(s[..position])))]; - } - - var outputs := Outputs(); - var produced := ProducedOf(outputs); - var streamed := StreamedOf(outputs); - if outputs == [] { - assert produced == []; - } else { - assert outputs == [Some(Success(s[..position]))]; - StreamedOfSingleton(s[..position]); - assert ValidHistory(history); - } } - method Invoke(t: ()) returns (r: Option, E>>) - requires Requires(t) - // reads this, Repr - modifies Modifies(t) - decreases Decreases(t), 0 - ensures Ensures(t, r) - ensures RemainingDecreasedBy(r) - { - assert Valid(); - r := Read(chunkSize); + function ContentLength(): Option { + Some(|s|) } - method Read(max: uint64) returns (r: Option, E>>) - requires Requires(()) - // reads this, Repr - modifies Modifies(()) - decreases Decreases(()) - ensures Ensures((), r) - ensures 0 < max ==> RemainingDecreasedBy(r) - ensures r.Some? && r.value.Success? ==> |r.value.value| <= max as int + method Reader() returns (p: Producer>) + ensures + && p.Valid() + && fresh(p.Repr) + && p.history == [] + && (ContentLength().Some? ==> p.Remaining() == Some(ContentLength().value + 1)) { - assert Requires(()); - - assert Valid(); - if position == |s| as uint64 { - r := None; - - ValidStreamedAfterNone(Outputs(), totalLength.value as int); - assert OutputsOf(history + [((), None)]) == Outputs() + [None]; - assert ValidHistory(history + [((), None)]); - ProduceNone(); - - assert StreamedOf([r]) == []; - StreamedOfComposition(old(Outputs()), [r]); - assert StreamedOf(Outputs()) == s[..position]; - } else { - var remaining := |s| as uint64 - position; - var size := if max <= remaining then max else remaining; - var newPosition := position + size; - var chunk := Success(s[position..newPosition]); - r := Some(chunk); - position := newPosition; - - ValidStreamedAfterMoreData(Outputs(), chunk.value, totalLength.value as int); - assert OutputsOf(history + [((), r)]) == Outputs() + [r]; - assert ValidHistory(history + [((), r)]); - ProduceSome(chunk); - - calc { - StreamedOf(Outputs()); - old(StreamedOf(Outputs())) + chunk.value; - s[..old(position)] + chunk.value; - s[..old(position)] + s[old(position)..position]; - s[..position]; - } - assert StreamedOf(Outputs()) == s[..position]; - } - - reveal TerminationMetric.Ordinal(); - assert Valid(); + var data := new BatchReader(s); + var eoi := new SeqReader([None]); + p := new ConcatenatedProducer(data, eoi); } } - // TODO: better name? - // TODO: Document the fact that this doesn't have control - // over the parameter to original.Read(), - // but also that this would be low value since some languages - // will be forced to buffer to obey that parameter anyway (e.g. Java Publishers) - @AssumeCrossModuleTermination - class OptionMappedProducer extends Producer { - - const original: Producer - const mapping: Action, Option> - - ghost const mappingTotalProof: TotalActionProof, Option> - ghost const base: TerminationMetric - - constructor (original: Producer, - mapping: Action, Option>, - ghost mappingTotalProof: TotalActionProof, Option>) - requires original.Valid() - requires original.history == [] - requires mapping.Valid() - requires mapping.history == [] - requires mappingTotalProof.Valid() - requires mappingTotalProof.Action() == mapping - requires original.Repr !! mapping.Repr !! mappingTotalProof.Repr - ensures Valid() - ensures history == [] - ensures fresh(Repr - original.Repr - mapping.Repr - mappingTotalProof.Repr) - { - this.original := original; - this.mapping := mapping; - this.mappingTotalProof := mappingTotalProof; - this.base := TMSucc(original.RemainingMetric()); - - Repr := {this} + original.Repr + mapping.Repr + mappingTotalProof.Repr; - history := []; - - new; - this.base.SuccDecreasesToOriginal(); - } - - ghost predicate Valid() - reads this, Repr - ensures Valid() ==> this in Repr - ensures Valid() ==> ValidHistory(history) - decreases Repr, 0 - { - && this in Repr - && ValidComponent(original) - && ValidComponent(mapping) - && ValidComponent(mappingTotalProof) - && mappingTotalProof.Action() == mapping - && original.Repr !! mapping.Repr !! mappingTotalProof.Repr - && ValidHistory(history) - && base.DecreasesTo(original.RemainingMetric()) - && (!original.Done() <==> !Done()) - } - - ghost predicate ValidOutputs(outputs: seq>) - requires Seq.Partitioned(outputs, IsSome) - decreases Repr - { - true - } - - ghost function RemainingMetric(): TerminationMetric - requires Valid() - reads this, Repr - decreases Repr, 3 - { - TMTuple(base, TMNat(0), original.RemainingMetric()) - } - - @ResourceLimit("1e7") - method Invoke(t: ()) returns (result: Option) - requires Requires(t) - // reads Reads(t) - modifies Modifies(t) - decreases Decreases(t), 0 - ensures Ensures(t, result) - ensures RemainingDecreasedBy(result) - { - assert Requires(t); - assert Valid(); - RemainingMetric().TupleDecreasesToSecond(); - assert Remaining() > original.Remaining(); - var next := original.Next(); - - mappingTotalProof.AnyInputIsValid(mapping.history, next); - var mappedNext := mapping.Invoke(next); - result := mappedNext; - - if mappedNext.Some? { - mappingTotalProof.AnyInputIsValid(mapping.history, next); - var nextValue := mapping.Invoke(next); - result := nextValue; - - OutputsPartitionedAfterOutputtingSome(mappedNext.value); - ProduceSome(mappedNext.value); - } else { - original.OutputtingNoneMeansNotAllSome(); - OutputsPartitionedAfterOutputtingNone(); - ProduceNone(); - - assert !IsSome(Seq.Last(Outputs())); - } - - Repr := {this} + original.Repr + mapping.Repr + mappingTotalProof.Repr; - assert Valid(); - assert original.RemainingDecreasedBy(next); - if next.Some? { - old(RemainingMetric()).TupleDecreasesToTuple(RemainingMetric()); - } else { - old(RemainingMetric()).TupleNonIncreasesToTuple(RemainingMetric()); - } - } - } } \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/UTF8.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/UTF8.dfy index 7a29161a2..734853de0 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/UTF8.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/UTF8.dfy @@ -22,7 +22,7 @@ module {:extern "UTF8"} UTF8 { // The tradeoff of assuming the external implementation of encode and decode is correct is worth the tradeoff // of unlocking being able to express and hence prove so many other specifications - function method {:extern "Encode"} Encode(s: string): (res: Result) + function method {:extern "Encode"} {:axiom} Encode(s: string): (res: Result) // US-ASCII only needs a single UTF-8 byte per character ensures IsASCIIString(s) ==> res.Success? && |res.value| == |s| // The following MUST be true for any correct implementation of Encode @@ -30,7 +30,7 @@ module {:extern "UTF8"} UTF8 { ensures res.Success? ==> Decode(res.value).Success? && Decode(res.value).value == s // Decode return a Result, therefore doesn't need to require utf8 input - function method {:extern "Decode"} Decode(b: seq): (res: Result) + function method {:extern "Decode"} {:axiom} Decode(b: seq): (res: Result) ensures res.Success? ==> ValidUTF8Seq(b) // The next four functions are for the benefit of the extern implementation to call, diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 4285f1057..f2b5de533 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 4285f10572ecf5e6a023cf26ae218b44554eb298 +Subproject commit f2b5de533ce30d8f62063f00ccc5db322bb1b0e3 diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index 99aa9d6c6..8dcdcc065 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -321,8 +321,8 @@ public TokenTree generateBlobTypeDefinition(final ShapeId blobShapeId) { .map(DafnyApiCodegen::generateLengthConstraint); if (blobShape.hasTrait(StreamingTrait.class)) { // TODO: need to handle @length too, - // something like `a.totalLength.Some? && min <= a.contentLength.value <= max - return generateTypeSynonym(blobShapeId, "DataStream"); + // something like `a.ContentLength().Some? && min <= a.ContentLength().value <= max + return generateTypeSynonym(blobShapeId, "DataStream"); } else { return generateSubsetType(blobShapeId, "seq", lengthConstraint); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index e3f6bebe8..c03388517 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -90,7 +90,7 @@ public String baseTypeForShape(final ShapeId shapeId) { MAP -> dafnyModulePrefixForShape(shape) + shapeName; case BLOB -> { if (shape.hasTrait(StreamingTrait.class)) { - yield "DataStream"; + yield "DataStream"; } else { yield dafnyModulePrefixForShape(shape) + shapeName; } @@ -386,7 +386,7 @@ public static Stream modulePreludeStandardImports( .stream() .anyMatch(s -> s.hasTrait(StreamingTrait.class)) ) { - return Streams.concat(basics, Stream.of("import opened Std.Streams")); + return Streams.concat(basics, Stream.of("import opened StandardLibrary.Streams")); } else { return basics; } From c39d9ea6301aacf06715813b0c301b3758e2b49d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 25 Apr 2025 10:58:48 -0700 Subject: [PATCH 20/72] Progress --- .../internaldafny/extern/streams.py | 32 +++++++++++-------- .../StandardLibrary/src/Streams.dfy | 6 ++-- TestModels/dafny-dependencies/dafny | 2 +- .../DafnyToAwsSdkShapeVisitor.java | 4 +-- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index c348e1846..c6b094b84 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -2,7 +2,8 @@ from _dafny import Seq from smithy_python.interfaces.blobs import ByteStream from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream -from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter +from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_EndOfInput +from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None # Adaptor classes for wrapping up Python-native types as their @@ -19,13 +20,13 @@ def __init__(self, data_stream): def read(self, size: int = -1) -> bytes: if size == -1: - writer = new BatchSeqWriter() + writer = BatchSeqWriter() self.reader.ForEach(writer) else: - writer = new BatchArrayWriter(size) + writer = BatchArrayWriter(size) self.reader.ForEachToCapacity(writer) # TODO: Check for errors. Fine to ignore EOI though. - return bytes(writer.elements) + return bytes(writer.Values()) def tell(self) -> int: return self.reader.ProducedCount() @@ -41,13 +42,13 @@ def seek(self, offset, whence=0): new_position = self.data_stream.ContentLength().value + offset if position > self.reader.Position(): - self.reader.ForEach(new IgnoreNConsumer(position - self.reader.Position())) - else if position < self.reader.Position(): + self.reader.ForEach(IgnoreNConsumer(position - self.reader.Position())) + elif position < self.reader.Position(): self.reader = data_stream.Reader() - self.reader.ForEach(new IgnoreNConsumer(position)) + self.reader.ForEach(IgnoreNConsumer(position)) -class StreamingBlobAsDafnyDataStream(DafnyDataStream): +class StreamingBlobAsDafnyDataStream(DataStream): """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" def __init__(self, streaming_blob): @@ -60,7 +61,7 @@ def Invoke(self, _) -> Option: # TODO: error handling next = self.streaming_blob.read(1) if next: - return Option_Some(Option_Some(Result_Success(next))) + return Option_Some(Batch_Value(next)) else: return Option_None() @@ -69,14 +70,19 @@ def ProducedCount(self): def ForEach(self, consumer): # TODO: error handling - while next = self.streaming_blob.read(4096): - batch = new BatchReader(Seq(next)) + while True: + next = self.streaming_blob.read(4096) + if not next: + break + batch = BatchReader(Seq(next)) batch.ForEach(consumer) - batch.Accept(Option_None) + + batch.Accept(Batched_EndOfInput) def ForEachToCapacity(self, consumer): # TODO: error handling size = consumer.Capacity() next = self.streaming_blob.read(size) - batch = new BatchReader(Seq(next)) + batch = BatchReader(Seq(next)) batch.ForEachToCapacity(consumer) + # TODO: EOI diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 2baf94425..88029efd4 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -20,7 +20,7 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { function ContentLength(): Option - method Reader() returns (p: Producer>) + method Reader() returns (p: Producer>) ensures && p.Valid() && fresh(p.Repr) @@ -42,7 +42,7 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { Some(|s|) } - method Reader() returns (p: Producer>) + method Reader() returns (p: Producer>) ensures && p.Valid() && fresh(p.Repr) @@ -50,7 +50,7 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { && (ContentLength().Some? ==> p.Remaining() == Some(ContentLength().value + 1)) { var data := new BatchReader(s); - var eoi := new SeqReader([None]); + var eoi := new SeqReader([EndOfInput]); p := new ConcatenatedProducer(data, eoi); } } diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index f2b5de533..9d7a4a957 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit f2b5de533ce30d8f62063f00ccc5db322bb1b0e3 +Subproject commit 9d7a4a957ff697bad5a224efcfbf0deb55335f14 diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java index d757d32e2..67b182fd3 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java @@ -78,9 +78,9 @@ public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( "smithy_dafny_standard_library.internaldafny.extern.streams", - "RewindableDafnyDataStreamAsByteStream" + "DafnyDataStreamAsByteStream" ); - return "RewindableDafnyDataStreamAsByteStream(%1$s)".formatted( + return "DafnyDataStreamAsByteStream(%1$s)".formatted( dataSource ); } else { From ad09a165fbd4f918ae91195882d9d1d4c4344990 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 25 Apr 2025 13:58:10 -0700 Subject: [PATCH 21/72] More fixes --- .../aws-sdks/s3/test/TestComAmazonawsS3.dfy | 2 +- .../internaldafny/extern/streams.py | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy index 1192a5115..5bf0466c9 100644 --- a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy +++ b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy @@ -110,7 +110,7 @@ module TestComAmazonawsS3 { var ret := client.DeleteObject(input); - expect(ret.Success?); + expect(ret.Success?), ret.error; } method Collect(e: DataStream) returns (s: Result) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index c6b094b84..df4a67596 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -1,5 +1,5 @@ -from _dafny import Seq +from _dafny import Seq, Array as DafnyArray from smithy_python.interfaces.blobs import ByteStream from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_EndOfInput @@ -21,11 +21,14 @@ def __init__(self, data_stream): def read(self, size: int = -1) -> bytes: if size == -1: writer = BatchSeqWriter() + writer.ctor__() self.reader.ForEach(writer) else: - writer = BatchArrayWriter(size) + writer = BatchArrayWriter() + writer.ctor__(DafnyArray(None, size)) self.reader.ForEachToCapacity(writer) # TODO: Check for errors. Fine to ignore EOI though. + # print(f"****** {writer.Values()} ******") return bytes(writer.Values()) def tell(self) -> int: @@ -33,21 +36,27 @@ def tell(self) -> int: def seek(self, offset, whence=0): # TODO: check whether invalid offsets must raise errors + # TODO: Need to -1 to account for EndOfInput match whence: case 0: new_position = offset case 1: - new_position = self.reader.Position() + offset + new_position = self.reader.ProducedCount() + offset case 2: new_position = self.data_stream.ContentLength().value + offset - if position > self.reader.Position(): - self.reader.ForEach(IgnoreNConsumer(position - self.reader.Position())) - elif position < self.reader.Position(): - self.reader = data_stream.Reader() - self.reader.ForEach(IgnoreNConsumer(position)) + if new_position > self.reader.ProducedCount(): + consumer = IgnoreNConsumer() + consumer.ctor__(new_position - self.reader.ProducedCount()) + self.reader.ForEach(consumer) + elif new_position < self.reader.ProducedCount(): + self.reader = self.data_stream.Reader() + consumer = IgnoreNConsumer() + consumer.ctor__(new_position) + self.reader.ForEach(consumer) +# TODO: Missing some methods like Remaining() class StreamingBlobAsDafnyDataStream(DataStream): """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" @@ -74,7 +83,8 @@ def ForEach(self, consumer): next = self.streaming_blob.read(4096) if not next: break - batch = BatchReader(Seq(next)) + batch = BatchReader() + batch.ctor__(Seq(next)) batch.ForEach(consumer) batch.Accept(Batched_EndOfInput) @@ -83,6 +93,7 @@ def ForEachToCapacity(self, consumer): # TODO: error handling size = consumer.Capacity() next = self.streaming_blob.read(size) - batch = BatchReader(Seq(next)) + batch = BatchReader() + batch.ctor__(Seq(next)) batch.ForEachToCapacity(consumer) # TODO: EOI From bb27d83ed14ff06f7a17553ee77c750d258e6348 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 25 Apr 2025 16:23:26 -0700 Subject: [PATCH 22/72] PutObject worked! --- .../internaldafny/extern/streams.py | 8 +++++--- TestModels/dafny-dependencies/dafny | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index df4a67596..cf08cf2dc 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -48,15 +48,17 @@ def seek(self, offset, whence=0): if new_position > self.reader.ProducedCount(): consumer = IgnoreNConsumer() consumer.ctor__(new_position - self.reader.ProducedCount()) - self.reader.ForEach(consumer) + self.reader.ForEachToCapacity(consumer) elif new_position < self.reader.ProducedCount(): self.reader = self.data_stream.Reader() consumer = IgnoreNConsumer() consumer.ctor__(new_position) - self.reader.ForEach(consumer) + self.reader.ForEachToCapacity(consumer) + -# TODO: Missing some methods like Remaining() +# TODO: Need to implement Producer, not DataStream! +# TODO: Even so, missing some methods like Remaining() class StreamingBlobAsDafnyDataStream(DataStream): """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 9d7a4a957..0e73df3cb 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 9d7a4a957ff697bad5a224efcfbf0deb55335f14 +Subproject commit 0e73df3cbdcf3e172856658f67c04b4f739871f2 From 591d0ba3fe6e3fe59878fb364e627e47c6fd1536 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 25 Apr 2025 18:22:18 -0700 Subject: [PATCH 23/72] S3 works on Python again! --- .../internaldafny/extern/streams.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index cf08cf2dc..858a6196c 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -2,8 +2,9 @@ from _dafny import Seq, Array as DafnyArray from smithy_python.interfaces.blobs import ByteStream from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream -from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_EndOfInput +from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_EndOfInput, BatchReader from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer +from smithy_dafny_standard_library.internaldafny.generated.Std_Producers import Producer from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None # Adaptor classes for wrapping up Python-native types as their @@ -56,10 +57,20 @@ def seek(self, offset, whence=0): self.reader.ForEachToCapacity(consumer) - -# TODO: Need to implement Producer, not DataStream! -# TODO: Even so, missing some methods like Remaining() class StreamingBlobAsDafnyDataStream(DataStream): + def __init__(self, streaming_blob): + self.streaming_blob = streaming_blob + self.read = False + + def Reader(self): + if self.read: + raise Exception("StreamingBlobAsDafnyDataStream.Reader() called twice") + self.read = True + return StreamingBlobAsDafnyProducer(self.streaming_blob) + + +# TODO: Missing some methods like Remaining() +class StreamingBlobAsDafnyProducer(Producer): """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" def __init__(self, streaming_blob): @@ -89,7 +100,7 @@ def ForEach(self, consumer): batch.ctor__(Seq(next)) batch.ForEach(consumer) - batch.Accept(Batched_EndOfInput) + consumer.Accept(Batched_EndOfInput()) def ForEachToCapacity(self, consumer): # TODO: error handling From 0aa7806adb773bbb60979c6c616af97b24e69b2c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 26 Apr 2025 11:04:17 -0700 Subject: [PATCH 24/72] Starting on Java --- .../runtimes/java/build.gradle.kts | 1 + .../DataStreamAsContentStreamProvider.java | 20 +++++ .../java/Streams/ProviderAsInputStream.java | 73 +++++++++++++++++++ TestModels/dafny-dependencies/dafny | 2 +- 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts index a7b8b1035..7d9411dbe 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts @@ -63,6 +63,7 @@ repositories { dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") + implementation("software.amazon.awssdk:sdk-core:2.28.28") } publishing { publications.create("maven") { diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java new file mode 100644 index 000000000..7a3ce8bb9 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java @@ -0,0 +1,20 @@ +package Streams; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import software.amazon.awssdk.http.ContentStreamProvider; + +import java.io.InputStream; + +public class DataStreamAsContentStreamProvider implements ContentStreamProvider { + + private final DataStream dataStream; + + public DataStreamAsContentStreamProvider(DataStream dataStream) { + this.dataStream = dataStream; + } + + @Override + public InputStream newStream() { + return new ProviderAsInputStream(dataStream.Reader()); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java new file mode 100644 index 000000000..192774f18 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java @@ -0,0 +1,73 @@ +package Streams; + +import Std_Compile.BulkActions_Compile.BatchArrayWriter; +import Std_Compile.BulkActions_Compile.Batched; +import Std_Compile.Consumers_Compile.IgnoreNConsumer; +import Std_Compile.Producers_Compile.Producer; +import Std_Compile.Wrappers_Compile.Option; +import dafny.Array; +import dafny.TypeDescriptor; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; + +public class ProviderAsInputStream extends InputStream { + + private final Producer> producer; + + public ProviderAsInputStream(Producer> producer) { + this.producer = producer; + } + + @Override + public int read() throws IOException { + Option> next = producer.Next(); + if (next.is_Some()) { + Batched batched = next.dtor_value(); + if (batched.is_BatchValue()) { + return next.dtor_value().dtor_value(); + } else if (batched.is_EndOfInput()) { + return -1; + } else if (batched.is_BatchError()) { + throw new IOException(batched.dtor_error()); + } else { + throw new RuntimeException("Unexpected batched value: " + batched); + } + } else { + return -1; + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + // TODO: Could optimize to use b directly, + // but that starts to introduce risk. + Array array = Array.newArray(TypeDescriptor.BYTE, len); + BatchArrayWriter consumer = new BatchArrayWriter<>( + TypeDescriptor.BYTE, TypeDescriptor.reference(Exception.class) + ); + consumer.__ctor(array); + producer.Fill(consumer); + if (consumer.state.is_Failure()) { + throw new IOException(consumer.state.dtor_error()); + } + byte[] results = (byte[])array.unwrap(); + int count = consumer.size.intValueExact(); + if (count == 0) { + return -1; + } else { + System.arraycopy(results, 0, b, off, count); + return count; + } + } + + @Override + public long skip(long n) throws IOException { + // TODO: Still need to check for errors and handle EOI + IgnoreNConsumer> consumer = new IgnoreNConsumer<>(null); + consumer.__ctor(BigInteger.valueOf(n)); + producer.Fill(consumer); + return consumer.consumedCount.longValueExact(); + } +} diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 0e73df3cb..371c37c4b 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 0e73df3cbdcf3e172856658f67c04b4f739871f2 +Subproject commit 371c37c4bc87a3fc363c3739e291bae506515ded From 6fde873c45eb4a1a0fd13434d3ed8fd69844a52c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 26 Apr 2025 11:44:52 -0700 Subject: [PATCH 25/72] InputStream adaptors --- .../java/Streams/InputStreamAsDataStream.java | 39 ++++++++ .../java/Streams/InputStreamAsProducer.java | 97 +++++++++++++++++++ .../internaldafny/extern/streams.py | 3 + .../StandardLibrary/src/Streams.dfy | 6 ++ 4 files changed, 145 insertions(+) create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java new file mode 100644 index 000000000..8b09a99f3 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java @@ -0,0 +1,39 @@ +package Streams; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Std_Compile.BulkActions_Compile.Batched; +import Std_Compile.Producers_Compile.Producer; +import Std_Compile.Wrappers_Compile.Option; + +import java.io.InputStream; +import java.math.BigInteger; + +public class InputStreamAsDataStream implements DataStream { + + private final InputStream inputStream; + private boolean read = false; + + public InputStreamAsDataStream(InputStream inputStream) { + this.inputStream = inputStream; + } + + @Override + public Option ContentLength() { + return Option.create_None(null); + } + + @Override + public boolean Replayable() { + return false; + } + + @Override + public Producer> Reader() { + if (read) { + throw new IllegalStateException("Already read"); + } + read = true; + + return new InputStreamAsProducer(inputStream); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java new file mode 100644 index 000000000..8e3041b97 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java @@ -0,0 +1,97 @@ +package Streams; + +import Std_Compile.BulkActions_Compile.BatchArrayWriter; +import Std_Compile.BulkActions_Compile.BatchReader; +import Std_Compile.BulkActions_Compile.Batched; +import Std_Compile.Consumers_Compile.Consumer; +import Std_Compile.Consumers_Compile.IConsumer; +import Std_Compile.Producers_Compile.Producer; +import Std_Compile.Producers_Compile._Companion_Producer; +import Std_Compile.Producers_Compile.__default; +import Std_Compile.Wrappers_Compile.Option; +import dafny.Array; +import dafny.DafnySequence; +import dafny.Tuple0; +import dafny.TypeDescriptor; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; + +public class InputStreamAsProducer implements Producer> { + + private final InputStream inputStream; + private long totalRead; + + private static final TypeDescriptor T_TD = TypeDescriptor.BYTE; + private static final TypeDescriptor E_TD = TypeDescriptor.reference(Exception.class); + private static final TypeDescriptor> BATCHED_TD = + Batched._typeDescriptor(T_TD, E_TD); + + public InputStreamAsProducer(InputStream inputStream) { + this.inputStream = inputStream; + this.totalRead = 0; + } + + @Override + public BigInteger ProducedCount() { + return BigInteger.valueOf(totalRead); + } + + @Override + public Option Remaining() { + return Option.create_None(TypeDescriptor.BIG_INTEGER); + } + + @Override + public Option> Next() { + try { + int value = inputStream.read(); + if (value == -1) { + // TODO: EOI + return Option.create_None(BATCHED_TD); + } else { + Batched batched = Batched.create_BatchValue(T_TD, E_TD, (byte)value); + return Option.create_Some(BATCHED_TD, batched); + } + } catch (IOException e) { + return Option.create_Some(null, Batched.create_BatchError(T_TD, E_TD, e)); + } + } + + @Override + public void ForEach(IConsumer> consumer) { + __default.DefaultForEach(BATCHED_TD, this, consumer); + } + + @Override + public Option> Fill(Consumer> consumer) { + if (consumer instanceof BatchArrayWriter) { + BatchArrayWriter writer = (BatchArrayWriter) consumer; + int n = writer.Capacity().dtor_value().intValueExact(); + byte[] buffer = new byte[n]; + try { + int count = inputStream.read(buffer, 0, n); + if (count == -1) { + consumer.Accept(Batched.create_EndOfInput(T_TD, E_TD)); + } else { + totalRead += count; + Array dafnyArray = Array.wrap(buffer); + BatchReader reader = new BatchReader(T_TD, E_TD); + reader.__ctor(DafnySequence.fromArrayRange(T_TD, dafnyArray, 0, count)); + reader.Fill(consumer); + } + } catch (IOException e) { + writer.Accept(Batched.create_BatchError(T_TD, E_TD, e)); + } + return Option.create_None(BATCHED_TD); + } + + return __default.DefaultFill(BATCHED_TD, this, consumer); + } + + @Override + public Option> Invoke(Tuple0 tuple0) { + return _Companion_Producer.Next(null, this); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index 858a6196c..2fa3ebc19 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -62,6 +62,9 @@ def __init__(self, streaming_blob): self.streaming_blob = streaming_blob self.read = False + def Replayable(self): + False + def Reader(self): if self.read: raise Exception("StreamingBlobAsDafnyDataStream.Reader() called twice") diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index 88029efd4..b19539146 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -20,6 +20,8 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { function ContentLength(): Option + predicate Replayable() + method Reader() returns (p: Producer>) ensures && p.Valid() @@ -42,6 +44,10 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { Some(|s|) } + predicate Replayable() { + true + } + method Reader() returns (p: Producer>) ensures && p.Valid() From e746d2a41ed10dcc4ca8cbd6f9209c10f0f70631 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 26 Apr 2025 13:04:33 -0700 Subject: [PATCH 26/72] Add Java files --- .../s3/runtimes/java/build.gradle.kts | 77 +++++++++++++++++++ .../services/s3/internaldafny/__default.java | 37 +++++++++ .../s3/internaldafny/types/__default.java | 5 ++ 3 files changed, 119 insertions(+) create mode 100644 TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts create mode 100644 TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/__default.java create mode 100644 TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/types/__default.java diff --git a/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts new file mode 100644 index 000000000..9cfd9f31f --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts @@ -0,0 +1,77 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import java.io.File +import java.io.FileInputStream +import java.util.Properties +import java.net.URI +import javax.annotation.Nullable + +tasks.wrapper { + gradleVersion = "7.6" +} + +plugins { + `java-library` + `maven-publish` +} + +var props = Properties().apply { + load(FileInputStream(File(rootProject.rootDir, "../../project.properties"))) +} +var dafnyVersion = props.getProperty("dafnyVersion") + +group = "com.amazonaws.s3" +version = "1.0-SNAPSHOT" +description = "AmazonS3" + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(8)) + sourceSets["main"].java { + srcDir("src/main/java") + srcDir("src/main/dafny-generated") + srcDir("src/main/smithy-generated") + } + sourceSets["test"].java { + srcDir("src/test/java") + srcDir("src/test/dafny-generated") + srcDir("src/test/smithy-generated") + } +} + +repositories { + mavenCentral() + mavenLocal() +} + +dependencies { + implementation("org.dafny:DafnyRuntime:${dafnyVersion}") + implementation("software.amazon.smithy.dafny:conversion:0.1.1") + implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") + implementation(platform("software.amazon.awssdk:bom:2.19.1")) + implementation("software.amazon.awssdk:s3") +} + +publishing { + publications.create("mavenLocal") { + groupId = group as String? + artifactId = description + from(components["java"]) + } + publications.create("maven") { + groupId = group as String? + artifactId = description + from(components["java"]) + } + repositories { mavenLocal() } +} + +tasks.withType() { + options.encoding = "UTF-8" +} + +tasks { + register("runTests", JavaExec::class.java) { + mainClass.set("TestsFromDafny") + classpath = sourceSets["test"].runtimeClasspath + } +} diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/__default.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/__default.java new file mode 100644 index 000000000..3db9f3242 --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/__default.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.cryptography.services.s3.internaldafny; + +import static software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence; +import static software.amazon.smithy.dafny.conversion.ToNative.Simple.String; + +import StandardLibraryInterop_Compile.WrappersInterop; +import Wrappers_Compile.Option; +import Wrappers_Compile.Result; +import dafny.DafnySequence; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.cryptography.services.s3.internaldafny.types.Error; +import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; + +public class __default + extends software.amazon.cryptography.services.s3.internaldafny._ExternBase___default { + + public static Result S3Client() { + try { + Region region = new DefaultAwsRegionProviderChain().getRegion(); + final S3Client nativeClient = S3Client + .builder() + .region(region) + .build(); + + IS3Client shim = new Shim(nativeClient, region.toString()); + return CreateSuccessOfClient(shim); + } catch (Exception e) { + Error dafny_error = Error.create_Opaque(e); + return CreateFailureOfError(dafny_error); + } + } +} diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/types/__default.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/types/__default.java new file mode 100644 index 000000000..0ce0f3e72 --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/java/software/amazon/cryptography/services/s3/internaldafny/types/__default.java @@ -0,0 +1,5 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.cryptography.services.s3.internaldafny.types; + +public class __default extends software.amazon.cryptography.services.s3.internaldafny.types._ExternBase___default {} From 7a08b817cceeeea456f01e6c843940aa09fb8a66 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 26 Apr 2025 15:13:16 -0700 Subject: [PATCH 27/72] Starting on Java async --- .../java/Streams/ConsumerAsSubscriber.java | 71 +++++++++++++++++++ .../Streams/DataStreamAsAsyncRequestBody.java | 34 +++++++++ 2 files changed, 105 insertions(+) create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java new file mode 100644 index 000000000..1aeec3c05 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java @@ -0,0 +1,71 @@ +package Streams; + +import Std_Compile.BulkActions_Compile.BatchReader; +import Std_Compile.BulkActions_Compile.Batched; +import Std_Compile.Consumers_Compile.IConsumer; +import dafny.Array; +import dafny.DafnySequence; +import dafny.TypeDescriptor; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import java.nio.ByteBuffer; +import java.util.concurrent.CompletableFuture; + +public class ConsumerAsSubscriber implements Subscriber { + + private final IConsumer> consumer + private final CompletableFuture future; + private Subscription subscription; + + private static final TypeDescriptor T_TD = TypeDescriptor.BYTE; + private static final TypeDescriptor E_TD = TypeDescriptor.reference(Throwable.class); + private static final TypeDescriptor> BATCHED_TD = + Batched._typeDescriptor(T_TD, E_TD); + + + public ConsumerAsSubscriber(IConsumer> consumer, CompletableFuture future) { + this.consumer = consumer; + this.future = future; + } + + @Override + public void onSubscribe(Subscription s) { + this.subscription = subscription; + subscription.request(1L); + } + + @Override + public void onNext(ByteBuffer o) { + try { + byte[] a = new byte[o.remaining()]; + o.get(a); + Array dafnyArray = Array.wrap(a); + DafnySequence dafnySeq = DafnySequence.unsafeWrapArray(dafnyArray); + BatchReader reader = new BatchReader(T_TD, E_TD); + reader.__ctor(dafnySeq); + reader.ForEach(consumer); + + this.subscription.request(1L); + } catch (RuntimeException e) { + this.subscription.cancel(); + this.future.completeExceptionally(e); + } + } + + @Override + public void onError(Throwable t) { + Batched event = Batched.create_BatchError(T_TD, E_TD, t); + consumer.Accept(event); + + this.future.completeExceptionally(t); + } + + @Override + public void onComplete() { + Batched event = Batched.create_EndOfInput(T_TD, E_TD); + consumer.Accept(event); + + this.future.complete(null); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java new file mode 100644 index 000000000..933c329a7 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java @@ -0,0 +1,34 @@ +package Streams; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Std_Compile.Wrappers_Compile.Option; +import org.reactivestreams.Subscriber; +import software.amazon.awssdk.core.async.AsyncRequestBody; + +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.util.Optional; + +public class DataStreamAsAsyncRequestBody implements AsyncRequestBody { + + private final DataStream dataStream; + + public DataStreamAsAsyncRequestBody(DataStream dataStream) { + this.dataStream = dataStream; + } + + @Override + public Optional contentLength() { + Option cl = dataStream.ContentLength(); + if (cl.is_Some()) { + return Optional.of(cl.dtor_value().longValueExact()); + } else { + return Optional.empty(); + } + } + + @Override + public void subscribe(Subscriber s) { + s.onSubscribe(null); + } +} From 6e28c3089ce7a1ba322ee406a1c4ce8c3c5c60fd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 27 Apr 2025 11:50:47 -0700 Subject: [PATCH 28/72] Fix other examples --- .../Streaming/Model/SimpleStreamingTypes.dfy | 12 +- TestModels/Streaming/src/Chunker.dfy | 266 +++++++++++++----- .../Streaming/src/SimpleStreamingImpl.dfy | 22 +- .../StandardLibrary/src/Streams.dfy | 16 +- .../smithydafny/DafnyApiCodegen.java | 2 +- .../smithydafny/DafnyNameResolver.java | 2 +- 6 files changed, 222 insertions(+), 98 deletions(-) diff --git a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy index dfaf96c7f..c44e78368 100644 --- a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy +++ b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // Do not modify this file. This file is machine generated, and any changes to it will be overwritten. include "../../dafny-dependencies/StandardLibrary/src/Index.dfy" -module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes +module SimpleStreamingTypes { import opened Wrappers import opened StandardLibrary.UInt @@ -17,17 +17,17 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes nameonly number: int32 ) datatype BinaryOfOutput = | BinaryOfOutput ( - nameonly binary: DataStream + nameonly binary: DataStream ) datatype ChunksInput = | ChunksInput ( - nameonly bytesIn: DataStream , + nameonly bytesIn: DataStream , nameonly chunkSize: CountingInteger ) datatype ChunksOutput = | ChunksOutput ( - nameonly bytesOut: DataStream + nameonly bytesOut: DataStream ) datatype CountBitsInput = | CountBitsInput ( - nameonly bits: DataStream + nameonly bits: DataStream ) datatype CountBitsOutput = | CountBitsOutput ( nameonly sum: int32 @@ -122,7 +122,7 @@ module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes datatype SimpleStreamingConfig = | SimpleStreamingConfig ( ) - type StreamingBlob = DataStream + type StreamingBlob = DataStream datatype Error = // Local Error structures are listed here | OverflowError ( diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 73483dbec..93a1233d0 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -2,8 +2,6 @@ include "../Model/SimpleStreamingTypes.dfy" module {:options "--function-syntax:4"} Chunker { - import Std.BoundedInts - import opened Std.Wrappers import opened Types = SimpleStreamingTypes import opened StandardLibrary.UInt @@ -13,13 +11,16 @@ module {:options "--function-syntax:4"} Chunker { import opened Std.Consumers import opened StandardLibrary.Streams + type BB = Batched + @AssumeCrossModuleTermination - class Chunker extends BulkAction>, Option>>> { + class Chunker extends BulkAction> { const chunkSize: CountingInteger - var chunkBuffer: BoundedInts.bytes + var chunkBuffer: seq constructor(chunkSize: CountingInteger) + requires 0 < chunkSize ensures Valid() ensures fresh(Repr) ensures history == [] @@ -36,84 +37,169 @@ module {:options "--function-syntax:4"} Chunker { ensures Valid() ==> ValidHistory(history) decreases Repr, 0 { - this in Repr + && this in Repr + && 0 < chunkSize + } + + twostate predicate ValidChange() + reads this, Repr + ensures ValidChange() ==> old(Valid()) && Valid() + ensures ValidChange() ==> fresh(Repr - old(Repr)) + ensures ValidChange() ==> old(history) <= history + { + && fresh(Repr - old(Repr)) + && old(Valid()) + && Valid() + && old(history) <= history } - ghost predicate ValidHistory(history: seq<(Option>, Option>>)>) + twostate lemma ValidImpliesValidChange() + requires old(Valid()) + requires unchanged(old(Repr)) + ensures ValidChange() + {} + + ghost predicate ValidHistory(history: seq<(BB, seq)>) decreases Repr { true } - ghost predicate ValidInput(history: seq<(Option>, Option>>)>, next: Option>) + ghost predicate ValidInput(history: seq<(BB, seq)>, next: BB) requires ValidHistory(history) decreases Repr { true } - ghost function Decreases(i: Option>): ORDINAL + ghost function Decreases(i: BB): ORDINAL requires Requires(i) reads Reads(i) { 0 } - // Ideally could use this in a MappedConsumer as well + @IsolateAssertions + method Invoke(i: BB) returns (o: seq) + requires Requires(i) + modifies Modifies(i) + decreases Decreases(i), 0 + ensures Ensures(i, o) + { + assert Valid(); + var input := new SeqReader([i]); + var output := new SeqWriter(); + var outputTotalProof := new SeqWriterTotalActionProof(output); + label before: + BulkInvoke(input, output, outputTotalProof); + assert |output.values| == 1; + o := output.values[0]; + assert Seq.Last(output.Inputs()) == o; + assert Seq.Last(Inputs()) == i; + } - method Single(input: Option>) returns (r: Option>>) + @ResourceLimit("0") + @IsolateAssertions + method BulkInvoke(input: Producer, + output: IConsumer>, + outputTotalProof: TotalActionProof, ()>) + requires Valid() + requires input.Valid() + requires output.Valid() + requires outputTotalProof.Valid() + requires outputTotalProof.Action() == output + requires Repr !! input.Repr !! output.Repr !! outputTotalProof.Repr + modifies Repr, input.Repr, output.Repr, outputTotalProof.Repr + ensures ValidChange() + ensures input.ValidChange() + ensures output.ValidChange() + ensures input.Done() + ensures input.NewProduced() == NewInputs() + ensures |input.NewProduced()| == |output.NewInputs()| + ensures output.NewInputs() == NewOutputs() { + assert Valid(); - } + var oldProducedCount := input.ProducedCount(); + var batchWriter := new BatchSeqWriter(); + var batchWriterTotalProof := new BatchSeqWriterTotalProof(batchWriter); + label before: + input.ForEach(batchWriter, batchWriterTotalProof); + label after: + assert input.ValidChange@before(); + assert input.ValidChange(); + input.ProducedAndNewProduced@before(); + + var newProducedCount := input.ProducedCount() - oldProducedCount; + assert newProducedCount == input.NewProducedCount(); + if newProducedCount == 0 { + // No-op + assert input.ValidChange(); + assert |batchWriter.Inputs()| == 0; + assert input.NewProduced() == batchWriter.Inputs(); + assert |input.NewProduced()| == 0; + output.ValidImpliesValidChange(); + return; + } + + chunkBuffer := chunkBuffer + batchWriter.elements; - method Bulk(input: Producer>, output: Consumer>) { - + var chunks, leftover := Chunkify(chunkBuffer); + var chunkBuffer := leftover; + + var outputProducer: Producer; + match batchWriter.state { + case Failure(error) => + outputProducer := new SeqReader([BatchError(error)]); + case Success(more) => + if !more && 0 < |chunkBuffer| { + // To make it more interesting, produce an error if outputChunks is non empty? + chunks := chunks + Seq.Reverse(chunkBuffer); + } + outputProducer := new BatchReader(chunks); + } + + // TODO: Find the right way to keep this as a batch, + // this is just to get it resolving again + var data := CollectToSeq(outputProducer); + var dataReader := new SeqReader([data]); + var padding := new RepeatProducer(newProducedCount - 1, []); + var concatenated: Producer> := new ConcatenatedProducer(padding, dataReader); + assert dataReader.Remaining() == Some(1); + assert padding.Remaining() == Some(newProducedCount - 1); + assert concatenated.Remaining() == Some(newProducedCount); + label beforeOutput: + concatenated.ForEach(output, outputTotalProof); + assert concatenated.ValidChange@beforeOutput(); + concatenated.ProducedAndNewProduced@beforeOutput(); + + assert |input.NewProduced()| == newProducedCount; + assert |concatenated.NewProduced@beforeOutput()| == newProducedCount; + assert |input.NewProduced()| == |output.NewInputs()|; + history := history + Seq.Zip(input.NewProduced(), output.NewInputs()); + assert input.NewProduced() == NewInputs(); } - method BulkInvoke(input: Producer>, output: IConsumer>) - requires Requires(input) - modifies Modifies(input) - decreases Decreases(input), 0 - ensures Ensures(input, r) + method Chunkify(data: seq) returns (chunks: seq, leftover: seq) + requires Valid() { - assert Valid(); - var outputChunks := []; - if input.Some? { - if input.value.Failure? { - outputChunks := outputChunks + [input.value]; - } else { - - chunkBuffer := chunkBuffer + input.value.value; - - while chunkSize as int <= |chunkBuffer| - invariant ValidAndDisjoint() - invariant history == old(history) - { - outputChunks := outputChunks + [Success(chunkBuffer[..chunkSize])]; - chunkBuffer := chunkBuffer[chunkSize..]; - } - } - } else { - if 0 < |chunkBuffer| { - outputChunks := outputChunks + [Success(chunkBuffer)]; - } else { - r := None; - UpdateHistory(input, r); - return; - } + leftover := data; + chunks := []; + while chunkSize as int <= |leftover| + decreases |leftover| + { + chunks := chunks + Seq.Reverse(leftover[..chunkSize]); + leftover := leftover[chunkSize..]; } - var output := new SeqReader(outputChunks); - r := Some(output); - UpdateHistory(input, r); } } @AssumeCrossModuleTermination - class ChunkerTotalProof extends TotalActionProof>, Option>>> { + class ChunkerTotalProof extends TotalActionProof> { - ghost const chunker: Chunker + ghost const chunker: Chunker - ghost constructor(chunker: Chunker) + ghost constructor(chunker: Chunker) requires chunker.Valid() ensures this.chunker == chunker ensures Valid() @@ -123,7 +209,7 @@ module {:options "--function-syntax:4"} Chunker { Repr := {this}; } - ghost function Action(): Action>, Option>>> { + ghost function Action(): Action> { chunker } @@ -135,34 +221,80 @@ module {:options "--function-syntax:4"} Chunker { this in Repr } - lemma AnyInputIsValid(history: seq<(Option>, Option>>)>, next: Option>) + twostate predicate ValidChange() + reads this, Repr + ensures ValidChange() ==> + old(Valid()) && Valid() && fresh(Repr - old(Repr)) + { + old(Valid()) && Valid() && fresh(Repr - old(Repr)) + } + + twostate lemma ValidImpliesValidChange() + requires old(Valid()) + requires unchanged(old(Repr)) + ensures ValidChange() + {} + + lemma AnyInputIsValid(history: seq<(BB, seq)>, next: BB) requires Valid() requires Action().ValidHistory(history) ensures Action().ValidInput(history, next) {} - } - method ChunkingStream(chunkSize: CountingInteger, s: DataStream) - requires s.Valid() - requires s.history == [] - { - var chunker := new Chunker(chunkSize); - ghost var chunkerTotalProof := new ChunkerTotalProof(chunker); - var chunkerStream := new OptionMappedProducer(s, chunker, chunkerTotalProof); - } - function SumBits(sum: int, maybeChunk: Result, Error>): int { - match maybeChunk - case Success(chunk) => sum + BytesBitCount(chunk) - case Failure(_) => sum + @AssumeCrossModuleTermination + class ChunkingStream extends DataStream { + + const chunkSize: CountingInteger + const original: DataStream + + constructor (original: DataStream, chunkSize: CountingInteger) + { + this.original := original; + this.chunkSize := chunkSize; + } + + function ContentLength(): Option { + original.ContentLength() + } + + predicate Replayable() { + original.Replayable() + } + + method Reader() returns (p: Producer) + ensures + && p.Valid() + && fresh(p.Repr) + && p.history == [] + && (ContentLength().Some? ==> p.Remaining() == Some(ContentLength().value as int + 1)) + { + var chunker := new Chunker(chunkSize); + var chunkerTotalProof := new ChunkerTotalProof(chunker); + var originalProducer := original.Reader(); + var chunkerStream := new MappedProducer(originalProducer, chunker, chunkerTotalProof); + } } - function BytesBitCount(b: seq): int { - Seq.FoldLeft((sum, byte) => sum + BitCount(byte), 0 as int, b) + function SumBits(sum: Result, batched: Batched): Result { + match batched + case BatchValue(b) => + if sum.Success? then + var next := BitCount(b); + if INT32_MAX_LIMIT < sum.value as int + next as int then + Failure(OverflowError(message := "Ah crap")) + else + Success(sum.value + next) + else + sum + case BatchError(error) => + // This could also ensure the first error is kept instead + Failure(error) + case EndOfInput => sum } - function BitCount(x: uint8): int { + function BitCount(x: uint8): int32 { if x == 0 then 0 else if x % 2 == 1 then diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index 6d4ec6287..dc1810314 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -23,17 +23,17 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl method CountBits ( config: InternalConfig , input: CountBitsInput ) returns (output: Result) { - var counter := new Consumers.FoldingConsumer(0, SumBits); + var counter := new Consumers.FoldingConsumer(Success(0 as int32), SumBits); var counterTotalProof := new Consumers.FoldingConsumerTotalActionProof(counter); - input.bits.ForEachRemaining(counter, counterTotalProof); + var inputReader := input.bits.Reader(); + inputReader.ForEach(counter, counterTotalProof); + var result := counter.value; - // Should really have the FoldingConsumer fail instead, - // but this is a simpler correct approach. - if 0 <= counter.value < INT32_MAX_LIMIT { - return Success(CountBitsOutput(sum := counter.value as int32)); + if result.Success? { + return Success(CountBitsOutput(sum := result.value)); } else { - return Failure(OverflowError(message := "Ah crap")); + return Failure(result.error); } } @@ -47,7 +47,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl { var binary := BinaryOfNumber(input.number); - var binaryStream := new SeqDataStream(binary, 3 as BoundedInts.uint64); + var binaryStream := new SeqDataStream(binary); return Success(BinaryOfOutput(binary := binaryStream)); } @@ -59,11 +59,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl method Chunks ( config: InternalConfig , input: ChunksInput ) returns (output: Result) { - // TODO: for now - assume {:axiom} input.bytesIn.history == []; - var chunker := new Chunker(input.chunkSize); - ghost var chunkerTotalProof := new ChunkerTotalProof(chunker); - var chunkerStream := new MappedDataStream(input.bytesIn, chunker, chunkerTotalProof); + var chunkerStream := new ChunkingStream(input.bytesIn, input.chunkSize); return Success(ChunksOutput(bytesOut := chunkerStream)); } diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy index b19539146..760f077f0 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy @@ -9,20 +9,16 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { import opened Std.Actions import opened Std.BulkActions import opened Std.Producers - import opened Std.BoundedInts import opened Std.Collections.Seq import opened Std.Termination - // - // A data stream, i.e. a fallable producer of batches of values. - // - trait DataStream { + trait DataStream { function ContentLength(): Option predicate Replayable() - method Reader() returns (p: Producer>) + method Reader() returns (p: Producer>) ensures && p.Valid() && fresh(p.Repr) @@ -30,11 +26,11 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { && (ContentLength().Some? ==> p.Remaining() == Some(ContentLength().value as int + 1)) } - class SeqDataStream extends DataStream { + class SeqDataStream extends DataStream { - const s: seq + const s: seq - constructor (s: seq) + constructor (s: seq) ensures this.s == s { this.s := s; @@ -48,7 +44,7 @@ module {:options "--function-syntax:4"} StandardLibrary.Streams { true } - method Reader() returns (p: Producer>) + method Reader() returns (p: Producer>) ensures && p.Valid() && fresh(p.Repr) diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index 8dcdcc065..fda0789a4 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -322,7 +322,7 @@ public TokenTree generateBlobTypeDefinition(final ShapeId blobShapeId) { if (blobShape.hasTrait(StreamingTrait.class)) { // TODO: need to handle @length too, // something like `a.ContentLength().Some? && min <= a.ContentLength().value <= max - return generateTypeSynonym(blobShapeId, "DataStream"); + return generateTypeSynonym(blobShapeId, "DataStream"); } else { return generateSubsetType(blobShapeId, "seq", lengthConstraint); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index c03388517..82c8b3d8c 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -90,7 +90,7 @@ public String baseTypeForShape(final ShapeId shapeId) { MAP -> dafnyModulePrefixForShape(shape) + shapeName; case BLOB -> { if (shape.hasTrait(StreamingTrait.class)) { - yield "DataStream"; + yield "DataStream"; } else { yield dafnyModulePrefixForShape(shape) + shapeName; } From 171230c7f37c8c637e36649597690cbc44a6dc7d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 27 Apr 2025 12:02:28 -0700 Subject: [PATCH 29/72] Comment --- TestModels/Streaming/src/Chunker.dfy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 93a1233d0..15af925c1 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -1,5 +1,10 @@ include "../Model/SimpleStreamingTypes.dfy" +// Separate module not just for better code organization, +// but also to work around the conflict between the Dafny standard library +// Std.Wrappers module and the smithy-dafny specific Wrappers module: +// it's not currently possible to use both Result types in the same module. + module {:options "--function-syntax:4"} Chunker { import opened Std.Wrappers From 04151856da8e4bf31255fd2d94ba767793efa0ea Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 09:44:58 -0700 Subject: [PATCH 30/72] Progress --- TestModels/Streaming/src/Chunker.dfy | 55 +++++++------------ .../Streaming/src/SimpleStreamingImpl.dfy | 18 ++++++ TestModels/dafny-dependencies/dafny | 2 +- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 15af925c1..b9c8b350b 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -16,10 +16,11 @@ module {:options "--function-syntax:4"} Chunker { import opened Std.Consumers import opened StandardLibrary.Streams + // "Batched Byte" type BB = Batched @AssumeCrossModuleTermination - class Chunker extends BulkAction> { + class Chunker extends BulkAction> { const chunkSize: CountingInteger var chunkBuffer: seq @@ -64,13 +65,13 @@ module {:options "--function-syntax:4"} Chunker { ensures ValidChange() {} - ghost predicate ValidHistory(history: seq<(BB, seq)>) + ghost predicate ValidHistory(history: seq<(BB, Producer)>) decreases Repr { true } - ghost predicate ValidInput(history: seq<(BB, seq)>, next: BB) + ghost predicate ValidInput(history: seq<(BB, Producer)>, next: BB) requires ValidHistory(history) decreases Repr { @@ -85,7 +86,7 @@ module {:options "--function-syntax:4"} Chunker { } @IsolateAssertions - method Invoke(i: BB) returns (o: seq) + method Invoke(i: BB) returns (o: Producer) requires Requires(i) modifies Modifies(i) decreases Decreases(i), 0 @@ -103,11 +104,11 @@ module {:options "--function-syntax:4"} Chunker { assert Seq.Last(Inputs()) == i; } - @ResourceLimit("0") + @ResourceLimit("1e9") @IsolateAssertions method BulkInvoke(input: Producer, - output: IConsumer>, - outputTotalProof: TotalActionProof, ()>) + output: IConsumer>, + outputTotalProof: TotalActionProof, ()>) requires Valid() requires input.Valid() requires output.Valid() @@ -164,13 +165,11 @@ module {:options "--function-syntax:4"} Chunker { outputProducer := new BatchReader(chunks); } - // TODO: Find the right way to keep this as a batch, - // this is just to get it resolving again - var data := CollectToSeq(outputProducer); - var dataReader := new SeqReader([data]); - var padding := new RepeatProducer(newProducedCount - 1, []); - var concatenated: Producer> := new ConcatenatedProducer(padding, dataReader); - assert dataReader.Remaining() == Some(1); + var empty := new EmptyProducer(); + var padding: Producer> := new RepeatProducer(newProducedCount - 1, empty); + var producerProducer := new SeqReader([outputProducer]); + var concatenated: Producer> := new ConcatenatedProducer(padding, producerProducer); + assert producerProducer.Remaining() == Some(1); assert padding.Remaining() == Some(newProducedCount - 1); assert concatenated.Remaining() == Some(newProducedCount); label beforeOutput: @@ -200,7 +199,7 @@ module {:options "--function-syntax:4"} Chunker { } @AssumeCrossModuleTermination - class ChunkerTotalProof extends TotalActionProof> { + class ChunkerTotalProof extends TotalActionProof> { ghost const chunker: Chunker @@ -214,7 +213,7 @@ module {:options "--function-syntax:4"} Chunker { Repr := {this}; } - ghost function Action(): Action> { + ghost function Action(): Action> { chunker } @@ -240,7 +239,7 @@ module {:options "--function-syntax:4"} Chunker { ensures ValidChange() {} - lemma AnyInputIsValid(history: seq<(BB, seq)>, next: BB) + lemma AnyInputIsValid(history: seq<(BB, Producer)>, next: BB) requires Valid() requires Action().ValidHistory(history) ensures Action().ValidInput(history, next) @@ -279,27 +278,11 @@ module {:options "--function-syntax:4"} Chunker { var chunkerTotalProof := new ChunkerTotalProof(chunker); var originalProducer := original.Reader(); var chunkerStream := new MappedProducer(originalProducer, chunker, chunkerTotalProof); + // TODO: Need FlattenedProducer, but can't implement ProducerOfNewProducers extrinsically } } - function SumBits(sum: Result, batched: Batched): Result { - match batched - case BatchValue(b) => - if sum.Success? then - var next := BitCount(b); - if INT32_MAX_LIMIT < sum.value as int + next as int then - Failure(OverflowError(message := "Ah crap")) - else - Success(sum.value + next) - else - sum - case BatchError(error) => - // This could also ensure the first error is kept instead - Failure(error) - case EndOfInput => sum - } - - function BitCount(x: uint8): int32 { + function BitCount(x: uint8): int { if x == 0 then 0 else if x % 2 == 1 then @@ -308,7 +291,7 @@ module {:options "--function-syntax:4"} Chunker { BitCount(x / 2) } - function BinaryOfNumber(x: int32): seq { + function BinaryOfNumber(x: int32): seq { // TODO: Actually compute the binary [12 as uint8, 34, 56] } diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index dc1810314..4f1a64131 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -6,6 +6,7 @@ include "Chunker.dfy" module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpleStreamingOperations { import Std.Actions + import Std.BulkActions import Std.Producers import Std.Consumers import Std.Collections.Seq @@ -37,6 +38,23 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl } } + function SumBits(sum: Result, batched: BulkActions.Batched): Result { + match batched + case BatchValue(b) => + if sum.Success? then + var next := BitCount(b); + if !(0 <= sum.value as int + next < INT32_MAX_LIMIT) then + Failure(OverflowError(message := "Ah crap")) + else + Success((sum.value as int + next) as int32) + else + sum + case BatchError(error) => + // This could also ensure the first error is kept instead + Failure(error) + case EndOfInput => sum + } + predicate BinaryOfEnsuresPublicly(input: BinaryOfInput , output: Result) {true} diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 371c37c4b..d40b30596 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 371c37c4bc87a3fc363c3739e291bae506515ded +Subproject commit d40b30596c21322290ac9adc260c8af5c27a0220 From 8e0d3c8ab91bc28a04f3dcc89077a9923ec50f12 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 11:36:37 -0700 Subject: [PATCH 31/72] Patching smithy-generated for S3 in Java --- TestModels/Streaming/src/Chunker.dfy | 2 +- .../s3/runtimes/java/build.gradle.kts | 2 +- .../services/s3/internaldafny/Shim.java | 224 ++ .../services/s3/internaldafny/ToDafny.java | 2883 +++++++++++++++++ .../services/s3/internaldafny/ToNative.java | 1725 ++++++++++ .../java/Streams/ConsumerAsSubscriber.java | 2 +- .../java/Streams/InputStreamAsDataStream.java | 16 +- .../java/Streams/InputStreamAsProducer.java | 31 +- 8 files changed, 4865 insertions(+), 20 deletions(-) create mode 100644 TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java create mode 100644 TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java create mode 100644 TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index b9c8b350b..0ae33ecf8 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -278,7 +278,7 @@ module {:options "--function-syntax:4"} Chunker { var chunkerTotalProof := new ChunkerTotalProof(chunker); var originalProducer := original.Reader(); var chunkerStream := new MappedProducer(originalProducer, chunker, chunkerTotalProof); - // TODO: Need FlattenedProducer, but can't implement ProducerOfNewProducers extrinsically + // p := new FlattenedProducer(chunkerStream); } } diff --git a/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts index 9cfd9f31f..5fe73ed04 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts +++ b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts @@ -47,7 +47,7 @@ dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") - implementation(platform("software.amazon.awssdk:bom:2.19.1")) + implementation(platform("software.amazon.awssdk:bom:2.31.30")) implementation("software.amazon.awssdk:s3") } diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java new file mode 100644 index 000000000..b7e9b4057 --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java @@ -0,0 +1,224 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Do not modify this file. This file is machine generated, and any changes to it will be overwritten. +package software.amazon.cryptography.services.s3.internaldafny; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Std_Compile.Producers_Compile.Producer; +import Streams.ProviderAsInputStream; +import Wrappers_Compile.Result; +import java.lang.Override; +import java.lang.String; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.Error; +import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; +import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; +import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request; +import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectRequest; + +public class Shim implements IS3Client { + + private final S3Client _impl; + + private final String region; + + public Shim(final S3Client impl, final String region) { + this._impl = impl; + this.region = region; + } + + public S3Client impl() { + return this._impl; + } + + public String region() { + return this.region; + } + + @Override + public Result DeleteObject( + DeleteObjectRequest input + ) { + software.amazon.awssdk.services.s3.model.DeleteObjectRequest converted = + ToNative.DeleteObjectRequest(input); + try { + DeleteObjectResponse result = _impl.deleteObject(converted); + DeleteObjectOutput dafnyResponse = ToDafny.DeleteObjectOutput(result); + return Result.create_Success( + DeleteObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + dafnyResponse + ); + } catch (S3Exception ex) { + return Result.create_Failure( + DeleteObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (Exception ex) { + return Result.create_Failure( + DeleteObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } + } + + @Override + public Result DeleteObjects( + DeleteObjectsRequest input + ) { + software.amazon.awssdk.services.s3.model.DeleteObjectsRequest converted = + ToNative.DeleteObjectsRequest(input); + try { + DeleteObjectsResponse result = _impl.deleteObjects(converted); + DeleteObjectsOutput dafnyResponse = ToDafny.DeleteObjectsOutput(result); + return Result.create_Success( + DeleteObjectsOutput._typeDescriptor(), + Error._typeDescriptor(), + dafnyResponse + ); + } catch (S3Exception ex) { + return Result.create_Failure( + DeleteObjectsOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (Exception ex) { + return Result.create_Failure( + DeleteObjectsOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } + } + + @Override + public Result GetObject(GetObjectRequest input) { + software.amazon.awssdk.services.s3.model.GetObjectRequest converted = + ToNative.GetObjectRequest(input); + try { + ResponseInputStream result = _impl.getObject(converted); + GetObjectOutput dafnyResponse = ToDafny.GetObjectOutput(result); + return Result.create_Success( + GetObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + dafnyResponse + ); + } catch (InvalidObjectStateException ex) { + return Result.create_Failure( + GetObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (NoSuchKeyException ex) { + return Result.create_Failure( + GetObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (S3Exception ex) { + return Result.create_Failure( + GetObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (Exception ex) { + return Result.create_Failure( + GetObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } + } + + @Override + public Result ListObjectsV2( + ListObjectsV2Request input + ) { + software.amazon.awssdk.services.s3.model.ListObjectsV2Request converted = + ToNative.ListObjectsV2Request(input); + try { + ListObjectsV2Response result = _impl.listObjectsV2(converted); + ListObjectsV2Output dafnyResponse = ToDafny.ListObjectsV2Output(result); + return Result.create_Success( + ListObjectsV2Output._typeDescriptor(), + Error._typeDescriptor(), + dafnyResponse + ); + } catch (NoSuchBucketException ex) { + return Result.create_Failure( + ListObjectsV2Output._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (S3Exception ex) { + return Result.create_Failure( + ListObjectsV2Output._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (Exception ex) { + return Result.create_Failure( + ListObjectsV2Output._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } + } + + @Override + public Result PutObject(PutObjectRequest input) { + software.amazon.awssdk.services.s3.model.PutObjectRequest converted = + ToNative.PutObjectRequest(input); + DataStream dataStream = input._Body.dtor_value(); + ContentStreamProvider provider = () -> { + Producer reader = dataStream.Reader(); + return new ProviderAsInputStream(reader); + }; + RequestBody body = dataStream.ContentLength().is_Some() + ? RequestBody.fromContentProvider(provider, dataStream.ContentLength().dtor_value().longValueExact(), "application/octet-stream") + : RequestBody.fromContentProvider(provider, "application/octet-stream"); + try { + PutObjectResponse result = _impl.putObject(converted, body); + PutObjectOutput dafnyResponse = ToDafny.PutObjectOutput(result); + return Result.create_Success( + PutObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + dafnyResponse + ); + } catch (S3Exception ex) { + return Result.create_Failure( + PutObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } catch (Exception ex) { + return Result.create_Failure( + PutObjectOutput._typeDescriptor(), + Error._typeDescriptor(), + ToDafny.Error(ex) + ); + } + } +} diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java new file mode 100644 index 000000000..6ff475c8f --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -0,0 +1,2883 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Do not modify this file. This file is machine generated, and any changes to it will be overwritten. +package software.amazon.cryptography.services.s3.internaldafny; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Streams.InputStreamAsDataStream; +import Wrappers_Compile.Option; +import dafny.DafnyMap; +import dafny.DafnySequence; +import dafny.TypeDescriptor; +import java.lang.Boolean; +import java.lang.Byte; +import java.lang.Character; +import java.lang.Exception; +import java.lang.Integer; +import java.lang.Long; +import java.lang.RuntimeException; +import java.lang.String; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm; +import software.amazon.cryptography.services.s3.internaldafny.types.ChecksumMode; +import software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix; +import software.amazon.cryptography.services.s3.internaldafny.types.Delete; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.DeletedObject; +import software.amazon.cryptography.services.s3.internaldafny.types.EncodingType; +import software.amazon.cryptography.services.s3.internaldafny.types.Error; +import software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_InvalidObjectState; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchBucket; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchKey; +import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; +import software.amazon.cryptography.services.s3.internaldafny.types.IntelligentTieringAccessTier; +import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; +import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request; +import software.amazon.cryptography.services.s3.internaldafny.types.S3Object; +import software.amazon.cryptography.services.s3.internaldafny.types.ObjectCannedACL; +import software.amazon.cryptography.services.s3.internaldafny.types.ObjectIdentifier; +import software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus; +import software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockMode; +import software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass; +import software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes; +import software.amazon.cryptography.services.s3.internaldafny.types.Owner; +import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectRequest; +import software.amazon.cryptography.services.s3.internaldafny.types.ReplicationStatus; +import software.amazon.cryptography.services.s3.internaldafny.types.RequestCharged; +import software.amazon.cryptography.services.s3.internaldafny.types.RequestPayer; +import software.amazon.cryptography.services.s3.internaldafny.types.RestoreStatus; +import software.amazon.cryptography.services.s3.internaldafny.types.S3Object; +import software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption; +import software.amazon.cryptography.services.s3.internaldafny.types.StorageClass; + +public class ToDafny { + + public static DafnySequence< + ? extends ChecksumAlgorithm + > ChecksumAlgorithmList( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::ChecksumAlgorithm, + ChecksumAlgorithm._typeDescriptor() + ); + } + + public static CommonPrefix CommonPrefix( + software.amazon.awssdk.services.s3.model.CommonPrefix nativeValue + ) { + Option> prefix; + prefix = + Objects.nonNull(nativeValue.prefix()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.prefix() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new CommonPrefix(prefix); + } + + public static DafnySequence CommonPrefixList( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::CommonPrefix, + CommonPrefix._typeDescriptor() + ); + } + + public static Delete Delete( + software.amazon.awssdk.services.s3.model.Delete nativeValue + ) { + DafnySequence objects; + objects = ToDafny.ObjectIdentifierList(nativeValue.objects()); + Option quiet; + quiet = + Objects.nonNull(nativeValue.quiet()) + ? Option.create_Some(TypeDescriptor.BOOLEAN, (nativeValue.quiet())) + : Option.create_None(TypeDescriptor.BOOLEAN); + return new Delete(objects, quiet); + } + + public static DeletedObject DeletedObject( + software.amazon.awssdk.services.s3.model.DeletedObject nativeValue + ) { + Option> key; + key = + Objects.nonNull(nativeValue.key()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option deleteMarker; + deleteMarker = + Objects.nonNull(nativeValue.deleteMarker()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.deleteMarker()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> deleteMarkerVersionId; + deleteMarkerVersionId = + Objects.nonNull(nativeValue.deleteMarkerVersionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.deleteMarkerVersionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new DeletedObject( + key, + versionId, + deleteMarker, + deleteMarkerVersionId + ); + } + + public static DafnySequence DeletedObjects( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::DeletedObject, + DeletedObject._typeDescriptor() + ); + } + + public static DeleteObjectOutput DeleteObjectOutput( + DeleteObjectResponse nativeValue + ) { + Option deleteMarker; + deleteMarker = + Objects.nonNull(nativeValue.deleteMarker()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.deleteMarker()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestCharged; + requestCharged = + Objects.nonNull(nativeValue.requestCharged()) + ? Option.create_Some( + RequestCharged._typeDescriptor(), + ToDafny.RequestCharged(nativeValue.requestCharged()) + ) + : Option.create_None(RequestCharged._typeDescriptor()); + return new DeleteObjectOutput(deleteMarker, versionId, requestCharged); + } + + public static DeleteObjectRequest DeleteObjectRequest( + software.amazon.awssdk.services.s3.model.DeleteObjectRequest nativeValue + ) { + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.bucket() + ); + DafnySequence key; + key = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ); + Option> mFA; + mFA = + Objects.nonNull(nativeValue.mfa()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.mfa() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestPayer; + requestPayer = + Objects.nonNull(nativeValue.requestPayer()) + ? Option.create_Some( + RequestPayer._typeDescriptor(), + ToDafny.RequestPayer(nativeValue.requestPayer()) + ) + : Option.create_None(RequestPayer._typeDescriptor()); + Option bypassGovernanceRetention; + bypassGovernanceRetention = + Objects.nonNull(nativeValue.bypassGovernanceRetention()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.bypassGovernanceRetention()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> expectedBucketOwner; + expectedBucketOwner = + Objects.nonNull(nativeValue.expectedBucketOwner()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expectedBucketOwner() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new DeleteObjectRequest( + bucket, + key, + mFA, + versionId, + requestPayer, + bypassGovernanceRetention, + expectedBucketOwner + ); + } + + public static DeleteObjectsOutput DeleteObjectsOutput( + DeleteObjectsResponse nativeValue + ) { + Option> deleted; + deleted = + (Objects.nonNull(nativeValue.deleted()) && + nativeValue.deleted().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor(DeletedObject._typeDescriptor()), + ToDafny.DeletedObjects(nativeValue.deleted()) + ) + : Option.create_None( + DafnySequence._typeDescriptor(DeletedObject._typeDescriptor()) + ); + Option requestCharged; + requestCharged = + Objects.nonNull(nativeValue.requestCharged()) + ? Option.create_Some( + RequestCharged._typeDescriptor(), + ToDafny.RequestCharged(nativeValue.requestCharged()) + ) + : Option.create_None(RequestCharged._typeDescriptor()); + Option> errors; + errors = + (Objects.nonNull(nativeValue.errors()) && nativeValue.errors().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor(ErrorShape._typeDescriptor()), + ToDafny.Errors(nativeValue.errors()) + ) + : Option.create_None( + DafnySequence._typeDescriptor(ErrorShape._typeDescriptor()) + ); + return new DeleteObjectsOutput(deleted, requestCharged, errors); + } + + public static DeleteObjectsRequest DeleteObjectsRequest( + software.amazon.awssdk.services.s3.model.DeleteObjectsRequest nativeValue + ) { + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.bucket() + ); + Delete delete; + delete = ToDafny.Delete(nativeValue.delete()); + Option> mFA; + mFA = + Objects.nonNull(nativeValue.mfa()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.mfa() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestPayer; + requestPayer = + Objects.nonNull(nativeValue.requestPayer()) + ? Option.create_Some( + RequestPayer._typeDescriptor(), + ToDafny.RequestPayer(nativeValue.requestPayer()) + ) + : Option.create_None(RequestPayer._typeDescriptor()); + Option bypassGovernanceRetention; + bypassGovernanceRetention = + Objects.nonNull(nativeValue.bypassGovernanceRetention()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.bypassGovernanceRetention()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> expectedBucketOwner; + expectedBucketOwner = + Objects.nonNull(nativeValue.expectedBucketOwner()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expectedBucketOwner() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option checksumAlgorithm; + checksumAlgorithm = + Objects.nonNull(nativeValue.checksumAlgorithm()) + ? Option.create_Some( + ChecksumAlgorithm._typeDescriptor(), + ToDafny.ChecksumAlgorithm(nativeValue.checksumAlgorithm()) + ) + : Option.create_None(ChecksumAlgorithm._typeDescriptor()); + return new DeleteObjectsRequest( + bucket, + delete, + mFA, + requestPayer, + bypassGovernanceRetention, + expectedBucketOwner, + checksumAlgorithm + ); + } + + public static DafnySequence Errors( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::ErrorShape, + ErrorShape._typeDescriptor() + ); + } + + public static ErrorShape ErrorShape( + software.amazon.awssdk.services.s3.model.S3Error nativeValue + ) { + Option> key; + key = + Objects.nonNull(nativeValue.key()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> code; + code = + Objects.nonNull(nativeValue.code()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.code() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> message; + message = + Objects.nonNull(nativeValue.message()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.message() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new ErrorShape(key, versionId, code, message); + } + + public static GetObjectOutput GetObjectOutput(ResponseInputStream responseInputStream) { + GetObjectResponse nativeValue = responseInputStream.response(); + Option> body; + body = + Option.create_Some( + TypeDescriptor.reference(DataStream.class), + new InputStreamAsDataStream(responseInputStream) + ); + Option deleteMarker; + deleteMarker = + Objects.nonNull(nativeValue.deleteMarker()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.deleteMarker()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> acceptRanges; + acceptRanges = + Objects.nonNull(nativeValue.acceptRanges()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.acceptRanges() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> expiration; + expiration = + Objects.nonNull(nativeValue.expiration()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expiration() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> restore; + restore = + Objects.nonNull(nativeValue.restore()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.restore() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> lastModified; + lastModified = + Objects.nonNull(nativeValue.lastModified()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.lastModified() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option contentLength; + contentLength = + Objects.nonNull(nativeValue.contentLength()) + ? Option.create_Some(TypeDescriptor.LONG, (nativeValue.contentLength())) + : Option.create_None(TypeDescriptor.LONG); + Option> eTag; + eTag = + Objects.nonNull(nativeValue.eTag()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.eTag() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumCRC32; + checksumCRC32 = + Objects.nonNull(nativeValue.checksumCRC32()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumCRC32C; + checksumCRC32C = + Objects.nonNull(nativeValue.checksumCRC32C()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32C() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA1; + checksumSHA1 = + Objects.nonNull(nativeValue.checksumSHA1()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA1() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA256; + checksumSHA256 = + Objects.nonNull(nativeValue.checksumSHA256()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA256() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option missingMeta; + missingMeta = + Objects.nonNull(nativeValue.missingMeta()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.missingMeta())) + : Option.create_None(TypeDescriptor.INT); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> cacheControl; + cacheControl = + Objects.nonNull(nativeValue.cacheControl()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.cacheControl() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentDisposition; + contentDisposition = + Objects.nonNull(nativeValue.contentDisposition()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentDisposition() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentEncoding; + contentEncoding = + Objects.nonNull(nativeValue.contentEncoding()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentEncoding() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentLanguage; + contentLanguage = + Objects.nonNull(nativeValue.contentLanguage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentLanguage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentRange; + contentRange = + Objects.nonNull(nativeValue.contentRange()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentRange() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentType; + contentType = + Objects.nonNull(nativeValue.contentType()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentType() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> expires; + expires = + Objects.nonNull(nativeValue.expires()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expires() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> websiteRedirectLocation; + websiteRedirectLocation = + Objects.nonNull(nativeValue.websiteRedirectLocation()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.websiteRedirectLocation() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option serverSideEncryption; + serverSideEncryption = + Objects.nonNull(nativeValue.serverSideEncryption()) + ? Option.create_Some( + ServerSideEncryption._typeDescriptor(), + ToDafny.ServerSideEncryption(nativeValue.serverSideEncryption()) + ) + : Option.create_None(ServerSideEncryption._typeDescriptor()); + Option< + DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > + > metadata; + metadata = + (Objects.nonNull(nativeValue.metadata()) && + nativeValue.metadata().size() > 0) + ? Option.create_Some( + DafnyMap._typeDescriptor( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ), + ToDafny.Metadata(nativeValue.metadata()) + ) + : Option.create_None( + DafnyMap._typeDescriptor( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ) + ); + Option> sSECustomerAlgorithm; + sSECustomerAlgorithm = + Objects.nonNull(nativeValue.sseCustomerAlgorithm()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerAlgorithm() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKeyMD5; + sSECustomerKeyMD5 = + Objects.nonNull(nativeValue.sseCustomerKeyMD5()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKeyMD5() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = + Objects.nonNull(nativeValue.sseKMSKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseKMSKeyId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option bucketKeyEnabled; + bucketKeyEnabled = + Objects.nonNull(nativeValue.bucketKeyEnabled()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.bucketKeyEnabled()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option storageClass; + storageClass = + Objects.nonNull(nativeValue.storageClass()) + ? Option.create_Some( + StorageClass._typeDescriptor(), + ToDafny.StorageClass(nativeValue.storageClass()) + ) + : Option.create_None(StorageClass._typeDescriptor()); + Option requestCharged; + requestCharged = + Objects.nonNull(nativeValue.requestCharged()) + ? Option.create_Some( + RequestCharged._typeDescriptor(), + ToDafny.RequestCharged(nativeValue.requestCharged()) + ) + : Option.create_None(RequestCharged._typeDescriptor()); + Option replicationStatus; + replicationStatus = + Objects.nonNull(nativeValue.replicationStatus()) + ? Option.create_Some( + ReplicationStatus._typeDescriptor(), + ToDafny.ReplicationStatus(nativeValue.replicationStatus()) + ) + : Option.create_None(ReplicationStatus._typeDescriptor()); + Option partsCount; + partsCount = + Objects.nonNull(nativeValue.partsCount()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.partsCount())) + : Option.create_None(TypeDescriptor.INT); + Option tagCount; + tagCount = + Objects.nonNull(nativeValue.tagCount()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.tagCount())) + : Option.create_None(TypeDescriptor.INT); + Option objectLockMode; + objectLockMode = + Objects.nonNull(nativeValue.objectLockMode()) + ? Option.create_Some( + ObjectLockMode._typeDescriptor(), + ToDafny.ObjectLockMode(nativeValue.objectLockMode()) + ) + : Option.create_None(ObjectLockMode._typeDescriptor()); + Option> objectLockRetainUntilDate; + objectLockRetainUntilDate = + Objects.nonNull(nativeValue.objectLockRetainUntilDate()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.objectLockRetainUntilDate() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option objectLockLegalHoldStatus; + objectLockLegalHoldStatus = + Objects.nonNull(nativeValue.objectLockLegalHoldStatus()) + ? Option.create_Some( + ObjectLockLegalHoldStatus._typeDescriptor(), + ToDafny.ObjectLockLegalHoldStatus( + nativeValue.objectLockLegalHoldStatus() + ) + ) + : Option.create_None(ObjectLockLegalHoldStatus._typeDescriptor()); + return new GetObjectOutput( + body, + deleteMarker, + acceptRanges, + expiration, + restore, + lastModified, + contentLength, + eTag, + checksumCRC32, + checksumCRC32C, + checksumSHA1, + checksumSHA256, + missingMeta, + versionId, + cacheControl, + contentDisposition, + contentEncoding, + contentLanguage, + contentRange, + contentType, + expires, + websiteRedirectLocation, + serverSideEncryption, + metadata, + sSECustomerAlgorithm, + sSECustomerKeyMD5, + sSEKMSKeyId, + bucketKeyEnabled, + storageClass, + requestCharged, + replicationStatus, + partsCount, + tagCount, + objectLockMode, + objectLockRetainUntilDate, + objectLockLegalHoldStatus + ); + } + + public static GetObjectRequest GetObjectRequest( + software.amazon.awssdk.services.s3.model.GetObjectRequest nativeValue + ) { + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.bucket() + ); + Option> ifMatch; + ifMatch = + Objects.nonNull(nativeValue.ifMatch()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ifMatch() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> ifModifiedSince; + ifModifiedSince = + Objects.nonNull(nativeValue.ifModifiedSince()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ifModifiedSince() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> ifNoneMatch; + ifNoneMatch = + Objects.nonNull(nativeValue.ifNoneMatch()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ifNoneMatch() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> ifUnmodifiedSince; + ifUnmodifiedSince = + Objects.nonNull(nativeValue.ifUnmodifiedSince()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ifUnmodifiedSince() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + DafnySequence key; + key = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ); + Option> range; + range = + Objects.nonNull(nativeValue.range()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.range() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseCacheControl; + responseCacheControl = + Objects.nonNull(nativeValue.responseCacheControl()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseCacheControl() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseContentDisposition; + responseContentDisposition = + Objects.nonNull(nativeValue.responseContentDisposition()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseContentDisposition() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseContentEncoding; + responseContentEncoding = + Objects.nonNull(nativeValue.responseContentEncoding()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseContentEncoding() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseContentLanguage; + responseContentLanguage = + Objects.nonNull(nativeValue.responseContentLanguage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseContentLanguage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseContentType; + responseContentType = + Objects.nonNull(nativeValue.responseContentType()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseContentType() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> responseExpires; + responseExpires = + Objects.nonNull(nativeValue.responseExpires()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.responseExpires() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerAlgorithm; + sSECustomerAlgorithm = + Objects.nonNull(nativeValue.sseCustomerAlgorithm()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerAlgorithm() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKey; + sSECustomerKey = + Objects.nonNull(nativeValue.sseCustomerKey()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKey() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKeyMD5; + sSECustomerKeyMD5 = + Objects.nonNull(nativeValue.sseCustomerKeyMD5()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKeyMD5() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestPayer; + requestPayer = + Objects.nonNull(nativeValue.requestPayer()) + ? Option.create_Some( + RequestPayer._typeDescriptor(), + ToDafny.RequestPayer(nativeValue.requestPayer()) + ) + : Option.create_None(RequestPayer._typeDescriptor()); + Option partNumber; + partNumber = + Objects.nonNull(nativeValue.partNumber()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.partNumber())) + : Option.create_None(TypeDescriptor.INT); + Option> expectedBucketOwner; + expectedBucketOwner = + Objects.nonNull(nativeValue.expectedBucketOwner()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expectedBucketOwner() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option checksumMode; + checksumMode = + Objects.nonNull(nativeValue.checksumMode()) + ? Option.create_Some( + ChecksumMode._typeDescriptor(), + ToDafny.ChecksumMode(nativeValue.checksumMode()) + ) + : Option.create_None(ChecksumMode._typeDescriptor()); + return new GetObjectRequest( + bucket, + ifMatch, + ifModifiedSince, + ifNoneMatch, + ifUnmodifiedSince, + key, + range, + responseCacheControl, + responseContentDisposition, + responseContentEncoding, + responseContentLanguage, + responseContentType, + responseExpires, + versionId, + sSECustomerAlgorithm, + sSECustomerKey, + sSECustomerKeyMD5, + requestPayer, + partNumber, + expectedBucketOwner, + checksumMode + ); + } + + public static ListObjectsV2Output ListObjectsV2Output( + ListObjectsV2Response nativeValue + ) { + Option isTruncated; + isTruncated = + Objects.nonNull(nativeValue.isTruncated()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.isTruncated()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> contents; + contents = + (Objects.nonNull(nativeValue.contents()) && + nativeValue.contents().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor(S3Object._typeDescriptor()), + ToDafny.ObjectList(nativeValue.contents()) + ) + : Option.create_None( + DafnySequence._typeDescriptor(S3Object._typeDescriptor()) + ); + Option> name; + name = + Objects.nonNull(nativeValue.name()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.name() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> prefix; + prefix = + Objects.nonNull(nativeValue.prefix()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.prefix() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> delimiter; + delimiter = + Objects.nonNull(nativeValue.delimiter()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.delimiter() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option maxKeys; + maxKeys = + Objects.nonNull(nativeValue.maxKeys()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.maxKeys())) + : Option.create_None(TypeDescriptor.INT); + Option> commonPrefixes; + commonPrefixes = + (Objects.nonNull(nativeValue.commonPrefixes()) && + nativeValue.commonPrefixes().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor(CommonPrefix._typeDescriptor()), + ToDafny.CommonPrefixList(nativeValue.commonPrefixes()) + ) + : Option.create_None( + DafnySequence._typeDescriptor(CommonPrefix._typeDescriptor()) + ); + Option encodingType; + encodingType = + Objects.nonNull(nativeValue.encodingType()) + ? Option.create_Some( + EncodingType._typeDescriptor(), + ToDafny.EncodingType(nativeValue.encodingType()) + ) + : Option.create_None(EncodingType._typeDescriptor()); + Option keyCount; + keyCount = + Objects.nonNull(nativeValue.keyCount()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.keyCount())) + : Option.create_None(TypeDescriptor.INT); + Option> continuationToken; + continuationToken = + Objects.nonNull(nativeValue.continuationToken()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.continuationToken() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> nextContinuationToken; + nextContinuationToken = + Objects.nonNull(nativeValue.nextContinuationToken()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.nextContinuationToken() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> startAfter; + startAfter = + Objects.nonNull(nativeValue.startAfter()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.startAfter() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestCharged; + requestCharged = + Objects.nonNull(nativeValue.requestCharged()) + ? Option.create_Some( + RequestCharged._typeDescriptor(), + ToDafny.RequestCharged(nativeValue.requestCharged()) + ) + : Option.create_None(RequestCharged._typeDescriptor()); + return new ListObjectsV2Output( + isTruncated, + contents, + name, + prefix, + delimiter, + maxKeys, + commonPrefixes, + encodingType, + keyCount, + continuationToken, + nextContinuationToken, + startAfter, + requestCharged + ); + } + + public static ListObjectsV2Request ListObjectsV2Request( + software.amazon.awssdk.services.s3.model.ListObjectsV2Request nativeValue + ) { + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.bucket() + ); + Option> delimiter; + delimiter = + Objects.nonNull(nativeValue.delimiter()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.delimiter() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option encodingType; + encodingType = + Objects.nonNull(nativeValue.encodingType()) + ? Option.create_Some( + EncodingType._typeDescriptor(), + ToDafny.EncodingType(nativeValue.encodingType()) + ) + : Option.create_None(EncodingType._typeDescriptor()); + Option maxKeys; + maxKeys = + Objects.nonNull(nativeValue.maxKeys()) + ? Option.create_Some(TypeDescriptor.INT, (nativeValue.maxKeys())) + : Option.create_None(TypeDescriptor.INT); + Option> prefix; + prefix = + Objects.nonNull(nativeValue.prefix()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.prefix() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> continuationToken; + continuationToken = + Objects.nonNull(nativeValue.continuationToken()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.continuationToken() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option fetchOwner; + fetchOwner = + Objects.nonNull(nativeValue.fetchOwner()) + ? Option.create_Some(TypeDescriptor.BOOLEAN, (nativeValue.fetchOwner())) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> startAfter; + startAfter = + Objects.nonNull(nativeValue.startAfter()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.startAfter() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option requestPayer; + requestPayer = + Objects.nonNull(nativeValue.requestPayer()) + ? Option.create_Some( + RequestPayer._typeDescriptor(), + ToDafny.RequestPayer(nativeValue.requestPayer()) + ) + : Option.create_None(RequestPayer._typeDescriptor()); + Option> expectedBucketOwner; + expectedBucketOwner = + Objects.nonNull(nativeValue.expectedBucketOwner()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expectedBucketOwner() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option< + DafnySequence + > optionalObjectAttributes; + optionalObjectAttributes = + (Objects.nonNull(nativeValue.optionalObjectAttributes()) && + nativeValue.optionalObjectAttributes().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor( + OptionalObjectAttributes._typeDescriptor() + ), + ToDafny.OptionalObjectAttributesList( + nativeValue.optionalObjectAttributes() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor( + OptionalObjectAttributes._typeDescriptor() + ) + ); + return new ListObjectsV2Request( + bucket, + delimiter, + encodingType, + maxKeys, + prefix, + continuationToken, + fetchOwner, + startAfter, + requestPayer, + expectedBucketOwner, + optionalObjectAttributes + ); + } + + public static DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > Metadata(Map nativeValue) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToMap( + nativeValue, + software.amazon.smithy.dafny.conversion.ToDafny.Simple::CharacterSequence, + software.amazon.smithy.dafny.conversion.ToDafny.Simple::CharacterSequence + ); + } + + public static S3Object Object( + software.amazon.awssdk.services.s3.model.S3Object nativeValue + ) { + Option> key; + key = + Objects.nonNull(nativeValue.key()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> lastModified; + lastModified = + Objects.nonNull(nativeValue.lastModified()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.lastModified() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> eTag; + eTag = + Objects.nonNull(nativeValue.eTag()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.eTag() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumAlgorithm; + checksumAlgorithm = + (Objects.nonNull(nativeValue.checksumAlgorithm()) && + nativeValue.checksumAlgorithm().size() > 0) + ? Option.create_Some( + DafnySequence._typeDescriptor(ChecksumAlgorithm._typeDescriptor()), + ToDafny.ChecksumAlgorithmList(nativeValue.checksumAlgorithm()) + ) + : Option.create_None( + DafnySequence._typeDescriptor(ChecksumAlgorithm._typeDescriptor()) + ); + Option size; + size = + Objects.nonNull(nativeValue.size()) + ? Option.create_Some(TypeDescriptor.LONG, (nativeValue.size())) + : Option.create_None(TypeDescriptor.LONG); + Option storageClass; + storageClass = + Objects.nonNull(nativeValue.storageClass()) + ? Option.create_Some( + ObjectStorageClass._typeDescriptor(), + ToDafny.ObjectStorageClass(nativeValue.storageClass()) + ) + : Option.create_None(ObjectStorageClass._typeDescriptor()); + Option owner; + owner = + Objects.nonNull(nativeValue.owner()) + ? Option.create_Some( + Owner._typeDescriptor(), + ToDafny.Owner(nativeValue.owner()) + ) + : Option.create_None(Owner._typeDescriptor()); + Option restoreStatus; + restoreStatus = + Objects.nonNull(nativeValue.restoreStatus()) + ? Option.create_Some( + RestoreStatus._typeDescriptor(), + ToDafny.RestoreStatus(nativeValue.restoreStatus()) + ) + : Option.create_None(RestoreStatus._typeDescriptor()); + return new S3Object( + key, + lastModified, + eTag, + checksumAlgorithm, + size, + storageClass, + owner, + restoreStatus + ); + } + + public static ObjectIdentifier ObjectIdentifier( + software.amazon.awssdk.services.s3.model.ObjectIdentifier nativeValue + ) { + DafnySequence key; + key = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new ObjectIdentifier(key, versionId); + } + + public static DafnySequence ObjectIdentifierList( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::ObjectIdentifier, + ObjectIdentifier._typeDescriptor() + ); + } + + public static DafnySequence ObjectList( + List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::Object, + S3Object._typeDescriptor() + ); + } + + public static DafnySequence< + ? extends OptionalObjectAttributes + > OptionalObjectAttributesList( + List< + software.amazon.awssdk.services.s3.model.OptionalObjectAttributes + > nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, + software.amazon.cryptography.services.s3.internaldafny.ToDafny::OptionalObjectAttributes, + OptionalObjectAttributes._typeDescriptor() + ); + } + + public static Owner Owner( + software.amazon.awssdk.services.s3.model.Owner nativeValue + ) { + Option> displayName; + displayName = + Objects.nonNull(nativeValue.displayName()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.displayName() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> iD; + iD = + Objects.nonNull(nativeValue.id()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.id() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new Owner(displayName, iD); + } + + public static PutObjectOutput PutObjectOutput(PutObjectResponse nativeValue) { + Option> expiration; + expiration = + Objects.nonNull(nativeValue.expiration()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expiration() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> eTag; + eTag = + Objects.nonNull(nativeValue.eTag()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.eTag() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumCRC32; + checksumCRC32 = + Objects.nonNull(nativeValue.checksumCRC32()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumCRC32C; + checksumCRC32C = + Objects.nonNull(nativeValue.checksumCRC32C()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32C() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA1; + checksumSHA1 = + Objects.nonNull(nativeValue.checksumSHA1()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA1() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA256; + checksumSHA256 = + Objects.nonNull(nativeValue.checksumSHA256()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA256() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option serverSideEncryption; + serverSideEncryption = + Objects.nonNull(nativeValue.serverSideEncryption()) + ? Option.create_Some( + ServerSideEncryption._typeDescriptor(), + ToDafny.ServerSideEncryption(nativeValue.serverSideEncryption()) + ) + : Option.create_None(ServerSideEncryption._typeDescriptor()); + Option> versionId; + versionId = + Objects.nonNull(nativeValue.versionId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.versionId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerAlgorithm; + sSECustomerAlgorithm = + Objects.nonNull(nativeValue.sseCustomerAlgorithm()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerAlgorithm() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKeyMD5; + sSECustomerKeyMD5 = + Objects.nonNull(nativeValue.sseCustomerKeyMD5()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKeyMD5() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = + Objects.nonNull(nativeValue.sseKMSKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseKMSKeyId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSEKMSEncryptionContext; + sSEKMSEncryptionContext = + Objects.nonNull(nativeValue.sseKMSEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseKMSEncryptionContext() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option bucketKeyEnabled; + bucketKeyEnabled = + Objects.nonNull(nativeValue.bucketKeyEnabled()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.bucketKeyEnabled()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option requestCharged; + requestCharged = + Objects.nonNull(nativeValue.requestCharged()) + ? Option.create_Some( + RequestCharged._typeDescriptor(), + ToDafny.RequestCharged(nativeValue.requestCharged()) + ) + : Option.create_None(RequestCharged._typeDescriptor()); + return new PutObjectOutput( + expiration, + eTag, + checksumCRC32, + checksumCRC32C, + checksumSHA1, + checksumSHA256, + serverSideEncryption, + versionId, + sSECustomerAlgorithm, + sSECustomerKeyMD5, + sSEKMSKeyId, + sSEKMSEncryptionContext, + bucketKeyEnabled, + requestCharged + ); + } + + public static PutObjectRequest PutObjectRequest( + software.amazon.awssdk.services.s3.model.PutObjectRequest nativeValue + ) { + Option aCL; + aCL = + Objects.nonNull(nativeValue.acl()) + ? Option.create_Some( + ObjectCannedACL._typeDescriptor(), + ToDafny.ObjectCannedACL(nativeValue.acl()) + ) + : Option.create_None(ObjectCannedACL._typeDescriptor()); + Option> body; +// body = +// Objects.nonNull(nativeValue.body()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.BYTE), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.ByteSequence( +// nativeValue.body().asByteArray() +// ) +// ) +// : Option.create_None( +// DafnySequence._typeDescriptor(TypeDescriptor.BYTE) +// ); + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.bucket() + ); + Option> cacheControl; + cacheControl = + Objects.nonNull(nativeValue.cacheControl()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.cacheControl() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentDisposition; + contentDisposition = + Objects.nonNull(nativeValue.contentDisposition()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentDisposition() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentEncoding; + contentEncoding = + Objects.nonNull(nativeValue.contentEncoding()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentEncoding() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentLanguage; + contentLanguage = + Objects.nonNull(nativeValue.contentLanguage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentLanguage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option contentLength; + contentLength = + Objects.nonNull(nativeValue.contentLength()) + ? Option.create_Some(TypeDescriptor.LONG, (nativeValue.contentLength())) + : Option.create_None(TypeDescriptor.LONG); + Option> contentMD5; + contentMD5 = + Objects.nonNull(nativeValue.contentMD5()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentMD5() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> contentType; + contentType = + Objects.nonNull(nativeValue.contentType()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.contentType() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option checksumAlgorithm; + checksumAlgorithm = + Objects.nonNull(nativeValue.checksumAlgorithm()) + ? Option.create_Some( + ChecksumAlgorithm._typeDescriptor(), + ToDafny.ChecksumAlgorithm(nativeValue.checksumAlgorithm()) + ) + : Option.create_None(ChecksumAlgorithm._typeDescriptor()); + Option> checksumCRC32; + checksumCRC32 = + Objects.nonNull(nativeValue.checksumCRC32()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumCRC32C; + checksumCRC32C = + Objects.nonNull(nativeValue.checksumCRC32C()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumCRC32C() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA1; + checksumSHA1 = + Objects.nonNull(nativeValue.checksumSHA1()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA1() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> checksumSHA256; + checksumSHA256 = + Objects.nonNull(nativeValue.checksumSHA256()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.checksumSHA256() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> expires; + expires = + Objects.nonNull(nativeValue.expires()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expires() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> grantFullControl; + grantFullControl = + Objects.nonNull(nativeValue.grantFullControl()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.grantFullControl() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> grantRead; + grantRead = + Objects.nonNull(nativeValue.grantRead()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.grantRead() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> grantReadACP; + grantReadACP = + Objects.nonNull(nativeValue.grantReadACP()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.grantReadACP() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> grantWriteACP; + grantWriteACP = + Objects.nonNull(nativeValue.grantWriteACP()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.grantWriteACP() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + DafnySequence key; + key = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.key() + ); + Option< + DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > + > metadata; + metadata = + (Objects.nonNull(nativeValue.metadata()) && + nativeValue.metadata().size() > 0) + ? Option.create_Some( + DafnyMap._typeDescriptor( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ), + ToDafny.Metadata(nativeValue.metadata()) + ) + : Option.create_None( + DafnyMap._typeDescriptor( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ) + ); + Option serverSideEncryption; + serverSideEncryption = + Objects.nonNull(nativeValue.serverSideEncryption()) + ? Option.create_Some( + ServerSideEncryption._typeDescriptor(), + ToDafny.ServerSideEncryption(nativeValue.serverSideEncryption()) + ) + : Option.create_None(ServerSideEncryption._typeDescriptor()); + Option storageClass; + storageClass = + Objects.nonNull(nativeValue.storageClass()) + ? Option.create_Some( + StorageClass._typeDescriptor(), + ToDafny.StorageClass(nativeValue.storageClass()) + ) + : Option.create_None(StorageClass._typeDescriptor()); + Option> websiteRedirectLocation; + websiteRedirectLocation = + Objects.nonNull(nativeValue.websiteRedirectLocation()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.websiteRedirectLocation() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerAlgorithm; + sSECustomerAlgorithm = + Objects.nonNull(nativeValue.sseCustomerAlgorithm()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerAlgorithm() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKey; + sSECustomerKey = + Objects.nonNull(nativeValue.sseCustomerKey()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKey() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSECustomerKeyMD5; + sSECustomerKeyMD5 = + Objects.nonNull(nativeValue.sseCustomerKeyMD5()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseCustomerKeyMD5() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = + Objects.nonNull(nativeValue.sseKMSKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseKMSKeyId() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option> sSEKMSEncryptionContext; + sSEKMSEncryptionContext = + Objects.nonNull(nativeValue.sseKMSEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.sseKMSEncryptionContext() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option bucketKeyEnabled; + bucketKeyEnabled = + Objects.nonNull(nativeValue.bucketKeyEnabled()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.bucketKeyEnabled()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option requestPayer; + requestPayer = + Objects.nonNull(nativeValue.requestPayer()) + ? Option.create_Some( + RequestPayer._typeDescriptor(), + ToDafny.RequestPayer(nativeValue.requestPayer()) + ) + : Option.create_None(RequestPayer._typeDescriptor()); + Option> tagging; + tagging = + Objects.nonNull(nativeValue.tagging()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.tagging() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option objectLockMode; + objectLockMode = + Objects.nonNull(nativeValue.objectLockMode()) + ? Option.create_Some( + ObjectLockMode._typeDescriptor(), + ToDafny.ObjectLockMode(nativeValue.objectLockMode()) + ) + : Option.create_None(ObjectLockMode._typeDescriptor()); + Option> objectLockRetainUntilDate; + objectLockRetainUntilDate = + Objects.nonNull(nativeValue.objectLockRetainUntilDate()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.objectLockRetainUntilDate() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + Option objectLockLegalHoldStatus; + objectLockLegalHoldStatus = + Objects.nonNull(nativeValue.objectLockLegalHoldStatus()) + ? Option.create_Some( + ObjectLockLegalHoldStatus._typeDescriptor(), + ToDafny.ObjectLockLegalHoldStatus( + nativeValue.objectLockLegalHoldStatus() + ) + ) + : Option.create_None(ObjectLockLegalHoldStatus._typeDescriptor()); + Option> expectedBucketOwner; + expectedBucketOwner = + Objects.nonNull(nativeValue.expectedBucketOwner()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.expectedBucketOwner() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new PutObjectRequest( + aCL, + body, + bucket, + cacheControl, + contentDisposition, + contentEncoding, + contentLanguage, + contentLength, + contentMD5, + contentType, + checksumAlgorithm, + checksumCRC32, + checksumCRC32C, + checksumSHA1, + checksumSHA256, + expires, + grantFullControl, + grantRead, + grantReadACP, + grantWriteACP, + key, + metadata, + serverSideEncryption, + storageClass, + websiteRedirectLocation, + sSECustomerAlgorithm, + sSECustomerKey, + sSECustomerKeyMD5, + sSEKMSKeyId, + sSEKMSEncryptionContext, + bucketKeyEnabled, + requestPayer, + tagging, + objectLockMode, + objectLockRetainUntilDate, + objectLockLegalHoldStatus, + expectedBucketOwner + ); + } + + public static RestoreStatus RestoreStatus( + software.amazon.awssdk.services.s3.model.RestoreStatus nativeValue + ) { + Option isRestoreInProgress; + isRestoreInProgress = + Objects.nonNull(nativeValue.isRestoreInProgress()) + ? Option.create_Some( + TypeDescriptor.BOOLEAN, + (nativeValue.isRestoreInProgress()) + ) + : Option.create_None(TypeDescriptor.BOOLEAN); + Option> restoreExpiryDate; + restoreExpiryDate = + Objects.nonNull(nativeValue.restoreExpiryDate()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.restoreExpiryDate() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new RestoreStatus(isRestoreInProgress, restoreExpiryDate); + } + + public static Error Error(InvalidObjectStateException nativeValue) { + Option storageClass; + storageClass = + Objects.nonNull(nativeValue.storageClass()) + ? Option.create_Some( + StorageClass._typeDescriptor(), + ToDafny.StorageClass(nativeValue.storageClass()) + ) + : Option.create_None(StorageClass._typeDescriptor()); + Option accessTier; + accessTier = + Objects.nonNull(nativeValue.accessTier()) + ? Option.create_Some( + IntelligentTieringAccessTier._typeDescriptor(), + ToDafny.IntelligentTieringAccessTier(nativeValue.accessTier()) + ) + : Option.create_None(IntelligentTieringAccessTier._typeDescriptor()); + Option> message; + message = + Objects.nonNull(nativeValue.getMessage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.getMessage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new Error_InvalidObjectState(storageClass, accessTier, message); + } + + public static Error Error(NoSuchBucketException nativeValue) { + Option> message; + message = + Objects.nonNull(nativeValue.getMessage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.getMessage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new Error_NoSuchBucket(message); + } + + public static Error Error(NoSuchKeyException nativeValue) { + Option> message; + message = + Objects.nonNull(nativeValue.getMessage()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.getMessage() + ) + ) + : Option.create_None( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR) + ); + return new Error_NoSuchKey(message); + } + + public static ChecksumAlgorithm ChecksumAlgorithm( + software.amazon.awssdk.services.s3.model.ChecksumAlgorithm nativeValue + ) { + switch (nativeValue) { + case CRC32: + { + return ChecksumAlgorithm.create_CRC32(); + } + case CRC32_C: + { + return ChecksumAlgorithm.create_CRC32C(); + } + case SHA1: + { + return ChecksumAlgorithm.create_SHA1(); + } + case SHA256: + { + return ChecksumAlgorithm.create_SHA256(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm." + ); + } + } + } + + public static ChecksumMode ChecksumMode( + software.amazon.awssdk.services.s3.model.ChecksumMode nativeValue + ) { + switch (nativeValue) { + case ENABLED: + { + return ChecksumMode.create(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ChecksumMode." + ); + } + } + } + + public static EncodingType EncodingType( + software.amazon.awssdk.services.s3.model.EncodingType nativeValue + ) { + switch (nativeValue) { + case URL: + { + return EncodingType.create(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.EncodingType." + ); + } + } + } + + public static IntelligentTieringAccessTier IntelligentTieringAccessTier( + software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier nativeValue + ) { + switch (nativeValue) { + case ARCHIVE_ACCESS: + { + return IntelligentTieringAccessTier.create_ARCHIVE__ACCESS(); + } + case DEEP_ARCHIVE_ACCESS: + { + return IntelligentTieringAccessTier.create_DEEP__ARCHIVE__ACCESS(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.IntelligentTieringAccessTier." + ); + } + } + } + + public static ObjectCannedACL ObjectCannedACL( + software.amazon.awssdk.services.s3.model.ObjectCannedACL nativeValue + ) { + switch (nativeValue) { + case PRIVATE: + { + return ObjectCannedACL.create_private(); + } + case PUBLIC_READ: + { + return ObjectCannedACL.create_public__read(); + } + case PUBLIC_READ_WRITE: + { + return ObjectCannedACL.create_public__read__write(); + } + case AUTHENTICATED_READ: + { + return ObjectCannedACL.create_authenticated__read(); + } + case AWS_EXEC_READ: + { + return ObjectCannedACL.create_aws__exec__read(); + } + case BUCKET_OWNER_READ: + { + return ObjectCannedACL.create_bucket__owner__read(); + } + case BUCKET_OWNER_FULL_CONTROL: + { + return ObjectCannedACL.create_bucket__owner__full__control(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ObjectCannedACL." + ); + } + } + } + + public static ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus( + software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus nativeValue + ) { + switch (nativeValue) { + case ON: + { + return ObjectLockLegalHoldStatus.create_ON(); + } + case OFF: + { + return ObjectLockLegalHoldStatus.create_OFF(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus." + ); + } + } + } + + public static ObjectLockMode ObjectLockMode( + software.amazon.awssdk.services.s3.model.ObjectLockMode nativeValue + ) { + switch (nativeValue) { + case GOVERNANCE: + { + return ObjectLockMode.create_GOVERNANCE(); + } + case COMPLIANCE: + { + return ObjectLockMode.create_COMPLIANCE(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockMode." + ); + } + } + } + + public static ObjectStorageClass ObjectStorageClass( + software.amazon.awssdk.services.s3.model.ObjectStorageClass nativeValue + ) { + switch (nativeValue) { + case STANDARD: + { + return ObjectStorageClass.create_STANDARD(); + } + case REDUCED_REDUNDANCY: + { + return ObjectStorageClass.create_REDUCED__REDUNDANCY(); + } + case GLACIER: + { + return ObjectStorageClass.create_GLACIER(); + } + case STANDARD_IA: + { + return ObjectStorageClass.create_STANDARD__IA(); + } + case ONEZONE_IA: + { + return ObjectStorageClass.create_ONEZONE__IA(); + } + case INTELLIGENT_TIERING: + { + return ObjectStorageClass.create_INTELLIGENT__TIERING(); + } + case DEEP_ARCHIVE: + { + return ObjectStorageClass.create_DEEP__ARCHIVE(); + } + case OUTPOSTS: + { + return ObjectStorageClass.create_OUTPOSTS(); + } + case GLACIER_IR: + { + return ObjectStorageClass.create_GLACIER__IR(); + } + case SNOW: + { + return ObjectStorageClass.create_SNOW(); + } + case EXPRESS_ONEZONE: + { + return ObjectStorageClass.create_EXPRESS__ONEZONE(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass." + ); + } + } + } + + public static OptionalObjectAttributes OptionalObjectAttributes( + software.amazon.awssdk.services.s3.model.OptionalObjectAttributes nativeValue + ) { + switch (nativeValue) { + case RESTORE_STATUS: + { + return OptionalObjectAttributes.create(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes." + ); + } + } + } + + public static ReplicationStatus ReplicationStatus( + software.amazon.awssdk.services.s3.model.ReplicationStatus nativeValue + ) { + switch (nativeValue) { + case COMPLETE: + { + return ReplicationStatus.create_COMPLETE(); + } + case PENDING: + { + return ReplicationStatus.create_PENDING(); + } + case FAILED: + { + return ReplicationStatus.create_FAILED(); + } + case REPLICA: + { + return ReplicationStatus.create_REPLICA(); + } + case COMPLETED: + { + return ReplicationStatus.create_COMPLETED(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ReplicationStatus." + ); + } + } + } + + public static RequestCharged RequestCharged( + software.amazon.awssdk.services.s3.model.RequestCharged nativeValue + ) { + switch (nativeValue) { + case REQUESTER: + { + return RequestCharged.create(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.RequestCharged." + ); + } + } + } + + public static RequestPayer RequestPayer( + software.amazon.awssdk.services.s3.model.RequestPayer nativeValue + ) { + switch (nativeValue) { + case REQUESTER: + { + return RequestPayer.create(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.RequestPayer." + ); + } + } + } + + public static ServerSideEncryption ServerSideEncryption( + software.amazon.awssdk.services.s3.model.ServerSideEncryption nativeValue + ) { + switch (nativeValue) { + case AES256: + { + return ServerSideEncryption.create_AES256(); + } + case AWS_KMS: + { + return ServerSideEncryption.create_aws__kms(); + } + case AWS_KMS_DSSE: + { + return ServerSideEncryption.create_aws__kms__dsse(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption." + ); + } + } + } + + public static StorageClass StorageClass( + software.amazon.awssdk.services.s3.model.StorageClass nativeValue + ) { + switch (nativeValue) { + case STANDARD: + { + return StorageClass.create_STANDARD(); + } + case REDUCED_REDUNDANCY: + { + return StorageClass.create_REDUCED__REDUNDANCY(); + } + case STANDARD_IA: + { + return StorageClass.create_STANDARD__IA(); + } + case ONEZONE_IA: + { + return StorageClass.create_ONEZONE__IA(); + } + case INTELLIGENT_TIERING: + { + return StorageClass.create_INTELLIGENT__TIERING(); + } + case GLACIER: + { + return StorageClass.create_GLACIER(); + } + case DEEP_ARCHIVE: + { + return StorageClass.create_DEEP__ARCHIVE(); + } + case OUTPOSTS: + { + return StorageClass.create_OUTPOSTS(); + } + case GLACIER_IR: + { + return StorageClass.create_GLACIER__IR(); + } + case SNOW: + { + return StorageClass.create_SNOW(); + } + case EXPRESS_ONEZONE: + { + return StorageClass.create_EXPRESS__ONEZONE(); + } + default: + { + throw new RuntimeException( + "Cannot convert " + + nativeValue + + " to software.amazon.cryptography.services.s3.internaldafny.types.StorageClass." + ); + } + } + } + + public static ChecksumAlgorithm ChecksumAlgorithm(String nativeValue) { + return ChecksumAlgorithm( + software.amazon.awssdk.services.s3.model.ChecksumAlgorithm.fromValue( + nativeValue + ) + ); + } + + public static ChecksumMode ChecksumMode(String nativeValue) { + return ChecksumMode( + software.amazon.awssdk.services.s3.model.ChecksumMode.fromValue( + nativeValue + ) + ); + } + + public static EncodingType EncodingType(String nativeValue) { + return EncodingType( + software.amazon.awssdk.services.s3.model.EncodingType.fromValue( + nativeValue + ) + ); + } + + public static IntelligentTieringAccessTier IntelligentTieringAccessTier( + String nativeValue + ) { + return IntelligentTieringAccessTier( + software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier.fromValue( + nativeValue + ) + ); + } + + public static ObjectCannedACL ObjectCannedACL(String nativeValue) { + return ObjectCannedACL( + software.amazon.awssdk.services.s3.model.ObjectCannedACL.fromValue( + nativeValue + ) + ); + } + + public static ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus( + String nativeValue + ) { + return ObjectLockLegalHoldStatus( + software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus.fromValue( + nativeValue + ) + ); + } + + public static ObjectLockMode ObjectLockMode(String nativeValue) { + return ObjectLockMode( + software.amazon.awssdk.services.s3.model.ObjectLockMode.fromValue( + nativeValue + ) + ); + } + + public static ObjectStorageClass ObjectStorageClass(String nativeValue) { + return ObjectStorageClass( + software.amazon.awssdk.services.s3.model.ObjectStorageClass.fromValue( + nativeValue + ) + ); + } + + public static OptionalObjectAttributes OptionalObjectAttributes( + String nativeValue + ) { + return OptionalObjectAttributes( + software.amazon.awssdk.services.s3.model.OptionalObjectAttributes.fromValue( + nativeValue + ) + ); + } + + public static ReplicationStatus ReplicationStatus(String nativeValue) { + return ReplicationStatus( + software.amazon.awssdk.services.s3.model.ReplicationStatus.fromValue( + nativeValue + ) + ); + } + + public static RequestCharged RequestCharged(String nativeValue) { + return RequestCharged( + software.amazon.awssdk.services.s3.model.RequestCharged.fromValue( + nativeValue + ) + ); + } + + public static RequestPayer RequestPayer(String nativeValue) { + return RequestPayer( + software.amazon.awssdk.services.s3.model.RequestPayer.fromValue( + nativeValue + ) + ); + } + + public static ServerSideEncryption ServerSideEncryption(String nativeValue) { + return ServerSideEncryption( + software.amazon.awssdk.services.s3.model.ServerSideEncryption.fromValue( + nativeValue + ) + ); + } + + public static StorageClass StorageClass(String nativeValue) { + return StorageClass( + software.amazon.awssdk.services.s3.model.StorageClass.fromValue( + nativeValue + ) + ); + } + + public static Error Error(S3Exception nativeValue) { + // While this is logically identical to the other Opaque Error case, + // it is semantically distinct. + // An un-modeled Service Error is different from a Java Heap Exhaustion error. + // In the future, Smithy-Dafny MAY allow for this distinction. + // Which would allow Dafny developers to treat the two differently. + return Error.create_OpaqueWithText( + nativeValue, + dafny.DafnySequence.asString(nativeValue.getMessage()) + ); + } + + public static Error Error(Exception nativeValue) { + // While this is logically identical to the other Opaque Error case, + // it is semantically distinct. + // An un-modeled Service Error is different from a Java Heap Exhaustion error. + // In the future, Smithy-Dafny MAY allow for this distinction. + // Which would allow Dafny developers to treat the two differently. + return Error.create_OpaqueWithText( + nativeValue, + dafny.DafnySequence.asString(nativeValue.getMessage()) + ); + } + + public static IS3Client AmazonS3(S3Client nativeValue) { + return new Shim(nativeValue, null); + } +} diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java new file mode 100644 index 000000000..a7a56f88f --- /dev/null +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java @@ -0,0 +1,1725 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Do not modify this file. This file is machine generated, and any changes to it will be overwritten. +package software.amazon.cryptography.services.s3.internaldafny; + +import dafny.DafnyMap; +import dafny.DafnySequence; +import java.lang.Character; +import java.lang.IllegalStateException; +import java.lang.RuntimeException; +import java.lang.String; +import java.lang.Throwable; +import java.util.List; +import java.util.Map; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.CommonPrefix; +import software.amazon.awssdk.services.s3.model.Delete; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.DeletedObject; +import software.amazon.awssdk.services.s3.model.EncodingType; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier; +import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.OptionalObjectAttributes; +import software.amazon.awssdk.services.s3.model.RestoreStatus; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.ObjectCannedACL; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; +import software.amazon.awssdk.services.s3.model.ObjectLockMode; +import software.amazon.awssdk.services.s3.model.ObjectStorageClass; +import software.amazon.awssdk.services.s3.model.Owner; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.ReplicationStatus; +import software.amazon.awssdk.services.s3.model.RequestCharged; +import software.amazon.awssdk.services.s3.model.RequestPayer; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.ServerSideEncryption; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.Error; +import software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_InvalidObjectState; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchBucket; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchKey; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_Opaque; +import software.amazon.cryptography.services.s3.internaldafny.types.Error_OpaqueWithText; +import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectOutput; +import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; +import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; +import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; + +public class ToNative { + + public static ChecksumAlgorithm ChecksumAlgorithm( + software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm dafnyValue + ) { + if (dafnyValue.is_CRC32()) { + return ChecksumAlgorithm.CRC32; + } + if (dafnyValue.is_CRC32C()) { + return ChecksumAlgorithm.CRC32_C; + } + if (dafnyValue.is_SHA1()) { + return ChecksumAlgorithm.SHA1; + } + if (dafnyValue.is_SHA256()) { + return ChecksumAlgorithm.SHA256; + } + return ChecksumAlgorithm.fromValue(dafnyValue.toString()); + } + + public static List ChecksumAlgorithmList( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::ChecksumAlgorithm + ); + } + + public static ChecksumMode ChecksumMode( + software.amazon.cryptography.services.s3.internaldafny.types.ChecksumMode dafnyValue + ) { + if (dafnyValue.is_ENABLED()) { + return ChecksumMode.ENABLED; + } + return ChecksumMode.fromValue(dafnyValue.toString()); + } + + public static CommonPrefix CommonPrefix( + software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix dafnyValue + ) { + CommonPrefix.Builder builder = CommonPrefix.builder(); + if (dafnyValue.dtor_Prefix().is_Some()) { + builder.prefix( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Prefix().dtor_value() + ) + ); + } + return builder.build(); + } + + public static List CommonPrefixList( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::CommonPrefix + ); + } + + public static Delete Delete( + software.amazon.cryptography.services.s3.internaldafny.types.Delete dafnyValue + ) { + Delete.Builder builder = Delete.builder(); + builder.objects(ToNative.ObjectIdentifierList(dafnyValue.dtor_Objects())); + if (dafnyValue.dtor_Quiet().is_Some()) { + builder.quiet((dafnyValue.dtor_Quiet().dtor_value())); + } + return builder.build(); + } + + public static DeletedObject DeletedObject( + software.amazon.cryptography.services.s3.internaldafny.types.DeletedObject dafnyValue + ) { + DeletedObject.Builder builder = DeletedObject.builder(); + if (dafnyValue.dtor_DeleteMarker().is_Some()) { + builder.deleteMarker((dafnyValue.dtor_DeleteMarker().dtor_value())); + } + if (dafnyValue.dtor_DeleteMarkerVersionId().is_Some()) { + builder.deleteMarkerVersionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_DeleteMarkerVersionId().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Key().is_Some()) { + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key().dtor_value() + ) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static List DeletedObjects( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.DeletedObject + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::DeletedObject + ); + } + + public static DeleteObjectResponse DeleteObjectOutput( + DeleteObjectOutput dafnyValue + ) { + DeleteObjectResponse.Builder builder = DeleteObjectResponse.builder(); + if (dafnyValue.dtor_DeleteMarker().is_Some()) { + builder.deleteMarker((dafnyValue.dtor_DeleteMarker().dtor_value())); + } + if (dafnyValue.dtor_RequestCharged().is_Some()) { + builder.requestCharged( + ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static DeleteObjectRequest DeleteObjectRequest( + software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectRequest dafnyValue + ) { + DeleteObjectRequest.Builder builder = DeleteObjectRequest.builder(); + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() + ) + ); + if (dafnyValue.dtor_BypassGovernanceRetention().is_Some()) { + builder.bypassGovernanceRetention( + (dafnyValue.dtor_BypassGovernanceRetention().dtor_value()) + ); + } + if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { + builder.expectedBucketOwner( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ExpectedBucketOwner().dtor_value() + ) + ); + } + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key() + ) + ); + if (dafnyValue.dtor_MFA().is_Some()) { + builder.mfa( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_MFA().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestPayer().is_Some()) { + builder.requestPayer( + ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static DeleteObjectsResponse DeleteObjectsOutput( + DeleteObjectsOutput dafnyValue + ) { + DeleteObjectsResponse.Builder builder = DeleteObjectsResponse.builder(); + if (dafnyValue.dtor_Deleted().is_Some()) { + builder.deleted( + ToNative.DeletedObjects(dafnyValue.dtor_Deleted().dtor_value()) + ); + } + if (dafnyValue.dtor_Errors().is_Some()) { + builder.errors(ToNative.Errors(dafnyValue.dtor_Errors().dtor_value())); + } + if (dafnyValue.dtor_RequestCharged().is_Some()) { + builder.requestCharged( + ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) + ); + } + return builder.build(); + } + + public static DeleteObjectsRequest DeleteObjectsRequest( + software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsRequest dafnyValue + ) { + DeleteObjectsRequest.Builder builder = DeleteObjectsRequest.builder(); + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() + ) + ); + if (dafnyValue.dtor_BypassGovernanceRetention().is_Some()) { + builder.bypassGovernanceRetention( + (dafnyValue.dtor_BypassGovernanceRetention().dtor_value()) + ); + } + if (dafnyValue.dtor_ChecksumAlgorithm().is_Some()) { + builder.checksumAlgorithm( + ToNative.ChecksumAlgorithm( + dafnyValue.dtor_ChecksumAlgorithm().dtor_value() + ) + ); + } + builder.delete(ToNative.Delete(dafnyValue.dtor_Delete())); + if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { + builder.expectedBucketOwner( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ExpectedBucketOwner().dtor_value() + ) + ); + } + if (dafnyValue.dtor_MFA().is_Some()) { + builder.mfa( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_MFA().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestPayer().is_Some()) { + builder.requestPayer( + ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) + ); + } + return builder.build(); + } + + public static EncodingType EncodingType( + software.amazon.cryptography.services.s3.internaldafny.types.EncodingType dafnyValue + ) { + if (dafnyValue.is_url()) { + return EncodingType.URL; + } + return EncodingType.fromValue(dafnyValue.toString()); + } + + public static List Errors( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::ErrorShape + ); + } + + public static S3Error ErrorShape( + software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape dafnyValue + ) { + S3Error.Builder builder = S3Error.builder(); + if (dafnyValue.dtor_Code().is_Some()) { + builder.code( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Code().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Key().is_Some()) { + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Message().is_Some()) { + builder.message( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Message().dtor_value() + ) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static GetObjectResponse GetObjectOutput(GetObjectOutput dafnyValue) { + GetObjectResponse.Builder builder = GetObjectResponse.builder(); + if (dafnyValue.dtor_AcceptRanges().is_Some()) { + builder.acceptRanges( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_AcceptRanges().dtor_value() + ) + ); + } +// if (dafnyValue.dtor_Body().is_Some()) { +// builder.body( +// SdkBytes.fromByteArray( +// (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) +// ) +// ); +// } + if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { + builder.bucketKeyEnabled( + (dafnyValue.dtor_BucketKeyEnabled().dtor_value()) + ); + } + if (dafnyValue.dtor_CacheControl().is_Some()) { + builder.cacheControl( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_CacheControl().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumCRC32().is_Some()) { + builder.checksumCRC32( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumCRC32C().is_Some()) { + builder.checksumCRC32C( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32C().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA1().is_Some()) { + builder.checksumSHA1( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA1().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA256().is_Some()) { + builder.checksumSHA256( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA256().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentDisposition().is_Some()) { + builder.contentDisposition( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentDisposition().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentEncoding().is_Some()) { + builder.contentEncoding( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentEncoding().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentLanguage().is_Some()) { + builder.contentLanguage( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentLanguage().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentLength().is_Some()) { + builder.contentLength((dafnyValue.dtor_ContentLength().dtor_value())); + } + if (dafnyValue.dtor_ContentRange().is_Some()) { + builder.contentRange( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentRange().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentType().is_Some()) { + builder.contentType( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentType().dtor_value() + ) + ); + } + if (dafnyValue.dtor_DeleteMarker().is_Some()) { + builder.deleteMarker((dafnyValue.dtor_DeleteMarker().dtor_value())); + } + if (dafnyValue.dtor_ETag().is_Some()) { + builder.eTag( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ETag().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Expiration().is_Some()) { + builder.expiration( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Expiration().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Expires().is_Some()) { + builder.expires( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_Expires().dtor_value() + ) + ); + } + if (dafnyValue.dtor_LastModified().is_Some()) { + builder.lastModified( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_LastModified().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Metadata().is_Some()) { + builder.metadata( + ToNative.Metadata(dafnyValue.dtor_Metadata().dtor_value()) + ); + } + if (dafnyValue.dtor_MissingMeta().is_Some()) { + builder.missingMeta((dafnyValue.dtor_MissingMeta().dtor_value())); + } + if (dafnyValue.dtor_ObjectLockLegalHoldStatus().is_Some()) { + builder.objectLockLegalHoldStatus( + ToNative.ObjectLockLegalHoldStatus( + dafnyValue.dtor_ObjectLockLegalHoldStatus().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ObjectLockMode().is_Some()) { + builder.objectLockMode( + ToNative.ObjectLockMode(dafnyValue.dtor_ObjectLockMode().dtor_value()) + ); + } + if (dafnyValue.dtor_ObjectLockRetainUntilDate().is_Some()) { + builder.objectLockRetainUntilDate( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_ObjectLockRetainUntilDate().dtor_value() + ) + ); + } + if (dafnyValue.dtor_PartsCount().is_Some()) { + builder.partsCount((dafnyValue.dtor_PartsCount().dtor_value())); + } + if (dafnyValue.dtor_ReplicationStatus().is_Some()) { + builder.replicationStatus( + ToNative.ReplicationStatus( + dafnyValue.dtor_ReplicationStatus().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestCharged().is_Some()) { + builder.requestCharged( + ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) + ); + } + if (dafnyValue.dtor_Restore().is_Some()) { + builder.restore( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Restore().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ServerSideEncryption().is_Some()) { + builder.serverSideEncryption( + ToNative.ServerSideEncryption( + dafnyValue.dtor_ServerSideEncryption().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerAlgorithm().is_Some()) { + builder.sseCustomerAlgorithm( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKeyMD5().is_Some()) { + builder.sseCustomerKeyMD5( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKeyMD5().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { + builder.sseKMSKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); + } + if (dafnyValue.dtor_StorageClass().is_Some()) { + builder.storageClass( + ToNative.StorageClass(dafnyValue.dtor_StorageClass().dtor_value()) + ); + } + if (dafnyValue.dtor_TagCount().is_Some()) { + builder.tagCount((dafnyValue.dtor_TagCount().dtor_value())); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + if (dafnyValue.dtor_WebsiteRedirectLocation().is_Some()) { + builder.websiteRedirectLocation( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_WebsiteRedirectLocation().dtor_value() + ) + ); + } + return builder.build(); + } + + public static GetObjectRequest GetObjectRequest( + software.amazon.cryptography.services.s3.internaldafny.types.GetObjectRequest dafnyValue + ) { + GetObjectRequest.Builder builder = GetObjectRequest.builder(); + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() + ) + ); + if (dafnyValue.dtor_ChecksumMode().is_Some()) { + builder.checksumMode( + ToNative.ChecksumMode(dafnyValue.dtor_ChecksumMode().dtor_value()) + ); + } + if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { + builder.expectedBucketOwner( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ExpectedBucketOwner().dtor_value() + ) + ); + } + if (dafnyValue.dtor_IfMatch().is_Some()) { + builder.ifMatch( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_IfMatch().dtor_value() + ) + ); + } + if (dafnyValue.dtor_IfModifiedSince().is_Some()) { + builder.ifModifiedSince( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_IfModifiedSince().dtor_value() + ) + ); + } + if (dafnyValue.dtor_IfNoneMatch().is_Some()) { + builder.ifNoneMatch( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_IfNoneMatch().dtor_value() + ) + ); + } + if (dafnyValue.dtor_IfUnmodifiedSince().is_Some()) { + builder.ifUnmodifiedSince( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_IfUnmodifiedSince().dtor_value() + ) + ); + } + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key() + ) + ); + if (dafnyValue.dtor_PartNumber().is_Some()) { + builder.partNumber((dafnyValue.dtor_PartNumber().dtor_value())); + } + if (dafnyValue.dtor_Range().is_Some()) { + builder.range( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Range().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestPayer().is_Some()) { + builder.requestPayer( + ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) + ); + } + if (dafnyValue.dtor_ResponseCacheControl().is_Some()) { + builder.responseCacheControl( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ResponseCacheControl().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ResponseContentDisposition().is_Some()) { + builder.responseContentDisposition( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ResponseContentDisposition().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ResponseContentEncoding().is_Some()) { + builder.responseContentEncoding( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ResponseContentEncoding().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ResponseContentLanguage().is_Some()) { + builder.responseContentLanguage( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ResponseContentLanguage().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ResponseContentType().is_Some()) { + builder.responseContentType( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ResponseContentType().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ResponseExpires().is_Some()) { + builder.responseExpires( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_ResponseExpires().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerAlgorithm().is_Some()) { + builder.sseCustomerAlgorithm( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKey().is_Some()) { + builder.sseCustomerKey( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKey().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKeyMD5().is_Some()) { + builder.sseCustomerKeyMD5( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKeyMD5().dtor_value() + ) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static IntelligentTieringAccessTier IntelligentTieringAccessTier( + software.amazon.cryptography.services.s3.internaldafny.types.IntelligentTieringAccessTier dafnyValue + ) { + if (dafnyValue.is_ARCHIVE__ACCESS()) { + return IntelligentTieringAccessTier.ARCHIVE_ACCESS; + } + if (dafnyValue.is_DEEP__ARCHIVE__ACCESS()) { + return IntelligentTieringAccessTier.DEEP_ARCHIVE_ACCESS; + } + return IntelligentTieringAccessTier.fromValue(dafnyValue.toString()); + } + + public static ListObjectsV2Response ListObjectsV2Output( + ListObjectsV2Output dafnyValue + ) { + ListObjectsV2Response.Builder builder = ListObjectsV2Response.builder(); + if (dafnyValue.dtor_CommonPrefixes().is_Some()) { + builder.commonPrefixes( + ToNative.CommonPrefixList(dafnyValue.dtor_CommonPrefixes().dtor_value()) + ); + } + if (dafnyValue.dtor_Contents().is_Some()) { + builder.contents( + ToNative.ObjectList(dafnyValue.dtor_Contents().dtor_value()) + ); + } + if (dafnyValue.dtor_ContinuationToken().is_Some()) { + builder.continuationToken( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContinuationToken().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Delimiter().is_Some()) { + builder.delimiter( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Delimiter().dtor_value() + ) + ); + } + if (dafnyValue.dtor_EncodingType().is_Some()) { + builder.encodingType( + ToNative.EncodingType(dafnyValue.dtor_EncodingType().dtor_value()) + ); + } + if (dafnyValue.dtor_IsTruncated().is_Some()) { + builder.isTruncated((dafnyValue.dtor_IsTruncated().dtor_value())); + } + if (dafnyValue.dtor_KeyCount().is_Some()) { + builder.keyCount((dafnyValue.dtor_KeyCount().dtor_value())); + } + if (dafnyValue.dtor_MaxKeys().is_Some()) { + builder.maxKeys((dafnyValue.dtor_MaxKeys().dtor_value())); + } + if (dafnyValue.dtor_Name().is_Some()) { + builder.name( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Name().dtor_value() + ) + ); + } + if (dafnyValue.dtor_NextContinuationToken().is_Some()) { + builder.nextContinuationToken( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_NextContinuationToken().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Prefix().is_Some()) { + builder.prefix( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Prefix().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestCharged().is_Some()) { + builder.requestCharged( + ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) + ); + } + if (dafnyValue.dtor_StartAfter().is_Some()) { + builder.startAfter( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_StartAfter().dtor_value() + ) + ); + } + return builder.build(); + } + + public static ListObjectsV2Request ListObjectsV2Request( + software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request dafnyValue + ) { + ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder(); + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() + ) + ); + if (dafnyValue.dtor_ContinuationToken().is_Some()) { + builder.continuationToken( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContinuationToken().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Delimiter().is_Some()) { + builder.delimiter( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Delimiter().dtor_value() + ) + ); + } + if (dafnyValue.dtor_EncodingType().is_Some()) { + builder.encodingType( + ToNative.EncodingType(dafnyValue.dtor_EncodingType().dtor_value()) + ); + } + if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { + builder.expectedBucketOwner( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ExpectedBucketOwner().dtor_value() + ) + ); + } + if (dafnyValue.dtor_FetchOwner().is_Some()) { + builder.fetchOwner((dafnyValue.dtor_FetchOwner().dtor_value())); + } + if (dafnyValue.dtor_MaxKeys().is_Some()) { + builder.maxKeys((dafnyValue.dtor_MaxKeys().dtor_value())); + } + if (dafnyValue.dtor_OptionalObjectAttributes().is_Some()) { + builder.optionalObjectAttributes( + ToNative.OptionalObjectAttributesList( + dafnyValue.dtor_OptionalObjectAttributes().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Prefix().is_Some()) { + builder.prefix( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Prefix().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestPayer().is_Some()) { + builder.requestPayer( + ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) + ); + } + if (dafnyValue.dtor_StartAfter().is_Some()) { + builder.startAfter( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_StartAfter().dtor_value() + ) + ); + } + return builder.build(); + } + + public static Map Metadata( + DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToMap( + dafnyValue, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String + ); + } + + public static S3Object S3Object( + software.amazon.cryptography.services.s3.internaldafny.types.S3Object dafnyValue + ) { + S3Object.Builder builder = S3Object.builder(); + if (dafnyValue.dtor_ChecksumAlgorithm().is_Some()) { + builder.checksumAlgorithm( + ToNative.ChecksumAlgorithmList( + dafnyValue.dtor_ChecksumAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ETag().is_Some()) { + builder.eTag( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ETag().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Key().is_Some()) { + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key().dtor_value() + ) + ); + } + if (dafnyValue.dtor_LastModified().is_Some()) { + builder.lastModified( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_LastModified().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Owner().is_Some()) { + builder.owner(ToNative.Owner(dafnyValue.dtor_Owner().dtor_value())); + } + if (dafnyValue.dtor_RestoreStatus().is_Some()) { + builder.restoreStatus( + ToNative.RestoreStatus(dafnyValue.dtor_RestoreStatus().dtor_value()) + ); + } + if (dafnyValue.dtor_Size().is_Some()) { + builder.size((dafnyValue.dtor_Size().dtor_value())); + } + if (dafnyValue.dtor_StorageClass().is_Some()) { + builder.storageClass( + ToNative.ObjectStorageClass(dafnyValue.dtor_StorageClass().dtor_value()) + ); + } + return builder.build(); + } + + public static ObjectCannedACL ObjectCannedACL( + software.amazon.cryptography.services.s3.internaldafny.types.ObjectCannedACL dafnyValue + ) { + if (dafnyValue.is_private()) { + return ObjectCannedACL.PRIVATE; + } + if (dafnyValue.is_public__read()) { + return ObjectCannedACL.PUBLIC_READ; + } + if (dafnyValue.is_public__read__write()) { + return ObjectCannedACL.PUBLIC_READ_WRITE; + } + if (dafnyValue.is_authenticated__read()) { + return ObjectCannedACL.AUTHENTICATED_READ; + } + if (dafnyValue.is_aws__exec__read()) { + return ObjectCannedACL.AWS_EXEC_READ; + } + if (dafnyValue.is_bucket__owner__read()) { + return ObjectCannedACL.BUCKET_OWNER_READ; + } + if (dafnyValue.is_bucket__owner__full__control()) { + return ObjectCannedACL.BUCKET_OWNER_FULL_CONTROL; + } + return ObjectCannedACL.fromValue(dafnyValue.toString()); + } + + public static ObjectIdentifier ObjectIdentifier( + software.amazon.cryptography.services.s3.internaldafny.types.ObjectIdentifier dafnyValue + ) { + ObjectIdentifier.Builder builder = ObjectIdentifier.builder(); + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key() + ) + ); + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static List ObjectIdentifierList( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.ObjectIdentifier + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::ObjectIdentifier + ); + } + + public static List ObjectList( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.S3Object + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::S3Object + ); + } + + public static ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus( + software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus dafnyValue + ) { + if (dafnyValue.is_ON()) { + return ObjectLockLegalHoldStatus.ON; + } + if (dafnyValue.is_OFF()) { + return ObjectLockLegalHoldStatus.OFF; + } + return ObjectLockLegalHoldStatus.fromValue(dafnyValue.toString()); + } + + public static ObjectLockMode ObjectLockMode( + software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockMode dafnyValue + ) { + if (dafnyValue.is_GOVERNANCE()) { + return ObjectLockMode.GOVERNANCE; + } + if (dafnyValue.is_COMPLIANCE()) { + return ObjectLockMode.COMPLIANCE; + } + return ObjectLockMode.fromValue(dafnyValue.toString()); + } + + public static ObjectStorageClass ObjectStorageClass( + software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass dafnyValue + ) { + if (dafnyValue.is_STANDARD()) { + return ObjectStorageClass.STANDARD; + } + if (dafnyValue.is_REDUCED__REDUNDANCY()) { + return ObjectStorageClass.REDUCED_REDUNDANCY; + } + if (dafnyValue.is_GLACIER()) { + return ObjectStorageClass.GLACIER; + } + if (dafnyValue.is_STANDARD__IA()) { + return ObjectStorageClass.STANDARD_IA; + } + if (dafnyValue.is_ONEZONE__IA()) { + return ObjectStorageClass.ONEZONE_IA; + } + if (dafnyValue.is_INTELLIGENT__TIERING()) { + return ObjectStorageClass.INTELLIGENT_TIERING; + } + if (dafnyValue.is_DEEP__ARCHIVE()) { + return ObjectStorageClass.DEEP_ARCHIVE; + } + if (dafnyValue.is_OUTPOSTS()) { + return ObjectStorageClass.OUTPOSTS; + } + if (dafnyValue.is_GLACIER__IR()) { + return ObjectStorageClass.GLACIER_IR; + } + if (dafnyValue.is_SNOW()) { + return ObjectStorageClass.SNOW; + } + if (dafnyValue.is_EXPRESS__ONEZONE()) { + return ObjectStorageClass.EXPRESS_ONEZONE; + } + return ObjectStorageClass.fromValue(dafnyValue.toString()); + } + + public static OptionalObjectAttributes OptionalObjectAttributes( + software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes dafnyValue + ) { + if (dafnyValue.is_RESTORE__STATUS()) { + return OptionalObjectAttributes.RESTORE_STATUS; + } + return OptionalObjectAttributes.fromValue(dafnyValue.toString()); + } + + public static List OptionalObjectAttributesList( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes + > dafnyValue + ) { + return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( + dafnyValue, + software.amazon.cryptography.services.s3.internaldafny.ToNative::OptionalObjectAttributes + ); + } + + public static Owner Owner( + software.amazon.cryptography.services.s3.internaldafny.types.Owner dafnyValue + ) { + Owner.Builder builder = Owner.builder(); + if (dafnyValue.dtor_DisplayName().is_Some()) { + builder.displayName( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_DisplayName().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ID().is_Some()) { + builder.id( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ID().dtor_value() + ) + ); + } + return builder.build(); + } + + public static PutObjectResponse PutObjectOutput(PutObjectOutput dafnyValue) { + PutObjectResponse.Builder builder = PutObjectResponse.builder(); + if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { + builder.bucketKeyEnabled( + (dafnyValue.dtor_BucketKeyEnabled().dtor_value()) + ); + } + if (dafnyValue.dtor_ChecksumCRC32().is_Some()) { + builder.checksumCRC32( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumCRC32C().is_Some()) { + builder.checksumCRC32C( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32C().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA1().is_Some()) { + builder.checksumSHA1( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA1().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA256().is_Some()) { + builder.checksumSHA256( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA256().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ETag().is_Some()) { + builder.eTag( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ETag().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Expiration().is_Some()) { + builder.expiration( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Expiration().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestCharged().is_Some()) { + builder.requestCharged( + ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) + ); + } + if (dafnyValue.dtor_ServerSideEncryption().is_Some()) { + builder.serverSideEncryption( + ToNative.ServerSideEncryption( + dafnyValue.dtor_ServerSideEncryption().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerAlgorithm().is_Some()) { + builder.sseCustomerAlgorithm( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKeyMD5().is_Some()) { + builder.sseCustomerKeyMD5( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKeyMD5().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { + builder.sseKMSEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { + builder.sseKMSKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); + } + if (dafnyValue.dtor_VersionId().is_Some()) { + builder.versionId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_VersionId().dtor_value() + ) + ); + } + return builder.build(); + } + + public static PutObjectRequest PutObjectRequest( + software.amazon.cryptography.services.s3.internaldafny.types.PutObjectRequest dafnyValue + ) { + PutObjectRequest.Builder builder = PutObjectRequest.builder(); + if (dafnyValue.dtor_ACL().is_Some()) { + builder.acl(ToNative.ObjectCannedACL(dafnyValue.dtor_ACL().dtor_value())); + } +// if (dafnyValue.dtor_Body().is_Some()) { +// builder.body( +// SdkBytes.fromByteArray( +// (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) +// ) +// ); +// } + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() + ) + ); + if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { + builder.bucketKeyEnabled( + (dafnyValue.dtor_BucketKeyEnabled().dtor_value()) + ); + } + if (dafnyValue.dtor_CacheControl().is_Some()) { + builder.cacheControl( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_CacheControl().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumAlgorithm().is_Some()) { + builder.checksumAlgorithm( + ToNative.ChecksumAlgorithm( + dafnyValue.dtor_ChecksumAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumCRC32().is_Some()) { + builder.checksumCRC32( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumCRC32C().is_Some()) { + builder.checksumCRC32C( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumCRC32C().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA1().is_Some()) { + builder.checksumSHA1( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA1().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ChecksumSHA256().is_Some()) { + builder.checksumSHA256( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ChecksumSHA256().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentDisposition().is_Some()) { + builder.contentDisposition( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentDisposition().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentEncoding().is_Some()) { + builder.contentEncoding( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentEncoding().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentLanguage().is_Some()) { + builder.contentLanguage( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentLanguage().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentLength().is_Some()) { + builder.contentLength((dafnyValue.dtor_ContentLength().dtor_value())); + } + if (dafnyValue.dtor_ContentMD5().is_Some()) { + builder.contentMD5( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentMD5().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ContentType().is_Some()) { + builder.contentType( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ContentType().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { + builder.expectedBucketOwner( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_ExpectedBucketOwner().dtor_value() + ) + ); + } + if (dafnyValue.dtor_Expires().is_Some()) { + builder.expires( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_Expires().dtor_value() + ) + ); + } + if (dafnyValue.dtor_GrantFullControl().is_Some()) { + builder.grantFullControl( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_GrantFullControl().dtor_value() + ) + ); + } + if (dafnyValue.dtor_GrantRead().is_Some()) { + builder.grantRead( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_GrantRead().dtor_value() + ) + ); + } + if (dafnyValue.dtor_GrantReadACP().is_Some()) { + builder.grantReadACP( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_GrantReadACP().dtor_value() + ) + ); + } + if (dafnyValue.dtor_GrantWriteACP().is_Some()) { + builder.grantWriteACP( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_GrantWriteACP().dtor_value() + ) + ); + } + builder.key( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Key() + ) + ); + if (dafnyValue.dtor_Metadata().is_Some()) { + builder.metadata( + ToNative.Metadata(dafnyValue.dtor_Metadata().dtor_value()) + ); + } + if (dafnyValue.dtor_ObjectLockLegalHoldStatus().is_Some()) { + builder.objectLockLegalHoldStatus( + ToNative.ObjectLockLegalHoldStatus( + dafnyValue.dtor_ObjectLockLegalHoldStatus().dtor_value() + ) + ); + } + if (dafnyValue.dtor_ObjectLockMode().is_Some()) { + builder.objectLockMode( + ToNative.ObjectLockMode(dafnyValue.dtor_ObjectLockMode().dtor_value()) + ); + } + if (dafnyValue.dtor_ObjectLockRetainUntilDate().is_Some()) { + builder.objectLockRetainUntilDate( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_ObjectLockRetainUntilDate().dtor_value() + ) + ); + } + if (dafnyValue.dtor_RequestPayer().is_Some()) { + builder.requestPayer( + ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) + ); + } + if (dafnyValue.dtor_ServerSideEncryption().is_Some()) { + builder.serverSideEncryption( + ToNative.ServerSideEncryption( + dafnyValue.dtor_ServerSideEncryption().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerAlgorithm().is_Some()) { + builder.sseCustomerAlgorithm( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerAlgorithm().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKey().is_Some()) { + builder.sseCustomerKey( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKey().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSECustomerKeyMD5().is_Some()) { + builder.sseCustomerKeyMD5( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSECustomerKeyMD5().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { + builder.sseKMSEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { + builder.sseKMSKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); + } + if (dafnyValue.dtor_StorageClass().is_Some()) { + builder.storageClass( + ToNative.StorageClass(dafnyValue.dtor_StorageClass().dtor_value()) + ); + } + if (dafnyValue.dtor_Tagging().is_Some()) { + builder.tagging( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Tagging().dtor_value() + ) + ); + } + if (dafnyValue.dtor_WebsiteRedirectLocation().is_Some()) { + builder.websiteRedirectLocation( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_WebsiteRedirectLocation().dtor_value() + ) + ); + } + return builder.build(); + } + + public static ReplicationStatus ReplicationStatus( + software.amazon.cryptography.services.s3.internaldafny.types.ReplicationStatus dafnyValue + ) { + if (dafnyValue.is_COMPLETE()) { + return ReplicationStatus.COMPLETE; + } + if (dafnyValue.is_PENDING()) { + return ReplicationStatus.PENDING; + } + if (dafnyValue.is_FAILED()) { + return ReplicationStatus.FAILED; + } + if (dafnyValue.is_REPLICA()) { + return ReplicationStatus.REPLICA; + } + if (dafnyValue.is_COMPLETED()) { + return ReplicationStatus.COMPLETED; + } + return ReplicationStatus.fromValue(dafnyValue.toString()); + } + + public static RequestCharged RequestCharged( + software.amazon.cryptography.services.s3.internaldafny.types.RequestCharged dafnyValue + ) { + if (dafnyValue.is_requester()) { + return RequestCharged.REQUESTER; + } + return RequestCharged.fromValue(dafnyValue.toString()); + } + + public static RequestPayer RequestPayer( + software.amazon.cryptography.services.s3.internaldafny.types.RequestPayer dafnyValue + ) { + if (dafnyValue.is_requester()) { + return RequestPayer.REQUESTER; + } + return RequestPayer.fromValue(dafnyValue.toString()); + } + + public static RestoreStatus RestoreStatus( + software.amazon.cryptography.services.s3.internaldafny.types.RestoreStatus dafnyValue + ) { + RestoreStatus.Builder builder = RestoreStatus.builder(); + if (dafnyValue.dtor_IsRestoreInProgress().is_Some()) { + builder.isRestoreInProgress( + (dafnyValue.dtor_IsRestoreInProgress().dtor_value()) + ); + } + if (dafnyValue.dtor_RestoreExpiryDate().is_Some()) { + builder.restoreExpiryDate( + software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( + dafnyValue.dtor_RestoreExpiryDate().dtor_value() + ) + ); + } + return builder.build(); + } + + public static ServerSideEncryption ServerSideEncryption( + software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption dafnyValue + ) { + if (dafnyValue.is_AES256()) { + return ServerSideEncryption.AES256; + } + if (dafnyValue.is_aws__kms()) { + return ServerSideEncryption.AWS_KMS; + } + if (dafnyValue.is_aws__kms__dsse()) { + return ServerSideEncryption.AWS_KMS_DSSE; + } + return ServerSideEncryption.fromValue(dafnyValue.toString()); + } + + public static StorageClass StorageClass( + software.amazon.cryptography.services.s3.internaldafny.types.StorageClass dafnyValue + ) { + if (dafnyValue.is_STANDARD()) { + return StorageClass.STANDARD; + } + if (dafnyValue.is_REDUCED__REDUNDANCY()) { + return StorageClass.REDUCED_REDUNDANCY; + } + if (dafnyValue.is_STANDARD__IA()) { + return StorageClass.STANDARD_IA; + } + if (dafnyValue.is_ONEZONE__IA()) { + return StorageClass.ONEZONE_IA; + } + if (dafnyValue.is_INTELLIGENT__TIERING()) { + return StorageClass.INTELLIGENT_TIERING; + } + if (dafnyValue.is_GLACIER()) { + return StorageClass.GLACIER; + } + if (dafnyValue.is_DEEP__ARCHIVE()) { + return StorageClass.DEEP_ARCHIVE; + } + if (dafnyValue.is_OUTPOSTS()) { + return StorageClass.OUTPOSTS; + } + if (dafnyValue.is_GLACIER__IR()) { + return StorageClass.GLACIER_IR; + } + if (dafnyValue.is_SNOW()) { + return StorageClass.SNOW; + } + if (dafnyValue.is_EXPRESS__ONEZONE()) { + return StorageClass.EXPRESS_ONEZONE; + } + return StorageClass.fromValue(dafnyValue.toString()); + } + + public static InvalidObjectStateException Error( + Error_InvalidObjectState dafnyValue + ) { + InvalidObjectStateException.Builder builder = + InvalidObjectStateException.builder(); + if (dafnyValue.dtor_AccessTier().is_Some()) { + builder.accessTier( + ToNative.IntelligentTieringAccessTier( + dafnyValue.dtor_AccessTier().dtor_value() + ) + ); + } + if (dafnyValue.dtor_message().is_Some()) { + builder.message( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_message().dtor_value() + ) + ); + } + if (dafnyValue.dtor_StorageClass().is_Some()) { + builder.storageClass( + ToNative.StorageClass(dafnyValue.dtor_StorageClass().dtor_value()) + ); + } + return builder.build(); + } + + public static NoSuchBucketException Error(Error_NoSuchBucket dafnyValue) { + NoSuchBucketException.Builder builder = NoSuchBucketException.builder(); + if (dafnyValue.dtor_message().is_Some()) { + builder.message( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_message().dtor_value() + ) + ); + } + return builder.build(); + } + + public static NoSuchKeyException Error(Error_NoSuchKey dafnyValue) { + NoSuchKeyException.Builder builder = NoSuchKeyException.builder(); + if (dafnyValue.dtor_message().is_Some()) { + builder.message( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_message().dtor_value() + ) + ); + } + return builder.build(); + } + + public static S3Client AmazonS3(IS3Client dafnyValue) { + return ((Shim) dafnyValue).impl(); + } + + public static RuntimeException Error(Error_Opaque dafnyValue) { + // While the first two cases are logically identical, + // there is a semantic distinction. + // An un-modeled Service Error is different from a Java Heap Exhaustion error. + // In the future, Smithy-Dafny MAY allow for this distinction. + // Which would allow Dafny developers to treat the two differently. + if (dafnyValue.dtor_obj() instanceof S3Exception) { + return (S3Exception) dafnyValue.dtor_obj(); + } else if (dafnyValue.dtor_obj() instanceof RuntimeException) { + return (RuntimeException) dafnyValue.dtor_obj(); + } else if (dafnyValue.dtor_obj() instanceof Throwable) { + return new RuntimeException( + String.format( + "Unknown error thrown while calling Amazon Simple Storage Service. %s", + (Throwable) dafnyValue.dtor_obj() + ) + ); + } + return new IllegalStateException( + String.format( + "Unknown error thrown while calling Amazon Simple Storage Service. %s", + dafnyValue + ) + ); + } + + public static RuntimeException Error(Error_OpaqueWithText dafnyValue) { + // While the first two cases are logically identical, + // there is a semantic distinction. + // An un-modeled Service Error is different from a Java Heap Exhaustion error. + // In the future, Smithy-Dafny MAY allow for this distinction. + // Which would allow Dafny developers to treat the two differently. + if (dafnyValue.dtor_obj() instanceof S3Exception) { + return (S3Exception) dafnyValue.dtor_obj(); + } else if (dafnyValue.dtor_obj() instanceof RuntimeException) { + return (RuntimeException) dafnyValue.dtor_obj(); + } else if (dafnyValue.dtor_obj() instanceof Throwable) { + return new RuntimeException( + String.format( + "Unknown error thrown while calling Amazon Simple Storage Service. %s", + (Throwable) dafnyValue.dtor_obj() + ) + ); + } + return new IllegalStateException( + String.format( + "Unknown error thrown while calling Amazon Simple Storage Service. %s", + dafnyValue + ) + ); + } + + public static RuntimeException Error(Error dafnyValue) { + if (dafnyValue.is_InvalidObjectState()) { + return ToNative.Error((Error_InvalidObjectState) dafnyValue); + } + if (dafnyValue.is_NoSuchBucket()) { + return ToNative.Error((Error_NoSuchBucket) dafnyValue); + } + if (dafnyValue.is_NoSuchKey()) { + return ToNative.Error((Error_NoSuchKey) dafnyValue); + } + if (dafnyValue.is_Opaque()) { + return ToNative.Error((Error_Opaque) dafnyValue); + } + if (dafnyValue.is_OpaqueWithText()) { + return ToNative.Error((Error_OpaqueWithText) dafnyValue); + } + // TODO This should indicate a codegen bug; every error Should have been taken care of. + return new IllegalStateException( + String.format( + "Unknown error thrown while calling service. %s", + dafnyValue + ) + ); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java index 1aeec3c05..919911953 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java @@ -14,7 +14,7 @@ public class ConsumerAsSubscriber implements Subscriber { - private final IConsumer> consumer + private final IConsumer> consumer; private final CompletableFuture future; private Subscription subscription; diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java index 8b09a99f3..e25e7d99c 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java @@ -4,17 +4,25 @@ import Std_Compile.BulkActions_Compile.Batched; import Std_Compile.Producers_Compile.Producer; import Std_Compile.Wrappers_Compile.Option; +import dafny.TypeDescriptor; +import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.util.function.Function; -public class InputStreamAsDataStream implements DataStream { +public class InputStreamAsDataStream implements DataStream { + + private final TypeDescriptor e_td; private final InputStream inputStream; + private final Function ioExceptionWrapper; private boolean read = false; - public InputStreamAsDataStream(InputStream inputStream) { + public InputStreamAsDataStream(TypeDescriptor e_td, InputStream inputStream, Function ioExceptionWrapper) { + this.e_td = e_td; this.inputStream = inputStream; + this.ioExceptionWrapper = ioExceptionWrapper; } @Override @@ -28,12 +36,12 @@ public boolean Replayable() { } @Override - public Producer> Reader() { + public Producer> Reader() { if (read) { throw new IllegalStateException("Already read"); } read = true; - return new InputStreamAsProducer(inputStream); + return new InputStreamAsProducer(e_td, inputStream, ioExceptionWrapper); } } diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java index 8e3041b97..8ffd1aa41 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java @@ -17,20 +17,25 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.util.function.Function; -public class InputStreamAsProducer implements Producer> { +public class InputStreamAsProducer implements Producer> { private final InputStream inputStream; private long totalRead; + private Function ioExceptionWrapper; private static final TypeDescriptor T_TD = TypeDescriptor.BYTE; - private static final TypeDescriptor E_TD = TypeDescriptor.reference(Exception.class); - private static final TypeDescriptor> BATCHED_TD = - Batched._typeDescriptor(T_TD, E_TD); + private final TypeDescriptor E_TD; + private final TypeDescriptor> BATCHED_TD; - public InputStreamAsProducer(InputStream inputStream) { + public InputStreamAsProducer(TypeDescriptor e_td, InputStream inputStream, Function ioExceptionWrapper) { this.inputStream = inputStream; this.totalRead = 0; + this.ioExceptionWrapper = ioExceptionWrapper; + + this.E_TD = e_td; + this.BATCHED_TD = Batched._typeDescriptor(T_TD, E_TD); } @Override @@ -44,7 +49,7 @@ public Option Remaining() { } @Override - public Option> Next() { + public Option> Next() { try { int value = inputStream.read(); if (value == -1) { @@ -55,19 +60,19 @@ public Option> Next() { return Option.create_Some(BATCHED_TD, batched); } } catch (IOException e) { - return Option.create_Some(null, Batched.create_BatchError(T_TD, E_TD, e)); + return Option.create_Some(null, Batched.create_BatchError(T_TD, E_TD, ioExceptionWrapper.apply(e))); } } @Override - public void ForEach(IConsumer> consumer) { + public void ForEach(IConsumer> consumer) { __default.DefaultForEach(BATCHED_TD, this, consumer); } @Override - public Option> Fill(Consumer> consumer) { + public Option> Fill(Consumer> consumer) { if (consumer instanceof BatchArrayWriter) { - BatchArrayWriter writer = (BatchArrayWriter) consumer; + BatchArrayWriter writer = (BatchArrayWriter) consumer; int n = writer.Capacity().dtor_value().intValueExact(); byte[] buffer = new byte[n]; try { @@ -77,12 +82,12 @@ public Option> Fill(Consumer> } else { totalRead += count; Array dafnyArray = Array.wrap(buffer); - BatchReader reader = new BatchReader(T_TD, E_TD); + BatchReader reader = new BatchReader(T_TD, E_TD); reader.__ctor(DafnySequence.fromArrayRange(T_TD, dafnyArray, 0, count)); reader.Fill(consumer); } } catch (IOException e) { - writer.Accept(Batched.create_BatchError(T_TD, E_TD, e)); + writer.Accept(Batched.create_BatchError(T_TD, E_TD, ioExceptionWrapper.apply(e))); } return Option.create_None(BATCHED_TD); } @@ -91,7 +96,7 @@ public Option> Fill(Consumer> } @Override - public Option> Invoke(Tuple0 tuple0) { + public Option> Invoke(Tuple0 tuple0) { return _Companion_Producer.Next(null, this); } } From 3a896e14b6b88258b230ac3585692b7f4c58bd4b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 12:50:40 -0700 Subject: [PATCH 32/72] More fixing/hacking --- .../services/s3/internaldafny/Shim.java | 2 +- .../services/s3/internaldafny/ToDafny.java | 97 ++++++++++--------- .../services/s3/internaldafny/ToNative.java | 58 +++++------ .../aws-sdks/s3/test/TestComAmazonawsS3.dfy | 5 +- .../Streams/DataStreamAsAsyncRequestBody.java | 4 +- .../java/Streams/InputStreamAsDataStream.java | 2 +- .../java/Streams/RequestBodyAsDataStream.java | 47 +++++++++ .../smithydafny/DafnyApiCodegen.java | 6 +- .../smithydafny/DafnyNameResolver.java | 3 + 9 files changed, 141 insertions(+), 83 deletions(-) create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java index b7e9b4057..03388026e 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java @@ -191,7 +191,7 @@ public Result ListObjectsV2( public Result PutObject(PutObjectRequest input) { software.amazon.awssdk.services.s3.model.PutObjectRequest converted = ToNative.PutObjectRequest(input); - DataStream dataStream = input._Body.dtor_value(); + DataStream dataStream = input._Body.dtor_value(); ContentStreamProvider provider = () -> { Producer reader = dataStream.Reader(); return new ProviderAsInputStream(reader); diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index 6ff475c8f..1943bb68c 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -22,6 +22,7 @@ import java.util.Objects; import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; @@ -471,11 +472,11 @@ public static ErrorShape ErrorShape( public static GetObjectOutput GetObjectOutput(ResponseInputStream responseInputStream) { GetObjectResponse nativeValue = responseInputStream.response(); - Option> body; + Option> body; body = Option.create_Some( - TypeDescriptor.reference(DataStream.class), - new InputStreamAsDataStream(responseInputStream) + null, + new InputStreamAsDataStream(Error._typeDescriptor(), responseInputStream, e -> Error.create_Opaque(e)) ); Option deleteMarker; deleteMarker = @@ -767,14 +768,15 @@ public static GetObjectOutput GetObjectOutput(ResponseInputStream> sSEKMSKeyId; sSEKMSKeyId = - Objects.nonNull(nativeValue.sseKMSKeyId()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.sseKMSKeyId() - ) - ) - : Option.create_None( +// Objects.nonNull(nativeValue.sseKMSKeyId()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +// nativeValue.sseKMSKeyId() +// ) +// ) +// : + Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; @@ -1724,26 +1726,28 @@ public static PutObjectOutput PutObjectOutput(PutObjectResponse nativeValue) { ); Option> sSEKMSKeyId; sSEKMSKeyId = - Objects.nonNull(nativeValue.sseKMSKeyId()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.sseKMSKeyId() - ) - ) - : Option.create_None( +// Objects.nonNull(nativeValue.sseKMSKeyId()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +// nativeValue.sseKMSKeyId() +// ) +// ) +// : + Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option> sSEKMSEncryptionContext; sSEKMSEncryptionContext = - Objects.nonNull(nativeValue.sseKMSEncryptionContext()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.sseKMSEncryptionContext() - ) - ) - : Option.create_None( +// Objects.nonNull(nativeValue.sseKMSEncryptionContext()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +// nativeValue.sseKMSEncryptionContext() +// ) +// ) +// : + Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; @@ -1781,7 +1785,8 @@ public static PutObjectOutput PutObjectOutput(PutObjectResponse nativeValue) { } public static PutObjectRequest PutObjectRequest( - software.amazon.awssdk.services.s3.model.PutObjectRequest nativeValue + software.amazon.awssdk.services.s3.model.PutObjectRequest nativeValue, + RequestBody nativeBody ) { Option aCL; aCL = @@ -1791,7 +1796,7 @@ public static PutObjectRequest PutObjectRequest( ToDafny.ObjectCannedACL(nativeValue.acl()) ) : Option.create_None(ObjectCannedACL._typeDescriptor()); - Option> body; + Option> body = new RequestBodyAsDataStream(); // body = // Objects.nonNull(nativeValue.body()) // ? Option.create_Some( @@ -2094,26 +2099,28 @@ public static PutObjectRequest PutObjectRequest( ); Option> sSEKMSKeyId; sSEKMSKeyId = - Objects.nonNull(nativeValue.sseKMSKeyId()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.sseKMSKeyId() - ) - ) - : Option.create_None( +// Objects.nonNull(nativeValue.sseKMSKeyId()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +// nativeValue.sseKMSKeyId() +// ) +// ) +// : + Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option> sSEKMSEncryptionContext; sSEKMSEncryptionContext = - Objects.nonNull(nativeValue.sseKMSEncryptionContext()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.sseKMSEncryptionContext() - ) - ) - : Option.create_None( +// Objects.nonNull(nativeValue.sseKMSEncryptionContext()) +// ? Option.create_Some( +// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), +// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +// nativeValue.sseKMSEncryptionContext() +// ) +// ) +// : + Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java index a7a56f88f..0a8b464cd 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java @@ -567,11 +567,11 @@ public static GetObjectResponse GetObjectOutput(GetObjectOutput dafnyValue) { ); } if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { - builder.sseKMSKeyId( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_SSEKMSKeyId().dtor_value() - ) - ); +// builder.sseKMSKeyId( +// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +// dafnyValue.dtor_SSEKMSKeyId().dtor_value() +// ) +// ); } if (dafnyValue.dtor_StorageClass().is_Some()) { builder.storageClass( @@ -1212,18 +1212,18 @@ public static PutObjectResponse PutObjectOutput(PutObjectOutput dafnyValue) { ); } if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { - builder.sseKMSEncryptionContext( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() - ) - ); +// builder.sseKMSEncryptionContext( +// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +// dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() +// ) +// ); } if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { - builder.sseKMSKeyId( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_SSEKMSKeyId().dtor_value() - ) - ); +// builder.sseKMSKeyId( +// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +// dafnyValue.dtor_SSEKMSKeyId().dtor_value() +// ) +// ); } if (dafnyValue.dtor_VersionId().is_Some()) { builder.versionId( @@ -1443,20 +1443,20 @@ public static PutObjectRequest PutObjectRequest( ) ); } - if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { - builder.sseKMSEncryptionContext( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() - ) - ); - } - if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { - builder.sseKMSKeyId( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_SSEKMSKeyId().dtor_value() - ) - ); - } +// if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { +// builder.sseKMSEncryptionContext( +// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +// dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() +// ) +// ); +// } +// if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { +// builder.sseKMSKeyId( +// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +// dafnyValue.dtor_SSEKMSKeyId().dtor_value() +// ) +// ); +// } if (dafnyValue.dtor_StorageClass().is_Some()) { builder.storageClass( ToNative.StorageClass(dafnyValue.dtor_StorageClass().dtor_value()) diff --git a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy index 5bf0466c9..a5bc22634 100644 --- a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy +++ b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy @@ -57,7 +57,7 @@ module TestComAmazonawsS3 { method GetObjectTest( nameonly input: S3.Types.GetObjectRequest, - nameonly expectedBody: BoundedInts.bytes + nameonly expectedBody: bytes ) { var client :- expect S3.S3Client(); @@ -113,8 +113,7 @@ module TestComAmazonawsS3 { expect(ret.Success?), ret.error; } - method Collect(e: DataStream) returns (s: Result) - + method Collect(e: DataStream) returns (s: Result) { var reader := e.Reader(); var a: BatchSeqWriter := new BatchSeqWriter(); diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java index 933c329a7..17a9d2c3b 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java @@ -11,9 +11,9 @@ public class DataStreamAsAsyncRequestBody implements AsyncRequestBody { - private final DataStream dataStream; + private final DataStream dataStream; - public DataStreamAsAsyncRequestBody(DataStream dataStream) { + public DataStreamAsAsyncRequestBody(DataStream dataStream) { this.dataStream = dataStream; } diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java index e25e7d99c..84aa4cbf6 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java @@ -11,7 +11,7 @@ import java.math.BigInteger; import java.util.function.Function; -public class InputStreamAsDataStream implements DataStream { +public class InputStreamAsDataStream implements DataStream { private final TypeDescriptor e_td; diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java new file mode 100644 index 000000000..4ac82c90f --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java @@ -0,0 +1,47 @@ +package Streams; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Std_Compile.BulkActions_Compile.Batched; +import Std_Compile.Producers_Compile.Producer; +import Std_Compile.Wrappers_Compile.Option; +import dafny.TypeDescriptor; +import software.amazon.awssdk.core.sync.RequestBody; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.util.Optional; +import java.util.function.Function; + +public class RequestBodyAsDataStream implements DataStream { + + private final TypeDescriptor e_td; + private final Function ioExceptionWrapper; + private final RequestBody requestBody; + + public RequestBodyAsDataStream(TypeDescriptor e_td, RequestBody requestBody, Function ioExceptionWrapper) { + this.e_td = e_td; + this.ioExceptionWrapper = ioExceptionWrapper; + this.requestBody = requestBody; + } + + @Override + public Option ContentLength() { + if (requestBody.optionalContentLength().isPresent()) { + return Option.create_Some(TypeDescriptor.BIG_INTEGER, BigInteger.valueOf(requestBody.optionalContentLength().get())); + } else { + return Option.create_None(TypeDescriptor.BIG_INTEGER); + } + } + + @Override + public boolean Replayable() { + return true; + } + + @Override + public Producer> Reader() { + InputStream is = requestBody.contentStreamProvider().newStream(); + return new InputStreamAsProducer<>(e_td, is, ioExceptionWrapper); + } +} diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index fda0789a4..bcae6df3f 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -454,7 +454,8 @@ public TokenTree generateStructureTypeDefinition( StructureShape.class ); - final String typeName = structureShapeId.getName(); + final String typeName = structureShapeId.getName().equals("Object") ? + "S3Object" : structureShapeId.getName(); return TokenTree.of( Token.of("datatype %1$s =".formatted(typeName)), generateDataTypeConstructorFromStructure(structureShapeId) @@ -1951,7 +1952,8 @@ public TokenTree generateDataTypeConstructorFromStructure( shapeId, StructureShape.class ); - final String typeName = shapeId.getName(); + final String typeName = shapeId.getName().equals("Object") ? + "S3Object" : shapeId.getName(); final TokenTree params = TokenTree .of( diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index 82c8b3d8c..d5e1d3d43 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -139,6 +139,9 @@ private String baseTypeForMember(final MemberShape memberShape) { } private String dafnyTypeNameShape(final Shape shape) { + if (shape.getId().getName().equals("Object")) { + return dafnyModulePrefixForShape(shape) + "S3Object"; + } return dafnyModulePrefixForShape(shape) + shape.getId().getName(); } From 8d81c61287431d47b5641be6442bddc0f648d9db Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 12:54:54 -0700 Subject: [PATCH 33/72] Java builds --- .../cryptography/services/s3/internaldafny/ToDafny.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index 1943bb68c..bdb418e14 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -5,6 +5,7 @@ import StandardLibrary_Compile.Streams_Compile.DataStream; import Streams.InputStreamAsDataStream; +import Streams.RequestBodyAsDataStream; import Wrappers_Compile.Option; import dafny.DafnyMap; import dafny.DafnySequence; @@ -1796,7 +1797,10 @@ public static PutObjectRequest PutObjectRequest( ToDafny.ObjectCannedACL(nativeValue.acl()) ) : Option.create_None(ObjectCannedACL._typeDescriptor()); - Option> body = new RequestBodyAsDataStream(); + Option> body = Option.create_Some( + null, + new RequestBodyAsDataStream( + null, nativeBody, Error::create_Opaque )); // body = // Objects.nonNull(nativeValue.body()) // ? Option.create_Some( From 82af16f3aa90cf2bf94435fe7e80251132370be0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 13:40:32 -0700 Subject: [PATCH 34/72] Typos in python --- .../internaldafny/extern/streams.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index 2fa3ebc19..953d13782 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -27,7 +27,7 @@ def read(self, size: int = -1) -> bytes: else: writer = BatchArrayWriter() writer.ctor__(DafnyArray(None, size)) - self.reader.ForEachToCapacity(writer) + self.reader.Fill(writer) # TODO: Check for errors. Fine to ignore EOI though. # print(f"****** {writer.Values()} ******") return bytes(writer.Values()) @@ -49,12 +49,12 @@ def seek(self, offset, whence=0): if new_position > self.reader.ProducedCount(): consumer = IgnoreNConsumer() consumer.ctor__(new_position - self.reader.ProducedCount()) - self.reader.ForEachToCapacity(consumer) + self.reader.Fill(consumer) elif new_position < self.reader.ProducedCount(): self.reader = self.data_stream.Reader() consumer = IgnoreNConsumer() consumer.ctor__(new_position) - self.reader.ForEachToCapacity(consumer) + self.reader.Fill(consumer) class StreamingBlobAsDafnyDataStream(DataStream): @@ -105,11 +105,11 @@ def ForEach(self, consumer): consumer.Accept(Batched_EndOfInput()) - def ForEachToCapacity(self, consumer): + def Fill(self, consumer): # TODO: error handling size = consumer.Capacity() next = self.streaming_blob.read(size) batch = BatchReader() batch.ctor__(Seq(next)) - batch.ForEachToCapacity(consumer) + batch.Fill(consumer) # TODO: EOI From aea4258348ed5be79bb82f1118c5a231456842fc Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 13:59:19 -0700 Subject: [PATCH 35/72] Remove Object from S3 model for now --- .../aws-sdks/s3/Model/custom-model.json | 274 ------------- .../services/s3/internaldafny/Shim.java | 37 -- .../services/s3/internaldafny/ToDafny.java | 381 ------------------ .../services/s3/internaldafny/ToNative.java | 225 +---------- .../smithydafny/DafnyApiCodegen.java | 6 +- .../smithydafny/DafnyNameResolver.java | 3 - 6 files changed, 9 insertions(+), 917 deletions(-) diff --git a/TestModels/aws-sdks/s3/Model/custom-model.json b/TestModels/aws-sdks/s3/Model/custom-model.json index 3acbb908b..d0661daab 100644 --- a/TestModels/aws-sdks/s3/Model/custom-model.json +++ b/TestModels/aws-sdks/s3/Model/custom-model.json @@ -42,9 +42,6 @@ { "target": "com.amazonaws.s3#GetObject" }, - { - "target": "com.amazonaws.s3#ListObjectsV2" - }, { "target": "com.amazonaws.s3#PutObject" } @@ -819,214 +816,6 @@ "smithy.api#input": {} } }, - "com.amazonaws.s3#ListObjectsV2": { - "type": "operation", - "input": { - "target": "com.amazonaws.s3#ListObjectsV2Request" - }, - "output": { - "target": "com.amazonaws.s3#ListObjectsV2Output" - }, - "errors": [ - { - "target": "com.amazonaws.s3#NoSuchBucket" - } - ], - "traits": { - "smithy.api#documentation": "

Returns some or all (up to 1,000) of the objects in a bucket with each request. You can\n use the request parameters as selection criteria to return a subset of the objects in a\n bucket. A 200 OK response can contain valid or invalid XML. Make sure to\n design your application to parse the contents of the response and handle it appropriately.\n \n For more information about listing objects, see Listing object keys\n programmatically in the Amazon S3 User Guide. To get a list of your buckets, see ListBuckets.

\n \n

\n Directory buckets - For directory buckets, you must make requests for this API operation to the Zonal endpoint. These endpoints support virtual-hosted-style requests in the format https://bucket_name.s3express-az_id.region.amazonaws.com/key-name\n . Path-style requests are not supported. For more information, see Regional and Zonal endpoints in the\n Amazon S3 User Guide.

\n \n
\n
Permissions
\n
\n
    \n
  • \n

    \n General purpose bucket permissions - To use this operation, you must have READ access to the bucket. You must have permission to perform\n the s3:ListBucket action. The bucket owner has this permission by default and\n can grant this permission to others. For more information about permissions, see Permissions Related to Bucket Subresource Operations and Managing\n Access Permissions to Your Amazon S3 Resources in the\n Amazon S3 User Guide.

    \n
    • \n
  • \n

    \n Directory bucket permissions - To grant access to this API operation on a directory bucket, we recommend that you use the \n CreateSession\n API operation for session-based authorization. Specifically, you grant the s3express:CreateSession permission to the directory bucket in a bucket policy or an IAM identity-based policy. Then, you make the CreateSession API call on the bucket to obtain a session token. With the session token in your request header, you can make API requests to this operation. After the session token expires, you make another CreateSession API call to generate a new session token for use. \nAmazon Web Services CLI or SDKs create session and refresh the session token automatically to avoid service interruptions when a session expires. For more information about authorization, see \n CreateSession\n .

    \n
    • \n
\n
\n
Sorting order of returned objects
\n
\n
    \n
  • \n

    \n General purpose bucket - For general purpose buckets, ListObjectsV2 returns objects in lexicographical order based on their key names.

    \n
    • \n
  • \n

    \n Directory bucket - For directory buckets, ListObjectsV2 does not return objects in lexicographical order.

    \n
    • \n
\n
\n
HTTP Host header syntax
\n
\n

\n Directory buckets - The HTTP Host header syntax is \n Bucket_name.s3express-az_id.region.amazonaws.com.

\n
\n
\n \n

This section describes the latest revision of this action. We recommend that you use\n this revised API operation for application development. For backward compatibility, Amazon S3\n continues to support the prior version of this API operation, ListObjects.

\n \n

The following operations are related to ListObjectsV2:

\n ", - "smithy.api#http": { - "method": "GET", - "uri": "/{Bucket}?list-type=2", - "code": 200 - }, - "smithy.api#paginated": { - "inputToken": "ContinuationToken", - "outputToken": "NextContinuationToken", - "pageSize": "MaxKeys" - } - } - }, - "com.amazonaws.s3#ListObjectsV2Output": { - "type": "structure", - "members": { - "IsTruncated": { - "target": "com.amazonaws.s3#IsTruncated", - "traits": { - "smithy.api#documentation": "

Set to false if all of the results were returned. Set to true\n if more keys are available to return. If the number of results exceeds that specified by\n MaxKeys, all of the results might not be returned.

" - } - }, - "Contents": { - "target": "com.amazonaws.s3#ObjectList", - "traits": { - "smithy.api#documentation": "

Metadata about each object returned.

", - "smithy.api#xmlFlattened": {} - } - }, - "Name": { - "target": "com.amazonaws.s3#BucketName", - "traits": { - "smithy.api#documentation": "

The bucket name.

" - } - }, - "Prefix": { - "target": "com.amazonaws.s3#Prefix", - "traits": { - "smithy.api#documentation": "

Keys that begin with the indicated prefix.

\n \n

\n Directory buckets - For directory buckets, only prefixes that end in a delimiter (/) are supported.

\n " - } - }, - "Delimiter": { - "target": "com.amazonaws.s3#Delimiter", - "traits": { - "smithy.api#documentation": "

Causes keys that contain the same string between the prefix and the first\n occurrence of the delimiter to be rolled up into a single result element in the\n CommonPrefixes collection. These rolled-up keys are not returned elsewhere\n in the response. Each rolled-up result counts as only one return against the\n MaxKeys value.

\n \n

\n Directory buckets - For directory buckets, / is the only supported delimiter.

\n " - } - }, - "MaxKeys": { - "target": "com.amazonaws.s3#MaxKeys", - "traits": { - "smithy.api#documentation": "

Sets the maximum number of keys returned in the response. By default, the action returns\n up to 1,000 key names. The response might contain fewer keys but will never contain\n more.

" - } - }, - "CommonPrefixes": { - "target": "com.amazonaws.s3#CommonPrefixList", - "traits": { - "smithy.api#documentation": "

All of the keys (up to 1,000) that share the same prefix are grouped together. When counting the total numbers of returns by this API operation, \n this group of keys is considered as one item.

\n

A response can contain CommonPrefixes only if you specify a\n delimiter.

\n

\n CommonPrefixes contains all (if there are any) keys between\n Prefix and the next occurrence of the string specified by a\n delimiter.

\n

\n CommonPrefixes lists keys that act like subdirectories in the directory\n specified by Prefix.

\n

For example, if the prefix is notes/ and the delimiter is a slash\n (/) as in notes/summer/july, the common prefix is\n notes/summer/. All of the keys that roll up into a common prefix count as a\n single return when calculating the number of returns.

\n \n
    \n
  • \n

    \n Directory buckets - For directory buckets, only prefixes that end in a delimiter (/) are supported.

    \n
    • \n
  • \n

    \n Directory buckets - When you query ListObjectsV2 with a delimiter during in-progress multipart uploads, the \n CommonPrefixes response parameter contains the prefixes that are associated with the in-progress multipart uploads. \n For more information about multipart uploads, see Multipart Upload Overview in the Amazon S3 User Guide.

    \n
    • \n
\n ", - "smithy.api#xmlFlattened": {} - } - }, - "EncodingType": { - "target": "com.amazonaws.s3#EncodingType", - "traits": { - "smithy.api#documentation": "

Encoding type used by Amazon S3 to encode object key names in the XML response.

\n

If you specify the encoding-type request parameter, Amazon S3 includes this\n element in the response, and returns encoded key name values in the following response\n elements:

\n

\n Delimiter, Prefix, Key, and StartAfter.

" - } - }, - "KeyCount": { - "target": "com.amazonaws.s3#KeyCount", - "traits": { - "smithy.api#documentation": "

\n KeyCount is the number of keys returned with this request.\n KeyCount will always be less than or equal to the MaxKeys\n field. For example, if you ask for 50 keys, your result will include 50 keys or\n fewer.

" - } - }, - "ContinuationToken": { - "target": "com.amazonaws.s3#Token", - "traits": { - "smithy.api#documentation": "

If ContinuationToken was sent with the request, it is included in the\n response. You can use the returned ContinuationToken for pagination of the list response. You can use this ContinuationToken for pagination of the list results.

" - } - }, - "NextContinuationToken": { - "target": "com.amazonaws.s3#NextToken", - "traits": { - "smithy.api#documentation": "

\n NextContinuationToken is sent when isTruncated is true, which\n means there are more keys in the bucket that can be listed. The next list requests to Amazon S3\n can be continued with this NextContinuationToken.\n NextContinuationToken is obfuscated and is not a real key

" - } - }, - "StartAfter": { - "target": "com.amazonaws.s3#StartAfter", - "traits": { - "smithy.api#documentation": "

If StartAfter was sent with the request, it is included in the response.

\n \n

This functionality is not supported for directory buckets.

\n " - } - }, - "RequestCharged": { - "target": "com.amazonaws.s3#RequestCharged", - "traits": { - "smithy.api#httpHeader": "x-amz-request-charged" - } - } - }, - "traits": { - "smithy.api#output": {}, - "smithy.api#xmlName": "ListBucketResult" - } - }, - "com.amazonaws.s3#ListObjectsV2Request": { - "type": "structure", - "members": { - "Bucket": { - "target": "com.amazonaws.s3#BucketName", - "traits": { - "smithy.api#documentation": "

\n Directory buckets - When you use this operation with a directory bucket, you must use virtual-hosted-style requests in the format \n Bucket_name.s3express-az_id.region.amazonaws.com. Path-style requests are not supported. Directory bucket names must be unique in the chosen Availability Zone. Bucket names must follow the format \n bucket_base_name--az-id--x-s3 (for example, \n DOC-EXAMPLE-BUCKET--usw2-az1--x-s3). For information about bucket naming\n restrictions, see Directory bucket naming\n rules in the Amazon S3 User Guide.

\n

\n Access points - When you use this action with an access point, you must provide the alias of the access point in place of the bucket name or specify the access point ARN. When using the access point ARN, you must direct requests to the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see Using access points in the Amazon S3 User Guide.

\n \n

Access points and Object Lambda access points are not supported by directory buckets.

\n \n

\n S3 on Outposts - When you use this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form \n AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When you use this action with S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts access point ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see What is S3 on Outposts? in the Amazon S3 User Guide.

", - "smithy.api#httpLabel": {}, - "smithy.api#required": {}, - "smithy.rules#contextParam": { - "name": "Bucket" - } - } - }, - "Delimiter": { - "target": "com.amazonaws.s3#Delimiter", - "traits": { - "smithy.api#documentation": "

A delimiter is a character that you use to group keys.

\n \n
    \n
  • \n

    \n Directory buckets - For directory buckets, / is the only supported delimiter.

    \n
    • \n
  • \n

    \n Directory buckets - When you query ListObjectsV2 with a delimiter during in-progress multipart uploads, the \n CommonPrefixes response parameter contains the prefixes that are associated with the in-progress multipart uploads. \n For more information about multipart uploads, see Multipart Upload Overview in the Amazon S3 User Guide.

    \n
    • \n
\n ", - "smithy.api#httpQuery": "delimiter" - } - }, - "EncodingType": { - "target": "com.amazonaws.s3#EncodingType", - "traits": { - "smithy.api#documentation": "

Encoding type used by Amazon S3 to encode object keys in the response. If using\n url, non-ASCII characters used in an object's key name will be URL encoded.\n For example, the object test_file(3).png will appear as test_file%283%29.png.

", - "smithy.api#httpQuery": "encoding-type" - } - }, - "MaxKeys": { - "target": "com.amazonaws.s3#MaxKeys", - "traits": { - "smithy.api#documentation": "

Sets the maximum number of keys returned in the response. By default, the action returns\n up to 1,000 key names. The response might contain fewer keys but will never contain\n more.

", - "smithy.api#httpQuery": "max-keys" - } - }, - "Prefix": { - "target": "com.amazonaws.s3#Prefix", - "traits": { - "smithy.api#documentation": "

Limits the response to keys that begin with the specified prefix.

\n \n

\n Directory buckets - For directory buckets, only prefixes that end in a delimiter (/) are supported.

\n ", - "smithy.api#httpQuery": "prefix", - "smithy.rules#contextParam": { - "name": "Prefix" - } - } - }, - "ContinuationToken": { - "target": "com.amazonaws.s3#Token", - "traits": { - "smithy.api#documentation": "

\n ContinuationToken indicates to Amazon S3 that the list is being continued on\n this bucket with a token. ContinuationToken is obfuscated and is not a real\n key. You can use this ContinuationToken for pagination of the list results.

", - "smithy.api#httpQuery": "continuation-token" - } - }, - "FetchOwner": { - "target": "com.amazonaws.s3#FetchOwner", - "traits": { - "smithy.api#documentation": "

The owner field is not present in ListObjectsV2 by default. If you want to\n return the owner field with each key in the result, then set the FetchOwner\n field to true.

\n \n

\n Directory buckets - For directory buckets, the bucket owner is returned as the object owner for all objects.

\n ", - "smithy.api#httpQuery": "fetch-owner" - } - }, - "StartAfter": { - "target": "com.amazonaws.s3#StartAfter", - "traits": { - "smithy.api#documentation": "

StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this\n specified key. StartAfter can be any key in the bucket.

\n \n

This functionality is not supported for directory buckets.

\n ", - "smithy.api#httpQuery": "start-after" - } - }, - "RequestPayer": { - "target": "com.amazonaws.s3#RequestPayer", - "traits": { - "smithy.api#documentation": "

Confirms that the requester knows that she or he will be charged for the list objects\n request in V2 style. Bucket owners need not specify this parameter in their\n requests.

\n \n

This functionality is not supported for directory buckets.

\n ", - "smithy.api#httpHeader": "x-amz-request-payer" - } - }, - "ExpectedBucketOwner": { - "target": "com.amazonaws.s3#AccountId", - "traits": { - "smithy.api#documentation": "

The account ID of the expected bucket owner. If the account ID that you provide does not match the actual owner of the bucket, the request fails with the HTTP status code 403 Forbidden (access denied).

", - "smithy.api#httpHeader": "x-amz-expected-bucket-owner" - } - }, - "OptionalObjectAttributes": { - "target": "com.amazonaws.s3#OptionalObjectAttributesList", - "traits": { - "smithy.api#documentation": "

Specifies the optional fields that you want returned in the response. Fields that you do\n not specify are not returned.

\n \n

This functionality is not supported for directory buckets.

\n ", - "smithy.api#httpHeader": "x-amz-optional-object-attributes" - } - } - }, - "traits": { - "smithy.api#input": {} - } - }, "com.amazonaws.s3#PutObject": { "type": "operation", "input": { @@ -2205,69 +1994,6 @@ "com.amazonaws.s3#IsTruncated": { "type": "boolean" }, - "com.amazonaws.s3#ObjectList": { - "type": "list", - "member": { - "target": "com.amazonaws.s3#Object" - } - }, - "com.amazonaws.s3#Object": { - "type": "structure", - "members": { - "Key": { - "target": "com.amazonaws.s3#ObjectKey", - "traits": { - "smithy.api#documentation": "

The name that you assign to an object. You use the object key to retrieve the\n object.

" - } - }, - "LastModified": { - "target": "com.amazonaws.s3#LastModified", - "traits": { - "smithy.api#documentation": "

Creation date of the object.

" - } - }, - "ETag": { - "target": "com.amazonaws.s3#ETag", - "traits": { - "smithy.api#documentation": "

The entity tag is a hash of the object. The ETag reflects changes only to the contents\n of an object, not its metadata. The ETag may or may not be an MD5 digest of the object\n data. Whether or not it is depends on how the object was created and how it is encrypted as\n described below:

\n
    \n
  • \n

    Objects created by the PUT Object, POST Object, or Copy operation, or through the\n Amazon Web Services Management Console, and are encrypted by SSE-S3 or plaintext, have ETags that\n are an MD5 digest of their object data.

    \n
    • \n
  • \n

    Objects created by the PUT Object, POST Object, or Copy operation, or through the\n Amazon Web Services Management Console, and are encrypted by SSE-C or SSE-KMS, have ETags that are\n not an MD5 digest of their object data.

    \n
    • \n
  • \n

    If an object is created by either the Multipart Upload or Part Copy operation, the\n ETag is not an MD5 digest, regardless of the method of encryption. If an object is\n larger than 16 MB, the Amazon Web Services Management Console will upload or copy that object as a\n Multipart Upload, and therefore the ETag will not be an MD5 digest.

    \n
    • \n
\n \n

\n Directory buckets - MD5 is not supported by directory buckets.

\n " - } - }, - "ChecksumAlgorithm": { - "target": "com.amazonaws.s3#ChecksumAlgorithmList", - "traits": { - "smithy.api#documentation": "

The algorithm that was used to create a checksum of the object.

", - "smithy.api#xmlFlattened": {} - } - }, - "Size": { - "target": "com.amazonaws.s3#Size", - "traits": { - "smithy.api#documentation": "

Size in bytes of the object

" - } - }, - "StorageClass": { - "target": "com.amazonaws.s3#ObjectStorageClass", - "traits": { - "smithy.api#documentation": "

The class of storage used to store the object.

\n \n

\n Directory buckets - Only the S3 Express One Zone storage class is supported by directory buckets to store objects.

\n " - } - }, - "Owner": { - "target": "com.amazonaws.s3#Owner", - "traits": { - "smithy.api#documentation": "

The owner of the object

\n \n

\n Directory buckets - The bucket owner is returned as the object owner.

\n " - } - }, - "RestoreStatus": { - "target": "com.amazonaws.s3#RestoreStatus", - "traits": { - "smithy.api#documentation": "

Specifies the restoration status of an object. Objects in certain storage classes must\n be restored before they can be retrieved. For more information about these storage classes\n and how to work with archived objects, see Working with archived\n objects in the Amazon S3 User Guide.

\n \n

This functionality is not supported for directory buckets. Only the S3 Express One Zone storage class is supported by directory buckets to store objects.

\n " - } - } - }, - "traits": { - "smithy.api#documentation": "

An object consists of data and its descriptive metadata.

" - } - }, "com.amazonaws.s3#ChecksumAlgorithmList": { "type": "list", "member": { diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java index 03388026e..9811f66fa 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java @@ -31,8 +31,6 @@ import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectRequest; import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; -import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; -import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request; import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectRequest; @@ -152,41 +150,6 @@ public Result GetObject(GetObjectRequest input) { } } - @Override - public Result ListObjectsV2( - ListObjectsV2Request input - ) { - software.amazon.awssdk.services.s3.model.ListObjectsV2Request converted = - ToNative.ListObjectsV2Request(input); - try { - ListObjectsV2Response result = _impl.listObjectsV2(converted); - ListObjectsV2Output dafnyResponse = ToDafny.ListObjectsV2Output(result); - return Result.create_Success( - ListObjectsV2Output._typeDescriptor(), - Error._typeDescriptor(), - dafnyResponse - ); - } catch (NoSuchBucketException ex) { - return Result.create_Failure( - ListObjectsV2Output._typeDescriptor(), - Error._typeDescriptor(), - ToDafny.Error(ex) - ); - } catch (S3Exception ex) { - return Result.create_Failure( - ListObjectsV2Output._typeDescriptor(), - Error._typeDescriptor(), - ToDafny.Error(ex) - ); - } catch (Exception ex) { - return Result.create_Failure( - ListObjectsV2Output._typeDescriptor(), - Error._typeDescriptor(), - ToDafny.Error(ex) - ); - } - } - @Override public Result PutObject(PutObjectRequest input) { software.amazon.awssdk.services.s3.model.PutObjectRequest converted = diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index bdb418e14..29bd59d28 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -53,9 +53,6 @@ import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectRequest; import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; import software.amazon.cryptography.services.s3.internaldafny.types.IntelligentTieringAccessTier; -import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; -import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request; -import software.amazon.cryptography.services.s3.internaldafny.types.S3Object; import software.amazon.cryptography.services.s3.internaldafny.types.ObjectCannedACL; import software.amazon.cryptography.services.s3.internaldafny.types.ObjectIdentifier; import software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus; @@ -69,7 +66,6 @@ import software.amazon.cryptography.services.s3.internaldafny.types.RequestCharged; import software.amazon.cryptography.services.s3.internaldafny.types.RequestPayer; import software.amazon.cryptography.services.s3.internaldafny.types.RestoreStatus; -import software.amazon.cryptography.services.s3.internaldafny.types.S3Object; import software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption; import software.amazon.cryptography.services.s3.internaldafny.types.StorageClass; @@ -1143,282 +1139,6 @@ public static GetObjectRequest GetObjectRequest( ); } - public static ListObjectsV2Output ListObjectsV2Output( - ListObjectsV2Response nativeValue - ) { - Option isTruncated; - isTruncated = - Objects.nonNull(nativeValue.isTruncated()) - ? Option.create_Some( - TypeDescriptor.BOOLEAN, - (nativeValue.isTruncated()) - ) - : Option.create_None(TypeDescriptor.BOOLEAN); - Option> contents; - contents = - (Objects.nonNull(nativeValue.contents()) && - nativeValue.contents().size() > 0) - ? Option.create_Some( - DafnySequence._typeDescriptor(S3Object._typeDescriptor()), - ToDafny.ObjectList(nativeValue.contents()) - ) - : Option.create_None( - DafnySequence._typeDescriptor(S3Object._typeDescriptor()) - ); - Option> name; - name = - Objects.nonNull(nativeValue.name()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.name() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> prefix; - prefix = - Objects.nonNull(nativeValue.prefix()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.prefix() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> delimiter; - delimiter = - Objects.nonNull(nativeValue.delimiter()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.delimiter() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option maxKeys; - maxKeys = - Objects.nonNull(nativeValue.maxKeys()) - ? Option.create_Some(TypeDescriptor.INT, (nativeValue.maxKeys())) - : Option.create_None(TypeDescriptor.INT); - Option> commonPrefixes; - commonPrefixes = - (Objects.nonNull(nativeValue.commonPrefixes()) && - nativeValue.commonPrefixes().size() > 0) - ? Option.create_Some( - DafnySequence._typeDescriptor(CommonPrefix._typeDescriptor()), - ToDafny.CommonPrefixList(nativeValue.commonPrefixes()) - ) - : Option.create_None( - DafnySequence._typeDescriptor(CommonPrefix._typeDescriptor()) - ); - Option encodingType; - encodingType = - Objects.nonNull(nativeValue.encodingType()) - ? Option.create_Some( - EncodingType._typeDescriptor(), - ToDafny.EncodingType(nativeValue.encodingType()) - ) - : Option.create_None(EncodingType._typeDescriptor()); - Option keyCount; - keyCount = - Objects.nonNull(nativeValue.keyCount()) - ? Option.create_Some(TypeDescriptor.INT, (nativeValue.keyCount())) - : Option.create_None(TypeDescriptor.INT); - Option> continuationToken; - continuationToken = - Objects.nonNull(nativeValue.continuationToken()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.continuationToken() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> nextContinuationToken; - nextContinuationToken = - Objects.nonNull(nativeValue.nextContinuationToken()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.nextContinuationToken() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> startAfter; - startAfter = - Objects.nonNull(nativeValue.startAfter()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.startAfter() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option requestCharged; - requestCharged = - Objects.nonNull(nativeValue.requestCharged()) - ? Option.create_Some( - RequestCharged._typeDescriptor(), - ToDafny.RequestCharged(nativeValue.requestCharged()) - ) - : Option.create_None(RequestCharged._typeDescriptor()); - return new ListObjectsV2Output( - isTruncated, - contents, - name, - prefix, - delimiter, - maxKeys, - commonPrefixes, - encodingType, - keyCount, - continuationToken, - nextContinuationToken, - startAfter, - requestCharged - ); - } - - public static ListObjectsV2Request ListObjectsV2Request( - software.amazon.awssdk.services.s3.model.ListObjectsV2Request nativeValue - ) { - DafnySequence bucket; - bucket = - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.bucket() - ); - Option> delimiter; - delimiter = - Objects.nonNull(nativeValue.delimiter()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.delimiter() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option encodingType; - encodingType = - Objects.nonNull(nativeValue.encodingType()) - ? Option.create_Some( - EncodingType._typeDescriptor(), - ToDafny.EncodingType(nativeValue.encodingType()) - ) - : Option.create_None(EncodingType._typeDescriptor()); - Option maxKeys; - maxKeys = - Objects.nonNull(nativeValue.maxKeys()) - ? Option.create_Some(TypeDescriptor.INT, (nativeValue.maxKeys())) - : Option.create_None(TypeDescriptor.INT); - Option> prefix; - prefix = - Objects.nonNull(nativeValue.prefix()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.prefix() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> continuationToken; - continuationToken = - Objects.nonNull(nativeValue.continuationToken()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.continuationToken() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option fetchOwner; - fetchOwner = - Objects.nonNull(nativeValue.fetchOwner()) - ? Option.create_Some(TypeDescriptor.BOOLEAN, (nativeValue.fetchOwner())) - : Option.create_None(TypeDescriptor.BOOLEAN); - Option> startAfter; - startAfter = - Objects.nonNull(nativeValue.startAfter()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.startAfter() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option requestPayer; - requestPayer = - Objects.nonNull(nativeValue.requestPayer()) - ? Option.create_Some( - RequestPayer._typeDescriptor(), - ToDafny.RequestPayer(nativeValue.requestPayer()) - ) - : Option.create_None(RequestPayer._typeDescriptor()); - Option> expectedBucketOwner; - expectedBucketOwner = - Objects.nonNull(nativeValue.expectedBucketOwner()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.expectedBucketOwner() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option< - DafnySequence - > optionalObjectAttributes; - optionalObjectAttributes = - (Objects.nonNull(nativeValue.optionalObjectAttributes()) && - nativeValue.optionalObjectAttributes().size() > 0) - ? Option.create_Some( - DafnySequence._typeDescriptor( - OptionalObjectAttributes._typeDescriptor() - ), - ToDafny.OptionalObjectAttributesList( - nativeValue.optionalObjectAttributes() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor( - OptionalObjectAttributes._typeDescriptor() - ) - ); - return new ListObjectsV2Request( - bucket, - delimiter, - encodingType, - maxKeys, - prefix, - continuationToken, - fetchOwner, - startAfter, - requestPayer, - expectedBucketOwner, - optionalObjectAttributes - ); - } - public static DafnyMap< ? extends DafnySequence, ? extends DafnySequence @@ -1430,97 +1150,6 @@ > Metadata(Map nativeValue) { ); } - public static S3Object Object( - software.amazon.awssdk.services.s3.model.S3Object nativeValue - ) { - Option> key; - key = - Objects.nonNull(nativeValue.key()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.key() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> lastModified; - lastModified = - Objects.nonNull(nativeValue.lastModified()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.lastModified() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> eTag; - eTag = - Objects.nonNull(nativeValue.eTag()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.eTag() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> checksumAlgorithm; - checksumAlgorithm = - (Objects.nonNull(nativeValue.checksumAlgorithm()) && - nativeValue.checksumAlgorithm().size() > 0) - ? Option.create_Some( - DafnySequence._typeDescriptor(ChecksumAlgorithm._typeDescriptor()), - ToDafny.ChecksumAlgorithmList(nativeValue.checksumAlgorithm()) - ) - : Option.create_None( - DafnySequence._typeDescriptor(ChecksumAlgorithm._typeDescriptor()) - ); - Option size; - size = - Objects.nonNull(nativeValue.size()) - ? Option.create_Some(TypeDescriptor.LONG, (nativeValue.size())) - : Option.create_None(TypeDescriptor.LONG); - Option storageClass; - storageClass = - Objects.nonNull(nativeValue.storageClass()) - ? Option.create_Some( - ObjectStorageClass._typeDescriptor(), - ToDafny.ObjectStorageClass(nativeValue.storageClass()) - ) - : Option.create_None(ObjectStorageClass._typeDescriptor()); - Option owner; - owner = - Objects.nonNull(nativeValue.owner()) - ? Option.create_Some( - Owner._typeDescriptor(), - ToDafny.Owner(nativeValue.owner()) - ) - : Option.create_None(Owner._typeDescriptor()); - Option restoreStatus; - restoreStatus = - Objects.nonNull(nativeValue.restoreStatus()) - ? Option.create_Some( - RestoreStatus._typeDescriptor(), - ToDafny.RestoreStatus(nativeValue.restoreStatus()) - ) - : Option.create_None(RestoreStatus._typeDescriptor()); - return new S3Object( - key, - lastModified, - eTag, - checksumAlgorithm, - size, - storageClass, - owner, - restoreStatus - ); - } - public static ObjectIdentifier ObjectIdentifier( software.amazon.awssdk.services.s3.model.ObjectIdentifier nativeValue ) { @@ -1554,16 +1183,6 @@ public static DafnySequence ObjectIdentifierList( ); } - public static DafnySequence ObjectList( - List nativeValue - ) { - return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( - nativeValue, - software.amazon.cryptography.services.s3.internaldafny.ToDafny::Object, - S3Object._typeDescriptor() - ); - } - public static DafnySequence< ? extends OptionalObjectAttributes > OptionalObjectAttributesList( diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java index 0a8b464cd..c5f4c9bea 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java @@ -61,7 +61,6 @@ import software.amazon.cryptography.services.s3.internaldafny.types.Error_OpaqueWithText; import software.amazon.cryptography.services.s3.internaldafny.types.GetObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.IS3Client; -import software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Output; import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; public class ToNative { @@ -752,218 +751,19 @@ public static IntelligentTieringAccessTier IntelligentTieringAccessTier( return IntelligentTieringAccessTier.fromValue(dafnyValue.toString()); } - public static ListObjectsV2Response ListObjectsV2Output( - ListObjectsV2Output dafnyValue - ) { - ListObjectsV2Response.Builder builder = ListObjectsV2Response.builder(); - if (dafnyValue.dtor_CommonPrefixes().is_Some()) { - builder.commonPrefixes( - ToNative.CommonPrefixList(dafnyValue.dtor_CommonPrefixes().dtor_value()) - ); - } - if (dafnyValue.dtor_Contents().is_Some()) { - builder.contents( - ToNative.ObjectList(dafnyValue.dtor_Contents().dtor_value()) - ); - } - if (dafnyValue.dtor_ContinuationToken().is_Some()) { - builder.continuationToken( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_ContinuationToken().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Delimiter().is_Some()) { - builder.delimiter( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Delimiter().dtor_value() - ) - ); - } - if (dafnyValue.dtor_EncodingType().is_Some()) { - builder.encodingType( - ToNative.EncodingType(dafnyValue.dtor_EncodingType().dtor_value()) - ); - } - if (dafnyValue.dtor_IsTruncated().is_Some()) { - builder.isTruncated((dafnyValue.dtor_IsTruncated().dtor_value())); - } - if (dafnyValue.dtor_KeyCount().is_Some()) { - builder.keyCount((dafnyValue.dtor_KeyCount().dtor_value())); - } - if (dafnyValue.dtor_MaxKeys().is_Some()) { - builder.maxKeys((dafnyValue.dtor_MaxKeys().dtor_value())); - } - if (dafnyValue.dtor_Name().is_Some()) { - builder.name( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Name().dtor_value() - ) - ); - } - if (dafnyValue.dtor_NextContinuationToken().is_Some()) { - builder.nextContinuationToken( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_NextContinuationToken().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Prefix().is_Some()) { - builder.prefix( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Prefix().dtor_value() - ) - ); - } - if (dafnyValue.dtor_RequestCharged().is_Some()) { - builder.requestCharged( - ToNative.RequestCharged(dafnyValue.dtor_RequestCharged().dtor_value()) - ); - } - if (dafnyValue.dtor_StartAfter().is_Some()) { - builder.startAfter( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_StartAfter().dtor_value() - ) - ); - } - return builder.build(); - } - - public static ListObjectsV2Request ListObjectsV2Request( - software.amazon.cryptography.services.s3.internaldafny.types.ListObjectsV2Request dafnyValue - ) { - ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder(); - builder.bucket( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Bucket() - ) - ); - if (dafnyValue.dtor_ContinuationToken().is_Some()) { - builder.continuationToken( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_ContinuationToken().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Delimiter().is_Some()) { - builder.delimiter( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Delimiter().dtor_value() - ) - ); - } - if (dafnyValue.dtor_EncodingType().is_Some()) { - builder.encodingType( - ToNative.EncodingType(dafnyValue.dtor_EncodingType().dtor_value()) - ); - } - if (dafnyValue.dtor_ExpectedBucketOwner().is_Some()) { - builder.expectedBucketOwner( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_ExpectedBucketOwner().dtor_value() - ) - ); - } - if (dafnyValue.dtor_FetchOwner().is_Some()) { - builder.fetchOwner((dafnyValue.dtor_FetchOwner().dtor_value())); - } - if (dafnyValue.dtor_MaxKeys().is_Some()) { - builder.maxKeys((dafnyValue.dtor_MaxKeys().dtor_value())); - } - if (dafnyValue.dtor_OptionalObjectAttributes().is_Some()) { - builder.optionalObjectAttributes( - ToNative.OptionalObjectAttributesList( - dafnyValue.dtor_OptionalObjectAttributes().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Prefix().is_Some()) { - builder.prefix( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Prefix().dtor_value() - ) - ); - } - if (dafnyValue.dtor_RequestPayer().is_Some()) { - builder.requestPayer( - ToNative.RequestPayer(dafnyValue.dtor_RequestPayer().dtor_value()) - ); - } - if (dafnyValue.dtor_StartAfter().is_Some()) { - builder.startAfter( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_StartAfter().dtor_value() - ) - ); - } - return builder.build(); - } - public static Map Metadata( - DafnyMap< - ? extends DafnySequence, - ? extends DafnySequence - > dafnyValue + DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > dafnyValue ) { return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToMap( - dafnyValue, - software.amazon.smithy.dafny.conversion.ToNative.Simple::String, - software.amazon.smithy.dafny.conversion.ToNative.Simple::String + dafnyValue, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String ); } - public static S3Object S3Object( - software.amazon.cryptography.services.s3.internaldafny.types.S3Object dafnyValue - ) { - S3Object.Builder builder = S3Object.builder(); - if (dafnyValue.dtor_ChecksumAlgorithm().is_Some()) { - builder.checksumAlgorithm( - ToNative.ChecksumAlgorithmList( - dafnyValue.dtor_ChecksumAlgorithm().dtor_value() - ) - ); - } - if (dafnyValue.dtor_ETag().is_Some()) { - builder.eTag( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_ETag().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Key().is_Some()) { - builder.key( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Key().dtor_value() - ) - ); - } - if (dafnyValue.dtor_LastModified().is_Some()) { - builder.lastModified( - software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( - dafnyValue.dtor_LastModified().dtor_value() - ) - ); - } - if (dafnyValue.dtor_Owner().is_Some()) { - builder.owner(ToNative.Owner(dafnyValue.dtor_Owner().dtor_value())); - } - if (dafnyValue.dtor_RestoreStatus().is_Some()) { - builder.restoreStatus( - ToNative.RestoreStatus(dafnyValue.dtor_RestoreStatus().dtor_value()) - ); - } - if (dafnyValue.dtor_Size().is_Some()) { - builder.size((dafnyValue.dtor_Size().dtor_value())); - } - if (dafnyValue.dtor_StorageClass().is_Some()) { - builder.storageClass( - ToNative.ObjectStorageClass(dafnyValue.dtor_StorageClass().dtor_value()) - ); - } - return builder.build(); - } - public static ObjectCannedACL ObjectCannedACL( software.amazon.cryptography.services.s3.internaldafny.types.ObjectCannedACL dafnyValue ) { @@ -1021,17 +821,6 @@ public static List ObjectIdentifierList( ); } - public static List ObjectList( - DafnySequence< - ? extends software.amazon.cryptography.services.s3.internaldafny.types.S3Object - > dafnyValue - ) { - return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( - dafnyValue, - software.amazon.cryptography.services.s3.internaldafny.ToNative::S3Object - ); - } - public static ObjectLockLegalHoldStatus ObjectLockLegalHoldStatus( software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus dafnyValue ) { diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index bcae6df3f..fda0789a4 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -454,8 +454,7 @@ public TokenTree generateStructureTypeDefinition( StructureShape.class ); - final String typeName = structureShapeId.getName().equals("Object") ? - "S3Object" : structureShapeId.getName(); + final String typeName = structureShapeId.getName(); return TokenTree.of( Token.of("datatype %1$s =".formatted(typeName)), generateDataTypeConstructorFromStructure(structureShapeId) @@ -1952,8 +1951,7 @@ public TokenTree generateDataTypeConstructorFromStructure( shapeId, StructureShape.class ); - final String typeName = shapeId.getName().equals("Object") ? - "S3Object" : shapeId.getName(); + final String typeName = shapeId.getName(); final TokenTree params = TokenTree .of( diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index d5e1d3d43..82c8b3d8c 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -139,9 +139,6 @@ private String baseTypeForMember(final MemberShape memberShape) { } private String dafnyTypeNameShape(final Shape shape) { - if (shape.getId().getName().equals("Object")) { - return dafnyModulePrefixForShape(shape) + "S3Object"; - } return dafnyModulePrefixForShape(shape) + shape.getId().getName(); } From 75252109ab856a34aae757ff9525dd17ab89574c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 14:15:41 -0700 Subject: [PATCH 36/72] Fix ProviderAsInputStream --- .../java/src/main/java/Streams/ProviderAsInputStream.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java index 192774f18..3d05b55e6 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java @@ -43,21 +43,20 @@ public int read() throws IOException { public int read(byte[] b, int off, int len) throws IOException { // TODO: Could optimize to use b directly, // but that starts to introduce risk. - Array array = Array.newArray(TypeDescriptor.BYTE, len); + byte[] storage = new byte[len]; BatchArrayWriter consumer = new BatchArrayWriter<>( TypeDescriptor.BYTE, TypeDescriptor.reference(Exception.class) ); - consumer.__ctor(array); + consumer.__ctor(storage); producer.Fill(consumer); if (consumer.state.is_Failure()) { throw new IOException(consumer.state.dtor_error()); } - byte[] results = (byte[])array.unwrap(); int count = consumer.size.intValueExact(); if (count == 0) { return -1; } else { - System.arraycopy(results, 0, b, off, count); + System.arraycopy(storage, 0, b, off, count); return count; } } From 1b52f0ec55dc2713c2c5410d5e031eb8cbc2b005 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Apr 2025 14:47:21 -0700 Subject: [PATCH 37/72] Partially fix Streaming --- TestModels/Streaming/{src => }/dfyconfig.toml | 0 .../src/WrappedSimpleStreamingImpl.dfy | 2 +- .../test/SimpleStreamingImplTest.dfy | 36 ++++++++++--------- .../test/WrappedSimpleStreamingTest.dfy | 1 + .../aws-sdks/s3/test/TestComAmazonawsS3.dfy | 2 -- 5 files changed, 21 insertions(+), 20 deletions(-) rename TestModels/Streaming/{src => }/dfyconfig.toml (100%) diff --git a/TestModels/Streaming/src/dfyconfig.toml b/TestModels/Streaming/dfyconfig.toml similarity index 100% rename from TestModels/Streaming/src/dfyconfig.toml rename to TestModels/Streaming/dfyconfig.toml diff --git a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy index 00827803f..58bf8aed5 100644 --- a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/SimpleStreamingTypesWrapped.dfy" -module WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { +module {:extern "simple.streaming.internaldafny.wrapped"} WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { import WrappedService = SimpleStreaming function method WrappedDefaultSimpleStreamingConfig(): SimpleStreamingConfig { SimpleStreamingConfig diff --git a/TestModels/Streaming/test/SimpleStreamingImplTest.dfy b/TestModels/Streaming/test/SimpleStreamingImplTest.dfy index 411dae3b8..cbf3d2fd8 100644 --- a/TestModels/Streaming/test/SimpleStreamingImplTest.dfy +++ b/TestModels/Streaming/test/SimpleStreamingImplTest.dfy @@ -6,11 +6,11 @@ include "../src/WrappedSimpleStreamingImpl.dfy" module SimpleStreamingImplTest { import SimpleStreaming import SimpleStreamingImpl - import Std.Enumerators - import Std.Aggregators - import Std.BoundedInts + import Std.Producers + import Std.Consumers + import Std.BulkActions import opened StandardLibrary.UInt - import opened Std.Streams + import opened StandardLibrary.Streams import opened SimpleStreamingTypes import opened Wrappers method{:test} TestClient(){ @@ -24,9 +24,8 @@ module SimpleStreamingImplTest { modifies client.Modifies ensures client.ValidState() { - var s: seq := [[0x0], [0x1, 0x2], [0x3], [], [0x4, 0x5]]; - var e := new Enumerators.SeqEnumerator(s); - var stream := new EnumeratorDataStream(e, length := 5 as BoundedInts.uint64); + var s: seq := [0x0, 0x1, 0x2, 0x3, 0x4, 0x5]; + var stream := new SeqDataStream(s); var input: CountBitsInput := CountBitsInput(bits := stream); var ret :- expect client.CountBits(input); @@ -43,11 +42,13 @@ module SimpleStreamingImplTest { var ret :- expect client.BinaryOf(input); - var collector := new Aggregators.Collector(); + var collector := new BulkActions.BatchSeqWriter(); + var collectorTotalProof := new BulkActions.BatchSeqWriterTotalProof(collector); - Enumerators.ForEach(ret.binary, collector); + var reader := ret.binary.Reader(); + reader.ForEach(collector, collectorTotalProof); - expect collector.values == [[12], [34, 56]]; + expect collector.elements == [12, 34, 56]; } method TestChunks(client: ISimpleStreamingClient) @@ -55,17 +56,18 @@ module SimpleStreamingImplTest { modifies client.Modifies ensures client.ValidState() { - var s: seq := [[0x0], [0x1, 0x2], [0x3], [], [0x4, 0x5]]; - var e := new Enumerators.SeqEnumerator(s); - var stream := new EnumeratorDataStream(e, 5 as BoundedInts.uint64); - var input: ChunksInput := ChunksInput(bytesIn := stream, chunkSize := 2); + var s: bytes := [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7]; + var stream := new SeqDataStream(s); + var input: ChunksInput := ChunksInput(bytesIn := stream, chunkSize := 3); var ret :- expect client.Chunks(input); - var collector := new Aggregators.Collector(); + var collector := new BulkActions.BatchSeqWriter(); + var collectorTotalProof := new BulkActions.BatchSeqWriterTotalProof(collector); - Enumerators.ForEach(ret.bytesOut, collector); + var reader := ret.bytesOut.Reader(); + reader.ForEach(collector, collectorTotalProof); - expect collector.values == [[0x0, 0x1], [0x2, 0x3], [0x4, 0x5]]; + expect collector.elements == [0x2, 0x1, 0x0, 0x5, 0x4, 0x3, 0x7, 0x6]; } } \ No newline at end of file diff --git a/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy b/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy index 20e848dbf..ef144c1ee 100644 --- a/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy +++ b/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy @@ -11,5 +11,6 @@ module WrappedSimpleTypesStringTest { var client :- expect WrappedSimpleStreamingService.WrappedSimpleStreaming(); SimpleStreamingImplTest.TestCountBits(client); SimpleStreamingImplTest.TestBinaryOf(client); + SimpleStreamingImplTest.TestChunks(client); } } diff --git a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy index a5bc22634..e2b1a7c72 100644 --- a/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy +++ b/TestModels/aws-sdks/s3/test/TestComAmazonawsS3.dfy @@ -24,8 +24,6 @@ module TestComAmazonawsS3 { Key := testObjectKey ) ); - // Note the chunk size has to ensure all but the last chunk is >= 8192 bytes. - // For a small stream like this that means just one chunk. var s: DataStream := new SeqDataStream([ 97, 115, 100, 102 ]); PutObjectTest( input := S3.Types.PutObjectRequest( From 3724a569a4679a523d548e55be95f5250442ef3a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 29 Apr 2025 11:23:05 -0700 Subject: [PATCH 38/72] Fix Java codegen --- TestModels/Streaming/src/Chunker.dfy | 4 +-- .../Streaming/src/SimpleStreamingImpl.dfy | 10 ++++---- .../services/s3/internaldafny/Shim.java | 10 ++------ .../services/s3/internaldafny/ToDafny.java | 2 +- .../runtimes/java/build.gradle.kts | 2 +- .../DataStreamAsContentStreamProvider.java | 20 --------------- .../java/Streams/DataStreamAsRequestBody.java | 24 ++++++++++++++++++ ...Stream.java => ProducerAsInputStream.java} | 5 ++-- .../smithyjava/generator/ToDafny.java | 4 +++ .../smithyjava/generator/ToNative.java | 5 ++++ .../generator/library/ToDafnyLibrary.java | 25 +++++++++++++++++++ .../generator/library/ToNativeLibrary.java | 4 +++ .../smithyjava/nameresolver/Constants.java | 15 +++++++++++ .../smithyjava/nameresolver/Dafny.java | 17 ++++++++++++- .../smithyjava/nameresolver/Native.java | 12 +++++++-- .../templates/src/$forLocalService;LIndex.dfy | 2 +- .../templates/src/Wrapped$service;LImpl.dfy | 4 +-- 17 files changed, 119 insertions(+), 46 deletions(-) delete mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java create mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java rename TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/{ProviderAsInputStream.java => ProducerAsInputStream.java} (94%) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 0ae33ecf8..18882f995 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -228,7 +228,7 @@ module {:options "--function-syntax:4"} Chunker { twostate predicate ValidChange() reads this, Repr ensures ValidChange() ==> - old(Valid()) && Valid() && fresh(Repr - old(Repr)) + old(Valid()) && Valid() && fresh(Repr - old(Repr)) { old(Valid()) && Valid() && fresh(Repr - old(Repr)) } @@ -268,7 +268,7 @@ module {:options "--function-syntax:4"} Chunker { } method Reader() returns (p: Producer) - ensures + ensures && p.Valid() && fresh(p.Repr) && p.history == [] diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index 4f1a64131..7478a7274 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -11,7 +11,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl import Std.Consumers import Std.Collections.Seq import opened Chunker - + datatype Config = Config type InternalConfig = Config predicate ValidInternalConfig?(config: InternalConfig) @@ -26,7 +26,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl { var counter := new Consumers.FoldingConsumer(Success(0 as int32), SumBits); var counterTotalProof := new Consumers.FoldingConsumerTotalActionProof(counter); - + var inputReader := input.bits.Reader(); inputReader.ForEach(counter, counterTotalProof); var result := counter.value; @@ -40,7 +40,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl function SumBits(sum: Result, batched: BulkActions.Batched): Result { match batched - case BatchValue(b) => + case BatchValue(b) => if sum.Success? then var next := BitCount(b); if !(0 <= sum.value as int + next < INT32_MAX_LIMIT) then @@ -66,7 +66,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl { var binary := BinaryOfNumber(input.number); var binaryStream := new SeqDataStream(binary); - + return Success(BinaryOfOutput(binary := binaryStream)); } @@ -78,7 +78,7 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl returns (output: Result) { var chunkerStream := new ChunkingStream(input.bytesIn, input.chunkSize); - + return Success(ChunksOutput(bytesOut := chunkerStream)); } diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java index 9811f66fa..1f2d4ea0f 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java @@ -5,6 +5,7 @@ import StandardLibrary_Compile.Streams_Compile.DataStream; import Std_Compile.Producers_Compile.Producer; +import Streams.DataStreamAsRequestBody; import Streams.ProviderAsInputStream; import Wrappers_Compile.Result; import java.lang.Override; @@ -154,14 +155,7 @@ public Result GetObject(GetObjectRequest input) { public Result PutObject(PutObjectRequest input) { software.amazon.awssdk.services.s3.model.PutObjectRequest converted = ToNative.PutObjectRequest(input); - DataStream dataStream = input._Body.dtor_value(); - ContentStreamProvider provider = () -> { - Producer reader = dataStream.Reader(); - return new ProviderAsInputStream(reader); - }; - RequestBody body = dataStream.ContentLength().is_Some() - ? RequestBody.fromContentProvider(provider, dataStream.ContentLength().dtor_value().longValueExact(), "application/octet-stream") - : RequestBody.fromContentProvider(provider, "application/octet-stream"); + RequestBody body = DataStreamAsRequestBody.of(input._Body.dtor_value()); try { PutObjectResponse result = _impl.putObject(converted, body); PutObjectOutput dafnyResponse = ToDafny.PutObjectOutput(result); diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index 29bd59d28..758be1378 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -473,7 +473,7 @@ public static GetObjectOutput GetObjectOutput(ResponseInputStream(Error._typeDescriptor(), responseInputStream, e -> Error.create_Opaque(e)) + new InputStreamAsDataStream(Error._typeDescriptor(), responseInputStream, Error::create_Opaque) ); Option deleteMarker; deleteMarker = diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts index 7d9411dbe..5d7329565 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts @@ -63,7 +63,7 @@ repositories { dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") - implementation("software.amazon.awssdk:sdk-core:2.28.28") + api("software.amazon.awssdk:sdk-core:2.28.28") } publishing { publications.create("maven") { diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java deleted file mode 100644 index 7a3ce8bb9..000000000 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsContentStreamProvider.java +++ /dev/null @@ -1,20 +0,0 @@ -package Streams; - -import StandardLibrary_Compile.Streams_Compile.DataStream; -import software.amazon.awssdk.http.ContentStreamProvider; - -import java.io.InputStream; - -public class DataStreamAsContentStreamProvider implements ContentStreamProvider { - - private final DataStream dataStream; - - public DataStreamAsContentStreamProvider(DataStream dataStream) { - this.dataStream = dataStream; - } - - @Override - public InputStream newStream() { - return new ProviderAsInputStream(dataStream.Reader()); - } -} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java new file mode 100644 index 000000000..cfd617a13 --- /dev/null +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java @@ -0,0 +1,24 @@ +package Streams; + +import StandardLibrary_Compile.Streams_Compile.DataStream; +import Std_Compile.Producers_Compile.Producer; +import Std_Compile.Wrappers_Compile.Option; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.ContentStreamProvider; + +import java.math.BigInteger; + +public class DataStreamAsRequestBody { + + public static RequestBody of(DataStream dataStream) { + final ContentStreamProvider provider = () -> { + Producer reader = dataStream.Reader(); + return new ProducerAsInputStream(reader); + }; + final Option contentLength = dataStream.ContentLength(); + + return contentLength.is_Some() + ? RequestBody.fromContentProvider(provider, contentLength.dtor_value().longValueExact(), "application/octet-stream") + : RequestBody.fromContentProvider(provider, "application/octet-stream"); + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java similarity index 94% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java rename to TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java index 3d05b55e6..460a2fe43 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProviderAsInputStream.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java @@ -5,18 +5,17 @@ import Std_Compile.Consumers_Compile.IgnoreNConsumer; import Std_Compile.Producers_Compile.Producer; import Std_Compile.Wrappers_Compile.Option; -import dafny.Array; import dafny.TypeDescriptor; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; -public class ProviderAsInputStream extends InputStream { +public class ProducerAsInputStream extends InputStream { private final Producer> producer; - public ProviderAsInputStream(Producer> producer) { + public ProducerAsInputStream(Producer> producer) { this.producer = producer; } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToDafny.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToDafny.java index 9ec737662..ae2343e8a 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToDafny.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToDafny.java @@ -39,6 +39,7 @@ import software.amazon.smithy.model.shapes.UnionShape; import software.amazon.smithy.model.traits.EnumDefinition; import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.traits.StreamingTrait; public abstract class ToDafny extends Generator { @@ -478,6 +479,9 @@ protected MethodReference conversionMethodReference(Shape shape) { ) ); } + if (shape.isBlobShape() && shape.hasTrait(StreamingTrait.class)) { + return new MethodReference(thisClassName, "DataStream"); + } // If the target is simple, use SIMPLE_CONVERSION_METHOD_FROM_SHAPE_TYPE if (ModelUtils.isSmithyApiOrSimpleShape(shape)) { return SIMPLE_CONVERSION_METHOD_FROM_SHAPE_TYPE.get(shape.getType()); diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java index 71d1e48b2..186622f5a 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java @@ -18,6 +18,7 @@ import software.amazon.polymorph.smithyjava.generator.CodegenSubject.AwsSdkVersion; import software.amazon.polymorph.smithyjava.generator.awssdk.v1.ToNativeAwsV1; import software.amazon.polymorph.smithyjava.generator.awssdk.v2.ToNativeAwsV2; +import software.amazon.polymorph.smithyjava.nameresolver.Constants; import software.amazon.polymorph.smithyjava.nameresolver.Dafny; import software.amazon.polymorph.utils.AwsSdkNameResolverHelpers; import software.amazon.polymorph.utils.ModelUtils; @@ -33,6 +34,7 @@ import software.amazon.smithy.model.shapes.UnionShape; import software.amazon.smithy.model.traits.EnumDefinition; import software.amazon.smithy.model.traits.EnumTrait; +import software.amazon.smithy.model.traits.StreamingTrait; public abstract class ToNative extends Generator { @@ -408,6 +410,9 @@ protected MethodReference conversionMethodReference(Shape shape) { "MemberShapes MUST BE de-referenced BEFORE calling ToNative.conversionMethodReference." ); } + if (shape.isBlobShape() && shape.hasTrait(StreamingTrait.class)) { + return new MethodReference(software.amazon.polymorph.smithyjava.nameresolver.Constants.DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME, "of"); + } // If the target is simple, use SIMPLE_CONVERSION_METHOD_FROM_SHAPE_TYPE if (ModelUtils.isSmithyApiOrSimpleShape(shape)) { return SIMPLE_CONVERSION_METHOD_FROM_SHAPE_TYPE.get(shape.getType()); diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java index a4aeacd97..89226b1df 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java @@ -19,8 +19,11 @@ import java.util.Set; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; + +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithyjava.MethodReference; import software.amazon.polymorph.smithyjava.generator.ToDafny; +import software.amazon.polymorph.smithyjava.nameresolver.Constants; import software.amazon.polymorph.smithyjava.nameresolver.Dafny; import software.amazon.polymorph.smithyjava.nameresolver.Native; import software.amazon.polymorph.smithyjava.unmodeled.CollectionOfErrors; @@ -87,6 +90,8 @@ TypeSpec toDafny() { toDafnyMethods.add(opaqueWithTextError()); // CollectionError toDafnyMethods.add(collectionError()); + // DataStreams + toDafnyMethods.add(dataStream()); // Structures subject .getStructuresInServiceNamespace() @@ -269,6 +274,26 @@ MethodSpec collectionError() { .build(); } + MethodSpec dataStream() { + ClassName dafnyDataStream = software.amazon.polymorph.smithyjava.nameresolver.Constants.DAFNY_DATA_STREAM_CLASS_NAME; + TypeName dafnyDataStreamParameterized = ParameterizedTypeName.get( + dafnyDataStream, + ClassName.get(Byte.class), + subject.dafnyNameResolver.abstractClassForError() + ); + return MethodSpec + .methodBuilder("DataStream") + .returns(dafnyDataStreamParameterized) + .addModifiers(PUBLIC_STATIC) + .addParameter(ClassName.get(RequestBody.class), VAR_INPUT) + .addStatement("return new $T(\n" + + " Error._typeDescriptor(),\n" + + " $L,\n" + + " Error::create_Opaque\n" + + ")", software.amazon.polymorph.smithyjava.nameresolver.Constants.REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME, VAR_INPUT) + .build(); + } + @Override protected MethodSpec modeledStructure(final StructureShape structureShape) { if (structureShape.hasTrait(PositionalTrait.class)) { diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java index 1e38389f4..54487c042 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java @@ -10,6 +10,7 @@ import com.squareup.javapoet.CodeBlock; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; import java.util.ArrayList; @@ -19,9 +20,12 @@ import java.util.Set; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; + +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithydafny.DafnyNameResolver; import software.amazon.polymorph.smithyjava.MethodReference; import software.amazon.polymorph.smithyjava.generator.ToNative; +import software.amazon.polymorph.smithyjava.nameresolver.Constants; import software.amazon.polymorph.smithyjava.nameresolver.Dafny; import software.amazon.polymorph.smithyjava.nameresolver.Native; import software.amazon.polymorph.smithyjava.unmodeled.CollectionOfErrors; diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java index ab2d8264f..2cf8fff40 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java @@ -6,6 +6,8 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Set; + +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.shapes.ShapeType; @@ -57,4 +59,17 @@ public class Constants { public static final ClassName BASE64_DECODER_CLASS_NAME = ClassName.get( Base64.Decoder.class ); + public static final ClassName DAFNY_DATA_STREAM_CLASS_NAME = ClassName.get( + "StandardLibrary_Compile.Streams_Compile", + "DataStream" + ); + public static final ClassName REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME = ClassName.get( + "Streams", + "RequestBodyAsDataStream" + ); + public static final ClassName DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME = ClassName.get( + "Streams", + "DataStreamAsRequestBody" + ); + } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Dafny.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Dafny.java index a754fb000..1a7200295 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Dafny.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Dafny.java @@ -36,6 +36,7 @@ import software.amazon.smithy.model.shapes.StructureShape; import software.amazon.smithy.model.traits.EnumTrait; import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.StreamingTrait; /** * Provides a consistent mapping between names of @@ -338,7 +339,13 @@ public TypeName typeForShape(final ShapeId shapeId) { new IllegalStateException("Cannot find shape " + shapeId) ); return switch (shape.getType()) { - case BLOB -> Dafny.typeForBlob(); + case BLOB -> { + if (shape.hasTrait(StreamingTrait.class)) { + yield typeForStreamingBlob(); + } else { + yield Dafny.typeForBlob(); + } + } case BOOLEAN -> TypeName.BOOLEAN.box(); case STRING -> typeForString(shape.asStringShape().get()); case TIMESTAMP -> typeForCharacterSequence(); @@ -370,6 +377,14 @@ private static TypeName typeForBlob() { ); } + public TypeName typeForStreamingBlob() { + return ParameterizedTypeName.get( + Constants.DAFNY_DATA_STREAM_CLASS_NAME, + TypeName.BYTE.box(), + abstractClassForError() + ); + } + static TypeName typeForDouble() { return typeForBlob(); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java index 2035d91e4..cdf4c9c85 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java @@ -18,6 +18,8 @@ import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; + +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithydafny.DafnyNameResolver; import software.amazon.polymorph.smithyjava.NamespaceHelper; import software.amazon.polymorph.smithyjava.generator.CodegenSubject; @@ -159,9 +161,15 @@ public TypeName typeForShape(final ShapeId shapeId) { ); yield shape.hasTrait(BoxTrait.class) ? typeName.box() : typeName; } + case BLOB -> { + if (shape.hasTrait(StreamingTrait.class)) { + yield ClassName.get(RequestBody.class); + } else { + yield NATIVE_TYPES_BY_SIMPLE_SHAPE_TYPE.get(shape.getType()); + } + } // For supported simple shapes, just map to native types - case BLOB, - TIMESTAMP, + case TIMESTAMP, BIG_DECIMAL, BIG_INTEGER -> NATIVE_TYPES_BY_SIMPLE_SHAPE_TYPE.get(shape.getType()); case STRING, ENUM -> classForStringOrEnum(shape); diff --git a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy index 0f4c46470..9afda92a7 100644 --- a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy +++ b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "$sdkID:LImpl.dfy" -module {:extern "$namespace:L.internaldafny" } $sdkID:L refines Abstract$dafnyModuleName:LService { +module {:extern "$namespace:L.internaldafny"} $sdkID:L refines Abstract$dafnyModuleName:LService { import Operations = $service:LImpl function method Default$serviceConfig:L(): $serviceConfig:L { diff --git a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy index 85ccdd15d..88f8014ad 100644 --- a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy +++ b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/$service:LTypesWrapped.dfy" -module {:options "--function-syntax:4"} {:extern "$namespace:L.internaldafny.wrapped"} Wrapped$service:LService refines WrappedAbstract$service:LService { +module {:extern "$namespace:L.internaldafny.wrapped"} Wrapped$service:LService refines WrappedAbstract$service:LService { import WrappedService = $sdkID:L - function WrappedDefault$serviceConfig:L(): $serviceConfig:L { + function method WrappedDefault$serviceConfig:L(): $serviceConfig:L { $serviceConfig:L } } From 6ae6eabbdd494d9a7f97883b5c97efc3e2f3beb3 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 2 May 2025 15:36:46 -0700 Subject: [PATCH 39/72] Update java after removing ListObjectsV2 --- .../services/s3/internaldafny/Shim.java | 6 - .../services/s3/internaldafny/ToDafny.java | 240 ------------------ .../services/s3/internaldafny/ToNative.java | 170 +------------ 3 files changed, 8 insertions(+), 408 deletions(-) diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java index 1f2d4ea0f..bc7fb494d 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java @@ -3,24 +3,18 @@ // Do not modify this file. This file is machine generated, and any changes to it will be overwritten. package software.amazon.cryptography.services.s3.internaldafny; -import StandardLibrary_Compile.Streams_Compile.DataStream; -import Std_Compile.Producers_Compile.Producer; import Streams.DataStreamAsRequestBody; -import Streams.ProviderAsInputStream; import Wrappers_Compile.Result; import java.lang.Override; import java.lang.String; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index 758be1378..4be3cef8d 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -29,21 +29,18 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm; import software.amazon.cryptography.services.s3.internaldafny.types.ChecksumMode; -import software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix; import software.amazon.cryptography.services.s3.internaldafny.types.Delete; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectRequest; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsOutput; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsRequest; import software.amazon.cryptography.services.s3.internaldafny.types.DeletedObject; -import software.amazon.cryptography.services.s3.internaldafny.types.EncodingType; import software.amazon.cryptography.services.s3.internaldafny.types.Error; import software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape; import software.amazon.cryptography.services.s3.internaldafny.types.Error_InvalidObjectState; @@ -57,60 +54,16 @@ import software.amazon.cryptography.services.s3.internaldafny.types.ObjectIdentifier; import software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockLegalHoldStatus; import software.amazon.cryptography.services.s3.internaldafny.types.ObjectLockMode; -import software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass; -import software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes; -import software.amazon.cryptography.services.s3.internaldafny.types.Owner; import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.PutObjectRequest; import software.amazon.cryptography.services.s3.internaldafny.types.ReplicationStatus; import software.amazon.cryptography.services.s3.internaldafny.types.RequestCharged; import software.amazon.cryptography.services.s3.internaldafny.types.RequestPayer; -import software.amazon.cryptography.services.s3.internaldafny.types.RestoreStatus; import software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption; import software.amazon.cryptography.services.s3.internaldafny.types.StorageClass; public class ToDafny { - public static DafnySequence< - ? extends ChecksumAlgorithm - > ChecksumAlgorithmList( - List nativeValue - ) { - return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( - nativeValue, - software.amazon.cryptography.services.s3.internaldafny.ToDafny::ChecksumAlgorithm, - ChecksumAlgorithm._typeDescriptor() - ); - } - - public static CommonPrefix CommonPrefix( - software.amazon.awssdk.services.s3.model.CommonPrefix nativeValue - ) { - Option> prefix; - prefix = - Objects.nonNull(nativeValue.prefix()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.prefix() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - return new CommonPrefix(prefix); - } - - public static DafnySequence CommonPrefixList( - List nativeValue - ) { - return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( - nativeValue, - software.amazon.cryptography.services.s3.internaldafny.ToDafny::CommonPrefix, - CommonPrefix._typeDescriptor() - ); - } - public static Delete Delete( software.amazon.awssdk.services.s3.model.Delete nativeValue ) { @@ -1183,50 +1136,6 @@ public static DafnySequence ObjectIdentifierList( ); } - public static DafnySequence< - ? extends OptionalObjectAttributes - > OptionalObjectAttributesList( - List< - software.amazon.awssdk.services.s3.model.OptionalObjectAttributes - > nativeValue - ) { - return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( - nativeValue, - software.amazon.cryptography.services.s3.internaldafny.ToDafny::OptionalObjectAttributes, - OptionalObjectAttributes._typeDescriptor() - ); - } - - public static Owner Owner( - software.amazon.awssdk.services.s3.model.Owner nativeValue - ) { - Option> displayName; - displayName = - Objects.nonNull(nativeValue.displayName()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.displayName() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - Option> iD; - iD = - Objects.nonNull(nativeValue.id()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.id() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - return new Owner(displayName, iD); - } - public static PutObjectOutput PutObjectOutput(PutObjectResponse nativeValue) { Option> expiration; expiration = @@ -1857,32 +1766,6 @@ public static PutObjectRequest PutObjectRequest( ); } - public static RestoreStatus RestoreStatus( - software.amazon.awssdk.services.s3.model.RestoreStatus nativeValue - ) { - Option isRestoreInProgress; - isRestoreInProgress = - Objects.nonNull(nativeValue.isRestoreInProgress()) - ? Option.create_Some( - TypeDescriptor.BOOLEAN, - (nativeValue.isRestoreInProgress()) - ) - : Option.create_None(TypeDescriptor.BOOLEAN); - Option> restoreExpiryDate; - restoreExpiryDate = - Objects.nonNull(nativeValue.restoreExpiryDate()) - ? Option.create_Some( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR), - software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( - nativeValue.restoreExpiryDate() - ) - ) - : Option.create_None( - DafnySequence._typeDescriptor(TypeDescriptor.CHAR) - ); - return new RestoreStatus(isRestoreInProgress, restoreExpiryDate); - } - public static Error Error(InvalidObjectStateException nativeValue) { Option storageClass; storageClass = @@ -1997,25 +1880,6 @@ public static ChecksumMode ChecksumMode( } } - public static EncodingType EncodingType( - software.amazon.awssdk.services.s3.model.EncodingType nativeValue - ) { - switch (nativeValue) { - case URL: - { - return EncodingType.create(); - } - default: - { - throw new RuntimeException( - "Cannot convert " + - nativeValue + - " to software.amazon.cryptography.services.s3.internaldafny.types.EncodingType." - ); - } - } - } - public static IntelligentTieringAccessTier IntelligentTieringAccessTier( software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier nativeValue ) { @@ -2128,84 +1992,6 @@ public static ObjectLockMode ObjectLockMode( } } - public static ObjectStorageClass ObjectStorageClass( - software.amazon.awssdk.services.s3.model.ObjectStorageClass nativeValue - ) { - switch (nativeValue) { - case STANDARD: - { - return ObjectStorageClass.create_STANDARD(); - } - case REDUCED_REDUNDANCY: - { - return ObjectStorageClass.create_REDUCED__REDUNDANCY(); - } - case GLACIER: - { - return ObjectStorageClass.create_GLACIER(); - } - case STANDARD_IA: - { - return ObjectStorageClass.create_STANDARD__IA(); - } - case ONEZONE_IA: - { - return ObjectStorageClass.create_ONEZONE__IA(); - } - case INTELLIGENT_TIERING: - { - return ObjectStorageClass.create_INTELLIGENT__TIERING(); - } - case DEEP_ARCHIVE: - { - return ObjectStorageClass.create_DEEP__ARCHIVE(); - } - case OUTPOSTS: - { - return ObjectStorageClass.create_OUTPOSTS(); - } - case GLACIER_IR: - { - return ObjectStorageClass.create_GLACIER__IR(); - } - case SNOW: - { - return ObjectStorageClass.create_SNOW(); - } - case EXPRESS_ONEZONE: - { - return ObjectStorageClass.create_EXPRESS__ONEZONE(); - } - default: - { - throw new RuntimeException( - "Cannot convert " + - nativeValue + - " to software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass." - ); - } - } - } - - public static OptionalObjectAttributes OptionalObjectAttributes( - software.amazon.awssdk.services.s3.model.OptionalObjectAttributes nativeValue - ) { - switch (nativeValue) { - case RESTORE_STATUS: - { - return OptionalObjectAttributes.create(); - } - default: - { - throw new RuntimeException( - "Cannot convert " + - nativeValue + - " to software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes." - ); - } - } - } - public static ReplicationStatus ReplicationStatus( software.amazon.awssdk.services.s3.model.ReplicationStatus nativeValue ) { @@ -2381,14 +2167,6 @@ public static ChecksumMode ChecksumMode(String nativeValue) { ); } - public static EncodingType EncodingType(String nativeValue) { - return EncodingType( - software.amazon.awssdk.services.s3.model.EncodingType.fromValue( - nativeValue - ) - ); - } - public static IntelligentTieringAccessTier IntelligentTieringAccessTier( String nativeValue ) { @@ -2425,24 +2203,6 @@ public static ObjectLockMode ObjectLockMode(String nativeValue) { ); } - public static ObjectStorageClass ObjectStorageClass(String nativeValue) { - return ObjectStorageClass( - software.amazon.awssdk.services.s3.model.ObjectStorageClass.fromValue( - nativeValue - ) - ); - } - - public static OptionalObjectAttributes OptionalObjectAttributes( - String nativeValue - ) { - return OptionalObjectAttributes( - software.amazon.awssdk.services.s3.model.OptionalObjectAttributes.fromValue( - nativeValue - ) - ); - } - public static ReplicationStatus ReplicationStatus(String nativeValue) { return ReplicationStatus( software.amazon.awssdk.services.s3.model.ReplicationStatus.fromValue( diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java index c5f4c9bea..48b5eac0b 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java @@ -16,44 +16,34 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.ChecksumMode; -import software.amazon.awssdk.services.s3.model.CommonPrefix; import software.amazon.awssdk.services.s3.model.Delete; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.DeletedObject; -import software.amazon.awssdk.services.s3.model.EncodingType; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier; import software.amazon.awssdk.services.s3.model.InvalidObjectStateException; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.OptionalObjectAttributes; -import software.amazon.awssdk.services.s3.model.RestoreStatus; -import software.amazon.awssdk.services.s3.model.S3Error; -import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.ObjectCannedACL; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus; import software.amazon.awssdk.services.s3.model.ObjectLockMode; -import software.amazon.awssdk.services.s3.model.ObjectStorageClass; -import software.amazon.awssdk.services.s3.model.Owner; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.ReplicationStatus; import software.amazon.awssdk.services.s3.model.RequestCharged; import software.amazon.awssdk.services.s3.model.RequestPayer; +import software.amazon.awssdk.services.s3.model.S3Error; import software.amazon.awssdk.services.s3.model.S3Exception; import software.amazon.awssdk.services.s3.model.ServerSideEncryption; import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectOutput; import software.amazon.cryptography.services.s3.internaldafny.types.DeleteObjectsOutput; import software.amazon.cryptography.services.s3.internaldafny.types.Error; -import software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape; import software.amazon.cryptography.services.s3.internaldafny.types.Error_InvalidObjectState; import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchBucket; import software.amazon.cryptography.services.s3.internaldafny.types.Error_NoSuchKey; @@ -83,17 +73,6 @@ public static ChecksumAlgorithm ChecksumAlgorithm( return ChecksumAlgorithm.fromValue(dafnyValue.toString()); } - public static List ChecksumAlgorithmList( - DafnySequence< - ? extends software.amazon.cryptography.services.s3.internaldafny.types.ChecksumAlgorithm - > dafnyValue - ) { - return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( - dafnyValue, - software.amazon.cryptography.services.s3.internaldafny.ToNative::ChecksumAlgorithm - ); - } - public static ChecksumMode ChecksumMode( software.amazon.cryptography.services.s3.internaldafny.types.ChecksumMode dafnyValue ) { @@ -103,31 +82,6 @@ public static ChecksumMode ChecksumMode( return ChecksumMode.fromValue(dafnyValue.toString()); } - public static CommonPrefix CommonPrefix( - software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix dafnyValue - ) { - CommonPrefix.Builder builder = CommonPrefix.builder(); - if (dafnyValue.dtor_Prefix().is_Some()) { - builder.prefix( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_Prefix().dtor_value() - ) - ); - } - return builder.build(); - } - - public static List CommonPrefixList( - DafnySequence< - ? extends software.amazon.cryptography.services.s3.internaldafny.types.CommonPrefix - > dafnyValue - ) { - return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( - dafnyValue, - software.amazon.cryptography.services.s3.internaldafny.ToNative::CommonPrefix - ); - } - public static Delete Delete( software.amazon.cryptography.services.s3.internaldafny.types.Delete dafnyValue ) { @@ -315,15 +269,6 @@ public static DeleteObjectsRequest DeleteObjectsRequest( return builder.build(); } - public static EncodingType EncodingType( - software.amazon.cryptography.services.s3.internaldafny.types.EncodingType dafnyValue - ) { - if (dafnyValue.is_url()) { - return EncodingType.URL; - } - return EncodingType.fromValue(dafnyValue.toString()); - } - public static List Errors( DafnySequence< ? extends software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape @@ -752,15 +697,15 @@ public static IntelligentTieringAccessTier IntelligentTieringAccessTier( } public static Map Metadata( - DafnyMap< - ? extends DafnySequence, - ? extends DafnySequence - > dafnyValue + DafnyMap< + ? extends DafnySequence, + ? extends DafnySequence + > dafnyValue ) { return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToMap( - dafnyValue, - software.amazon.smithy.dafny.conversion.ToNative.Simple::String, - software.amazon.smithy.dafny.conversion.ToNative.Simple::String + dafnyValue, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String, + software.amazon.smithy.dafny.conversion.ToNative.Simple::String ); } @@ -845,86 +790,6 @@ public static ObjectLockMode ObjectLockMode( return ObjectLockMode.fromValue(dafnyValue.toString()); } - public static ObjectStorageClass ObjectStorageClass( - software.amazon.cryptography.services.s3.internaldafny.types.ObjectStorageClass dafnyValue - ) { - if (dafnyValue.is_STANDARD()) { - return ObjectStorageClass.STANDARD; - } - if (dafnyValue.is_REDUCED__REDUNDANCY()) { - return ObjectStorageClass.REDUCED_REDUNDANCY; - } - if (dafnyValue.is_GLACIER()) { - return ObjectStorageClass.GLACIER; - } - if (dafnyValue.is_STANDARD__IA()) { - return ObjectStorageClass.STANDARD_IA; - } - if (dafnyValue.is_ONEZONE__IA()) { - return ObjectStorageClass.ONEZONE_IA; - } - if (dafnyValue.is_INTELLIGENT__TIERING()) { - return ObjectStorageClass.INTELLIGENT_TIERING; - } - if (dafnyValue.is_DEEP__ARCHIVE()) { - return ObjectStorageClass.DEEP_ARCHIVE; - } - if (dafnyValue.is_OUTPOSTS()) { - return ObjectStorageClass.OUTPOSTS; - } - if (dafnyValue.is_GLACIER__IR()) { - return ObjectStorageClass.GLACIER_IR; - } - if (dafnyValue.is_SNOW()) { - return ObjectStorageClass.SNOW; - } - if (dafnyValue.is_EXPRESS__ONEZONE()) { - return ObjectStorageClass.EXPRESS_ONEZONE; - } - return ObjectStorageClass.fromValue(dafnyValue.toString()); - } - - public static OptionalObjectAttributes OptionalObjectAttributes( - software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes dafnyValue - ) { - if (dafnyValue.is_RESTORE__STATUS()) { - return OptionalObjectAttributes.RESTORE_STATUS; - } - return OptionalObjectAttributes.fromValue(dafnyValue.toString()); - } - - public static List OptionalObjectAttributesList( - DafnySequence< - ? extends software.amazon.cryptography.services.s3.internaldafny.types.OptionalObjectAttributes - > dafnyValue - ) { - return software.amazon.smithy.dafny.conversion.ToNative.Aggregate.GenericToList( - dafnyValue, - software.amazon.cryptography.services.s3.internaldafny.ToNative::OptionalObjectAttributes - ); - } - - public static Owner Owner( - software.amazon.cryptography.services.s3.internaldafny.types.Owner dafnyValue - ) { - Owner.Builder builder = Owner.builder(); - if (dafnyValue.dtor_DisplayName().is_Some()) { - builder.displayName( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_DisplayName().dtor_value() - ) - ); - } - if (dafnyValue.dtor_ID().is_Some()) { - builder.id( - software.amazon.smithy.dafny.conversion.ToNative.Simple.String( - dafnyValue.dtor_ID().dtor_value() - ) - ); - } - return builder.build(); - } - public static PutObjectResponse PutObjectOutput(PutObjectOutput dafnyValue) { PutObjectResponse.Builder builder = PutObjectResponse.builder(); if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { @@ -1307,25 +1172,6 @@ public static RequestPayer RequestPayer( return RequestPayer.fromValue(dafnyValue.toString()); } - public static RestoreStatus RestoreStatus( - software.amazon.cryptography.services.s3.internaldafny.types.RestoreStatus dafnyValue - ) { - RestoreStatus.Builder builder = RestoreStatus.builder(); - if (dafnyValue.dtor_IsRestoreInProgress().is_Some()) { - builder.isRestoreInProgress( - (dafnyValue.dtor_IsRestoreInProgress().dtor_value()) - ); - } - if (dafnyValue.dtor_RestoreExpiryDate().is_Some()) { - builder.restoreExpiryDate( - software.amazon.smithy.dafny.conversion.ToNative.Simple.Instant( - dafnyValue.dtor_RestoreExpiryDate().dtor_value() - ) - ); - } - return builder.build(); - } - public static ServerSideEncryption ServerSideEncryption( software.amazon.cryptography.services.s3.internaldafny.types.ServerSideEncryption dafnyValue ) { From 5ee063d856bf402afd749d4f75be15a961e6dd8b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 2 May 2025 16:39:30 -0700 Subject: [PATCH 40/72] both test models working for Java and Python! --- TestModels/Streaming/src/Chunker.dfy | 26 ++++++-- .../Streaming/src/SimpleStreamingImpl.dfy | 17 ++++++ .../test/SimpleStreamingImplTest.dfy | 9 ++- .../test/WrappedSimpleStreamingTest.dfy | 2 +- .../java/Streams/InputStreamAsProducer.java | 9 ++- .../internaldafny/extern/streams.py | 60 +++++++++++++++---- TestModels/dafny-dependencies/dafny | 2 +- 7 files changed, 101 insertions(+), 24 deletions(-) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 18882f995..102ce31d7 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -14,13 +14,14 @@ module {:options "--function-syntax:4"} Chunker { import opened Std.BulkActions import opened Std.Producers import opened Std.Consumers + import opened Std.Termination import opened StandardLibrary.Streams // "Batched Byte" type BB = Batched @AssumeCrossModuleTermination - class Chunker extends BulkAction> { + class Chunker extends BulkAction>, OutputterOfNewProducers { const chunkSize: CountingInteger var chunkBuffer: seq @@ -85,12 +86,23 @@ module {:options "--function-syntax:4"} Chunker { 0 } + ghost function MaxProduced(): TerminationMetric { + TMTop + } + + lemma AnyInputIsValid(history: seq<(BB, Producer)>, next: BB) + requires Valid() + requires Action().ValidHistory(history) + ensures Action().ValidInput(history, next) + {} + @IsolateAssertions method Invoke(i: BB) returns (o: Producer) requires Requires(i) modifies Modifies(i) decreases Decreases(i), 0 ensures Ensures(i, o) + ensures OutputFresh(o) { assert Valid(); var input := new SeqReader([i]); @@ -123,6 +135,7 @@ module {:options "--function-syntax:4"} Chunker { ensures input.NewProduced() == NewInputs() ensures |input.NewProduced()| == |output.NewInputs()| ensures output.NewInputs() == NewOutputs() + ensures forall o <- NewOutputs() :: OutputFresh(o) { assert Valid(); @@ -149,9 +162,10 @@ module {:options "--function-syntax:4"} Chunker { } chunkBuffer := chunkBuffer + batchWriter.elements; + print "chunkBuffer: ", chunkBuffer, "\n"; var chunks, leftover := Chunkify(chunkBuffer); - var chunkBuffer := leftover; + chunkBuffer := leftover; var outputProducer: Producer; match batchWriter.state { @@ -161,6 +175,7 @@ module {:options "--function-syntax:4"} Chunker { if !more && 0 < |chunkBuffer| { // To make it more interesting, produce an error if outputChunks is non empty? chunks := chunks + Seq.Reverse(chunkBuffer); + chunkBuffer := []; } outputProducer := new BatchReader(chunks); } @@ -260,7 +275,7 @@ module {:options "--function-syntax:4"} Chunker { } function ContentLength(): Option { - original.ContentLength() + None } predicate Replayable() { @@ -277,8 +292,8 @@ module {:options "--function-syntax:4"} Chunker { var chunker := new Chunker(chunkSize); var chunkerTotalProof := new ChunkerTotalProof(chunker); var originalProducer := original.Reader(); - var chunkerStream := new MappedProducer(originalProducer, chunker, chunkerTotalProof); - // p := new FlattenedProducer(chunkerStream); + var chunkerStream := new MappedProducerOfNewProducers(originalProducer, chunker); + p := new FlattenedProducer(chunkerStream); } } @@ -295,4 +310,5 @@ module {:options "--function-syntax:4"} Chunker { // TODO: Actually compute the binary [12 as uint8, 34, 56] } + } \ No newline at end of file diff --git a/TestModels/Streaming/src/SimpleStreamingImpl.dfy b/TestModels/Streaming/src/SimpleStreamingImpl.dfy index 7478a7274..bb6d9a366 100644 --- a/TestModels/Streaming/src/SimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/SimpleStreamingImpl.dfy @@ -82,4 +82,21 @@ module {:options "/functionSyntax:4" } SimpleStreamingImpl refines AbstractSimpl return Success(ChunksOutput(bytesOut := chunkerStream)); } + + method PrintProduced(p: Producers.Producer) + requires p.Valid() + modifies p.Repr + { + while true + invariant fresh(p.Repr - old(p.Repr)) + invariant p.Valid() + decreases p.Decreasing() + { + var next := p.Next(); + if next.None? { break; } + var value := next.value; + + print value, "\n"; + } + } } diff --git a/TestModels/Streaming/test/SimpleStreamingImplTest.dfy b/TestModels/Streaming/test/SimpleStreamingImplTest.dfy index cbf3d2fd8..6d8922330 100644 --- a/TestModels/Streaming/test/SimpleStreamingImplTest.dfy +++ b/TestModels/Streaming/test/SimpleStreamingImplTest.dfy @@ -13,10 +13,14 @@ module SimpleStreamingImplTest { import opened StandardLibrary.Streams import opened SimpleStreamingTypes import opened Wrappers + + import opened Chunker + method{:test} TestClient(){ var client :- expect SimpleStreaming.SimpleStreaming(); TestCountBits(client); TestBinaryOf(client); + TestChunks(client); } method TestCountBits(client: ISimpleStreamingClient) @@ -30,7 +34,7 @@ module SimpleStreamingImplTest { var ret :- expect client.CountBits(input); - expect ret.sum == 7; + expect ret.sum == 7, ret.sum; } method TestBinaryOf(client: ISimpleStreamingClient) @@ -68,6 +72,7 @@ module SimpleStreamingImplTest { var reader := ret.bytesOut.Reader(); reader.ForEach(collector, collectorTotalProof); - expect collector.elements == [0x2, 0x1, 0x0, 0x5, 0x4, 0x3, 0x7, 0x6]; + expect collector.elements == [0x2, 0x1, 0x0, 0x5, 0x4, 0x3, 0x7, 0x6], collector.elements; } + } \ No newline at end of file diff --git a/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy b/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy index ef144c1ee..60de02262 100644 --- a/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy +++ b/TestModels/Streaming/test/WrappedSimpleStreamingTest.dfy @@ -7,7 +7,7 @@ module WrappedSimpleTypesStringTest { import WrappedSimpleStreamingService import SimpleStreamingImplTest import opened Wrappers - method{:test} TestCountBits() { + method{:test} TestWrappedClient() { var client :- expect WrappedSimpleStreamingService.WrappedSimpleStreaming(); SimpleStreamingImplTest.TestCountBits(client); SimpleStreamingImplTest.TestBinaryOf(client); diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java index 8ffd1aa41..5b54d6883 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java @@ -24,6 +24,7 @@ public class InputStreamAsProducer implements Producer> { private final InputStream inputStream; private long totalRead; private Function ioExceptionWrapper; + private boolean producedEOI = false; private static final TypeDescriptor T_TD = TypeDescriptor.BYTE; private final TypeDescriptor E_TD; @@ -53,8 +54,12 @@ public Option> Next() { try { int value = inputStream.read(); if (value == -1) { - // TODO: EOI - return Option.create_None(BATCHED_TD); + if (producedEOI) { + return Option.create_None(BATCHED_TD); + } else { + producedEOI = true; + return Option.create_Some(BATCHED_TD, Batched.create_EndOfInput(T_TD, E_TD)); + } } else { Batched batched = Batched.create_BatchValue(T_TD, E_TD, (byte)value); return Option.create_Some(BATCHED_TD, batched); diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index 953d13782..dcfec5321 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -2,7 +2,7 @@ from _dafny import Seq, Array as DafnyArray from smithy_python.interfaces.blobs import ByteStream from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream -from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_EndOfInput, BatchReader +from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_BatchValue, Batched_EndOfInput, BatchReader from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer from smithy_dafny_standard_library.internaldafny.generated.Std_Producers import Producer from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None @@ -18,8 +18,20 @@ class DafnyDataStreamAsByteStream(ByteStream): def __init__(self, data_stream): self.data_stream = data_stream self.reader = data_stream.Reader() + self.leftover = Option_None() def read(self, size: int = -1) -> bytes: + result = bytes() + if size == 0: + return result + + if self.leftover.is_Some: + if self.leftover.value.is_BatchValue: + result += bytes([self.leftover.value.value]) + if size > 0: + size -= 1 + self.leftover = Option_None() + if size == -1: writer = BatchSeqWriter() writer.ctor__() @@ -27,13 +39,18 @@ def read(self, size: int = -1) -> bytes: else: writer = BatchArrayWriter() writer.ctor__(DafnyArray(None, size)) - self.reader.Fill(writer) + self.leftover = self.reader.Fill(writer) + # TODO: Check for errors. Fine to ignore EOI though. - # print(f"****** {writer.Values()} ******") - return bytes(writer.Values()) + result += bytes(writer.Values()) + return result def tell(self) -> int: - return self.reader.ProducedCount() + if self.leftover.is_Some: + result = self.reader.ProducedCount() - 1 + else: + result = self.reader.ProducedCount() + return result def seek(self, offset, whence=0): # TODO: check whether invalid offsets must raise errors @@ -49,12 +66,12 @@ def seek(self, offset, whence=0): if new_position > self.reader.ProducedCount(): consumer = IgnoreNConsumer() consumer.ctor__(new_position - self.reader.ProducedCount()) - self.reader.Fill(consumer) + self.leftover = self.reader.Fill(consumer) elif new_position < self.reader.ProducedCount(): self.reader = self.data_stream.Reader() consumer = IgnoreNConsumer() consumer.ctor__(new_position) - self.reader.Fill(consumer) + self.leftover = self.reader.Fill(consumer) class StreamingBlobAsDafnyDataStream(DataStream): @@ -78,22 +95,30 @@ class StreamingBlobAsDafnyProducer(Producer): def __init__(self, streaming_blob): self.streaming_blob = streaming_blob + self.emitted_eoi = False def Next(self): return Producer.Next(self) def Invoke(self, _) -> Option: + if self.emitted_eoi: + return Option_None() + # TODO: error handling next = self.streaming_blob.read(1) if next: - return Option_Some(Batch_Value(next)) + return Option_Some(Batched_BatchValue(next[0])) else: - return Option_None() + self.emitted_eoi = True + return Option_Some(Batched_EndOfInput()) def ProducedCount(self): return self.streaming_blob.position def ForEach(self, consumer): + if self.emitted_eoi: + return + # TODO: error handling while True: next = self.streaming_blob.read(4096) @@ -104,12 +129,21 @@ def ForEach(self, consumer): batch.ForEach(consumer) consumer.Accept(Batched_EndOfInput()) + self.emitted_eoi = True def Fill(self, consumer): + if self.emitted_eoi: + return Option_None() + # TODO: error handling size = consumer.Capacity() next = self.streaming_blob.read(size) - batch = BatchReader() - batch.ctor__(Seq(next)) - batch.Fill(consumer) - # TODO: EOI + if not next: + self.emitted_eoi = True + eoi = Batched_EndOfInput() + if not consumer.Accept(eoi): + return Option_Some(eoi) + else: + batch = BatchReader() + batch.ctor__(Seq(next)) + return batch.Fill(consumer) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index d40b30596..e3d746369 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit d40b30596c21322290ac9adc260c8af5c27a0220 +Subproject commit e3d7463690375d94655f5e15d19335002c0a6fc1 From 4d6191567afe3eae8f979051f8dc990a36a6b92f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 3 May 2025 10:39:09 -0700 Subject: [PATCH 41/72] Better version of Fill --- .../java/Streams/InputStreamAsProducer.java | 5 ++- .../internaldafny/extern/streams.py | 34 +++++-------------- TestModels/dafny-dependencies/dafny | 2 +- 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java index 5b54d6883..e1f1eba82 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java @@ -75,7 +75,7 @@ public void ForEach(IConsumer> consumer) { } @Override - public Option> Fill(Consumer> consumer) { + public void Fill(Consumer> consumer) { if (consumer instanceof BatchArrayWriter) { BatchArrayWriter writer = (BatchArrayWriter) consumer; int n = writer.Capacity().dtor_value().intValueExact(); @@ -94,10 +94,9 @@ public Option> Fill(Consumer> consumer) { } catch (IOException e) { writer.Accept(Batched.create_BatchError(T_TD, E_TD, ioExceptionWrapper.apply(e))); } - return Option.create_None(BATCHED_TD); } - return __default.DefaultFill(BATCHED_TD, this, consumer); + __default.DefaultFill(BATCHED_TD, this, consumer); } @Override diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py index dcfec5321..e6e05eafa 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py @@ -18,20 +18,8 @@ class DafnyDataStreamAsByteStream(ByteStream): def __init__(self, data_stream): self.data_stream = data_stream self.reader = data_stream.Reader() - self.leftover = Option_None() def read(self, size: int = -1) -> bytes: - result = bytes() - if size == 0: - return result - - if self.leftover.is_Some: - if self.leftover.value.is_BatchValue: - result += bytes([self.leftover.value.value]) - if size > 0: - size -= 1 - self.leftover = Option_None() - if size == -1: writer = BatchSeqWriter() writer.ctor__() @@ -39,18 +27,13 @@ def read(self, size: int = -1) -> bytes: else: writer = BatchArrayWriter() writer.ctor__(DafnyArray(None, size)) - self.leftover = self.reader.Fill(writer) + self.reader.Fill(writer) # TODO: Check for errors. Fine to ignore EOI though. - result += bytes(writer.Values()) - return result + return bytes(writer.Values()) def tell(self) -> int: - if self.leftover.is_Some: - result = self.reader.ProducedCount() - 1 - else: - result = self.reader.ProducedCount() - return result + return self.reader.ProducedCount() def seek(self, offset, whence=0): # TODO: check whether invalid offsets must raise errors @@ -66,12 +49,12 @@ def seek(self, offset, whence=0): if new_position > self.reader.ProducedCount(): consumer = IgnoreNConsumer() consumer.ctor__(new_position - self.reader.ProducedCount()) - self.leftover = self.reader.Fill(consumer) + self.reader.Fill(consumer) elif new_position < self.reader.ProducedCount(): self.reader = self.data_stream.Reader() consumer = IgnoreNConsumer() consumer.ctor__(new_position) - self.leftover = self.reader.Fill(consumer) + self.reader.Fill(consumer) class StreamingBlobAsDafnyDataStream(DataStream): @@ -133,7 +116,7 @@ def ForEach(self, consumer): def Fill(self, consumer): if self.emitted_eoi: - return Option_None() + return # TODO: error handling size = consumer.Capacity() @@ -141,9 +124,8 @@ def Fill(self, consumer): if not next: self.emitted_eoi = True eoi = Batched_EndOfInput() - if not consumer.Accept(eoi): - return Option_Some(eoi) + consumer.Accept(eoi) else: batch = BatchReader() batch.ctor__(Seq(next)) - return batch.Fill(consumer) + batch.Fill(consumer) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index e3d746369..db83eb24f 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit e3d7463690375d94655f5e15d19335002c0a6fc1 +Subproject commit db83eb24f923adb987f55843e7ab6a2e64363cbd From 8097e49e16c02eafe16f0f825bb27918b0954808 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 4 May 2025 08:55:31 -0700 Subject: [PATCH 42/72] Improve java patch --- .../services/s3/internaldafny/ToDafny.java | 102 ++++++++---------- .../services/s3/internaldafny/ToNative.java | 72 +++++-------- 2 files changed, 72 insertions(+), 102 deletions(-) diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java index 4be3cef8d..3746cdacc 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java @@ -718,15 +718,14 @@ public static GetObjectOutput GetObjectOutput(ResponseInputStream> sSEKMSKeyId; sSEKMSKeyId = -// Objects.nonNull(nativeValue.sseKMSKeyId()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( -// nativeValue.sseKMSKeyId() -// ) -// ) -// : - Option.create_None( + Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; @@ -1255,28 +1254,26 @@ public static PutObjectOutput PutObjectOutput(PutObjectResponse nativeValue) { ); Option> sSEKMSKeyId; sSEKMSKeyId = -// Objects.nonNull(nativeValue.sseKMSKeyId()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( -// nativeValue.sseKMSKeyId() -// ) -// ) -// : - Option.create_None( + Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option> sSEKMSEncryptionContext; sSEKMSEncryptionContext = -// Objects.nonNull(nativeValue.sseKMSEncryptionContext()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( -// nativeValue.sseKMSEncryptionContext() -// ) -// ) -// : - Option.create_None( + Objects.nonNull(nativeValue.ssekmsEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ssekmsEncryptionContext() + ) + ) + : Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; @@ -1326,20 +1323,9 @@ public static PutObjectRequest PutObjectRequest( ) : Option.create_None(ObjectCannedACL._typeDescriptor()); Option> body = Option.create_Some( - null, - new RequestBodyAsDataStream( - null, nativeBody, Error::create_Opaque )); -// body = -// Objects.nonNull(nativeValue.body()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.BYTE), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.ByteSequence( -// nativeValue.body().asByteArray() -// ) -// ) -// : Option.create_None( -// DafnySequence._typeDescriptor(TypeDescriptor.BYTE) -// ); + null, + new RequestBodyAsDataStream( + null, nativeBody, Error::create_Opaque )); DafnySequence bucket; bucket = software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( @@ -1631,28 +1617,26 @@ public static PutObjectRequest PutObjectRequest( ); Option> sSEKMSKeyId; sSEKMSKeyId = -// Objects.nonNull(nativeValue.sseKMSKeyId()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( -// nativeValue.sseKMSKeyId() -// ) -// ) -// : - Option.create_None( + Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option> sSEKMSEncryptionContext; sSEKMSEncryptionContext = -// Objects.nonNull(nativeValue.sseKMSEncryptionContext()) -// ? Option.create_Some( -// DafnySequence._typeDescriptor(TypeDescriptor.CHAR), -// software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( -// nativeValue.sseKMSEncryptionContext() -// ) -// ) -// : - Option.create_None( + Objects.nonNull(nativeValue.ssekmsEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( + nativeValue.ssekmsEncryptionContext() + ) + ) + : Option.create_None( DafnySequence._typeDescriptor(TypeDescriptor.CHAR) ); Option bucketKeyEnabled; diff --git a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java index 48b5eac0b..433c3f16c 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +++ b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java @@ -324,13 +324,6 @@ public static GetObjectResponse GetObjectOutput(GetObjectOutput dafnyValue) { ) ); } -// if (dafnyValue.dtor_Body().is_Some()) { -// builder.body( -// SdkBytes.fromByteArray( -// (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) -// ) -// ); -// } if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { builder.bucketKeyEnabled( (dafnyValue.dtor_BucketKeyEnabled().dtor_value()) @@ -511,11 +504,11 @@ public static GetObjectResponse GetObjectOutput(GetObjectOutput dafnyValue) { ); } if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { -// builder.sseKMSKeyId( -// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( -// dafnyValue.dtor_SSEKMSKeyId().dtor_value() -// ) -// ); + builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); } if (dafnyValue.dtor_StorageClass().is_Some()) { builder.storageClass( @@ -866,18 +859,18 @@ public static PutObjectResponse PutObjectOutput(PutObjectOutput dafnyValue) { ); } if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { -// builder.sseKMSEncryptionContext( -// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( -// dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() -// ) -// ); + builder.ssekmsEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); } if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { -// builder.sseKMSKeyId( -// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( -// dafnyValue.dtor_SSEKMSKeyId().dtor_value() -// ) -// ); + builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); } if (dafnyValue.dtor_VersionId().is_Some()) { builder.versionId( @@ -896,13 +889,6 @@ public static PutObjectRequest PutObjectRequest( if (dafnyValue.dtor_ACL().is_Some()) { builder.acl(ToNative.ObjectCannedACL(dafnyValue.dtor_ACL().dtor_value())); } -// if (dafnyValue.dtor_Body().is_Some()) { -// builder.body( -// SdkBytes.fromByteArray( -// (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) -// ) -// ); -// } builder.bucket( software.amazon.smithy.dafny.conversion.ToNative.Simple.String( dafnyValue.dtor_Bucket() @@ -1097,20 +1083,20 @@ public static PutObjectRequest PutObjectRequest( ) ); } -// if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { -// builder.sseKMSEncryptionContext( -// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( -// dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() -// ) -// ); -// } -// if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { -// builder.sseKMSKeyId( -// software.amazon.smithy.dafny.conversion.ToNative.Simple.String( -// dafnyValue.dtor_SSEKMSKeyId().dtor_value() -// ) -// ); -// } + if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { + builder.ssekmsEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { + builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) + ); + } if (dafnyValue.dtor_StorageClass().is_Some()) { builder.storageClass( ToNative.StorageClass(dafnyValue.dtor_StorageClass().dtor_value()) From 8f442c90417626cce4750ed3d605622dccb4a26d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 20 May 2025 11:29:27 -0700 Subject: [PATCH 43/72] Generate Java files for Streaming --- .../Streaming/runtimes/java/build.gradle.kts | 80 +++++++++++++++++++ .../Dafny/simple/streaming/__default.java | 5 ++ .../simple/streaming/types/__default.java | 5 ++ .../internaldafny/wrapped/__default.java | 32 ++++++++ TestModels/dafny-dependencies/dafny | 2 +- 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 TestModels/Streaming/runtimes/java/build.gradle.kts create mode 100644 TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/__default.java create mode 100644 TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/types/__default.java create mode 100644 TestModels/Streaming/runtimes/java/src/test/java/simple/streaming/internaldafny/wrapped/__default.java diff --git a/TestModels/Streaming/runtimes/java/build.gradle.kts b/TestModels/Streaming/runtimes/java/build.gradle.kts new file mode 100644 index 000000000..1aafd2631 --- /dev/null +++ b/TestModels/Streaming/runtimes/java/build.gradle.kts @@ -0,0 +1,80 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import java.io.File +import java.io.FileInputStream +import java.util.Properties +import java.net.URI +import javax.annotation.Nullable + +tasks.wrapper { + gradleVersion = "7.6" +} + +plugins { + `java-library` + `maven-publish` +} + +var props = Properties().apply { + load(FileInputStream(File(rootProject.rootDir, "../../project.properties"))) +} +var dafnyVersion = props.getProperty("dafnyVersion") + +group = "simple.streaming" +version = "1.0-SNAPSHOT" +description = "SimpleStreaming" + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(8)) + sourceSets["main"].java { + srcDir("src/main/java") + srcDir("src/main/dafny-generated") + srcDir("src/main/smithy-generated") + } + sourceSets["test"].java { + srcDir("src/test/java") + srcDir("src/test/dafny-generated") + srcDir("src/test/smithy-generated") + } +} + +repositories { + mavenCentral() + mavenLocal() +} + +dependencies { + implementation("org.dafny:DafnyRuntime:${dafnyVersion}") + implementation("software.amazon.smithy.dafny:conversion:0.1.1") + implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") + testImplementation("org.testng:testng:7.5") +} + +publishing { + publications.create("mavenLocal") { + groupId = group as String? + artifactId = description + from(components["java"]) + } + publications.create("maven") { + groupId = group as String? + artifactId = description + from(components["java"]) + } + repositories { mavenLocal() } +} + +tasks.withType() { + options.encoding = "UTF-8" +} + +tasks { + register("runTests", JavaExec::class.java) { + mainClass.set("TestsFromDafny") + classpath = sourceSets["test"].runtimeClasspath + } +} + +tasks.named("test") { + useTestNG() +} diff --git a/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/__default.java b/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/__default.java new file mode 100644 index 000000000..2a5ee30cc --- /dev/null +++ b/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/__default.java @@ -0,0 +1,5 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package simple.streaming.internaldafny; + +public class __default extends _ExternBase___default {} diff --git a/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/types/__default.java b/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/types/__default.java new file mode 100644 index 000000000..0272702b3 --- /dev/null +++ b/TestModels/Streaming/runtimes/java/src/main/java/Dafny/simple/streaming/types/__default.java @@ -0,0 +1,5 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package simple.streaming.internaldafny.types; + +public class __default extends _ExternBase___default {} diff --git a/TestModels/Streaming/runtimes/java/src/test/java/simple/streaming/internaldafny/wrapped/__default.java b/TestModels/Streaming/runtimes/java/src/test/java/simple/streaming/internaldafny/wrapped/__default.java new file mode 100644 index 000000000..ad8953902 --- /dev/null +++ b/TestModels/Streaming/runtimes/java/src/test/java/simple/streaming/internaldafny/wrapped/__default.java @@ -0,0 +1,32 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package simple.streaming.internaldafny.wrapped; + +import Wrappers_Compile.Result; +import simple.streaming.SimpleStreaming; +import simple.streaming.ToNative; +import simple.streaming.internaldafny.types.Error; +import simple.streaming.internaldafny.types.ISimpleStreamingClient; +import simple.streaming.internaldafny.types.SimpleStreamingConfig; +import simple.streaming.wrapped.TestSimpleStreaming; + +public class __default extends _ExternBase___default { + + public static Result WrappedSimpleStreaming( + SimpleStreamingConfig config + ) { + simple.streaming.model.SimpleStreamingConfig wrappedConfig = + ToNative.SimpleStreamingConfig(config); + simple.streaming.SimpleStreaming impl = SimpleStreaming + .builder() + .SimpleStreamingConfig(wrappedConfig) + .build(); + TestSimpleStreaming wrappedClient = TestSimpleStreaming + .builder() + .impl(impl) + .build(); + return simple.streaming.internaldafny.__default.CreateSuccessOfClient( + wrappedClient + ); + } +} diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index db83eb24f..cdcfd29f4 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit db83eb24f923adb987f55843e7ab6a2e64363cbd +Subproject commit cdcfd29f43e7934f9c6d4364e44d5d4f91ae6175 From bce55c36474aa8e248c36a0163a021138d8d1ede Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 21 Jun 2025 15:13:46 -0700 Subject: [PATCH 44/72] Interface update --- TestModels/Streaming/src/Chunker.dfy | 12 +++++++----- .../dafny-dependencies/StandardLibrary/Makefile | 2 +- TestModels/dafny-dependencies/dafny | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 102ce31d7..14b2dc568 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -107,9 +107,10 @@ module {:options "--function-syntax:4"} Chunker { assert Valid(); var input := new SeqReader([i]); var output := new SeqWriter(); - var outputTotalProof := new SeqWriterTotalActionProof(output); + ghost var thisTotalProof := new ChunkerTotalProof(this); + ghost var outputTotalProof := new SeqWriterTotalActionProof(output); label before: - BulkInvoke(input, output, outputTotalProof); + Map(input, output, thisTotalProof, outputTotalProof); assert |output.values| == 1; o := output.values[0]; assert Seq.Last(output.Inputs()) == o; @@ -118,9 +119,10 @@ module {:options "--function-syntax:4"} Chunker { @ResourceLimit("1e9") @IsolateAssertions - method BulkInvoke(input: Producer, - output: IConsumer>, - outputTotalProof: TotalActionProof, ()>) + method Map(input: Producer, + output: IConsumer>, + ghost thisTotalProof: TotalActionProof>, + ghost outputTotalProof: TotalActionProof, ()>) requires Valid() requires input.Valid() requires output.Valid() diff --git a/TestModels/dafny-dependencies/StandardLibrary/Makefile b/TestModels/dafny-dependencies/StandardLibrary/Makefile index f3984ba7f..f4d1c2d32 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/Makefile +++ b/TestModels/dafny-dependencies/StandardLibrary/Makefile @@ -107,7 +107,7 @@ transpile_implementation: --unicode-char:false \ --function-syntax:3 \ --output $(OUT) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ./bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(DAFNY_OPTIONS) \ $(DAFNY_OTHER_FILES) \ $(TRANSPILE_MODULE_NAME) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index cdcfd29f4..92e04df08 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit cdcfd29f43e7934f9c6d4364e44d5d4f91ae6175 +Subproject commit 92e04df0862d65baf46b85794e0b50eed600c9fe From 11687ab42d11e37eeabbca679f0e4db75aab880c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 23 Jun 2025 15:51:22 -0700 Subject: [PATCH 45/72] Enable Java in testing --- TestModels/dafny-dependencies/dafny | 2 +- .../software/amazon/polymorph/smithyjava/JavaTestModels.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 92e04df08..1233be2d2 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 92e04df0862d65baf46b85794e0b50eed600c9fe +Subproject commit 1233be2d26997531eccae1f12ad21f8180115e4d diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java index 8cf0ec995..c12aeb65c 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java @@ -43,15 +43,12 @@ class JavaTestModels extends TestModelTest { DISABLED_TESTS.add("SimpleTypes/SimpleShort"); DISABLED_TESTS.add("SimpleTypes/SimpleString"); DISABLED_TESTS.add("SimpleTypes/SimpleTimestamp"); - DISABLED_TESTS.add("Streaming"); DISABLED_TESTS.add("Union"); DISABLED_TESTS.add("aws-sdks/kms-lite"); DISABLED_TESTS.add("aws-sdks/sqs"); DISABLED_TESTS.add("aws-sdks/sqs-via-cli"); //TODO: Add support for Recursive shapes. DISABLED_TESTS.add("RecursiveShape"); - // S3 is not yet supported - DISABLED_TESTS.add("aws-sdks/s3"); //TODO: https://github.com/smithy-lang/smithy-dafny/issues/599 DISABLED_TESTS.add("Positional"); From df3feb33bfe10395e4a6b2a7a65226cbb2ba1409 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 23 Jun 2025 16:05:13 -0700 Subject: [PATCH 46/72] Delete submodule --- examples/tutorial/smithy-dafny | 1 - 1 file changed, 1 deletion(-) delete mode 160000 examples/tutorial/smithy-dafny diff --git a/examples/tutorial/smithy-dafny b/examples/tutorial/smithy-dafny deleted file mode 160000 index 21ecddbb4..000000000 --- a/examples/tutorial/smithy-dafny +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 21ecddbb4eaf719d1720e650bdc8738bdb9e78c6 From 7d5654c6f2ef0f9a16fd16fe2ced66f7e3888243 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 23 Jun 2025 16:06:40 -0700 Subject: [PATCH 47/72] Delete content --- examples/tutorial/Makefile | 0 examples/tutorial/model/weather.smithy | 136 ------------------------- examples/tutorial/smithy-build.json | 10 -- 3 files changed, 146 deletions(-) delete mode 100644 examples/tutorial/Makefile delete mode 100644 examples/tutorial/model/weather.smithy delete mode 100644 examples/tutorial/smithy-build.json diff --git a/examples/tutorial/Makefile b/examples/tutorial/Makefile deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/tutorial/model/weather.smithy b/examples/tutorial/model/weather.smithy deleted file mode 100644 index df1cf637e..000000000 --- a/examples/tutorial/model/weather.smithy +++ /dev/null @@ -1,136 +0,0 @@ -$version: "2" - -namespace example.weather - -/// Provides weather forecasts. -@paginated(inputToken: "nextToken", outputToken: "nextToken", pageSize: "pageSize") -service Weather { - version: "2006-03-01" - resources: [ - City - ] - operations: [ - GetCurrentTime - ] -} - -resource City { - identifiers: { cityId: CityId } - properties: { coordinates: CityCoordinates } - read: GetCity - list: ListCities - resources: [ - Forecast - ] -} - -resource Forecast { - identifiers: { cityId: CityId } - properties: { chanceOfRain: Float } - read: GetForecast -} - -// "pattern" is a trait. -@pattern("^[A-Za-z0-9 ]+$") -string CityId - -@readonly -operation GetCity { - input := for City { - // "cityId" provides the identifier for the resource and - // has to be marked as required. - @required - $cityId - } - - output := for City { - // "required" is used on output to indicate if the service - // will always provide a value for the member. - @required - @notProperty - name: String - - @required - $coordinates - } - - errors: [ - NoSuchResource - ] -} - -// This structure is nested within GetCityOutput. -structure CityCoordinates { - @required - latitude: Float - - @required - longitude: Float -} - -// "error" is a trait that is used to specialize -// a structure as an error. -@error("client") -structure NoSuchResource { - @required - resourceType: String -} - -// The paginated trait indicates that the operation may -// return truncated results. -@readonly -@paginated(items: "items") -operation ListCities { - input := { - nextToken: String - pageSize: Integer - } - - output := { - nextToken: String - - @required - items: CitySummaries - } -} - -// CitySummaries is a list of CitySummary structures. -list CitySummaries { - member: CitySummary -} - -// CitySummary contains a reference to a City. -@references([ - { - resource: City - } -]) -structure CitySummary { - @required - cityId: CityId - - @required - name: String -} - -@readonly -operation GetCurrentTime { - output := { - @required - time: Timestamp - } -} - -@readonly -operation GetForecast { - input := for Forecast { - // "cityId" provides the only identifier for the resource since - // a Forecast doesn't have its own. - @required - $cityId - } - - output := for Forecast { - $chanceOfRain - } -} diff --git a/examples/tutorial/smithy-build.json b/examples/tutorial/smithy-build.json deleted file mode 100644 index 942839164..000000000 --- a/examples/tutorial/smithy-build.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "version": "1.0", - "sources": ["model"], - "maven": { - "dependencies": [ - ], - "repositories": [ - ] - } -} \ No newline at end of file From 44d63bf2d5d869e24472413595aa960cbafe117e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 23 Jun 2025 16:29:02 -0700 Subject: [PATCH 48/72] m --- SmithyDafnyMakefile.mk | 3 +++ smithy-templates.json | 11 ----------- 2 files changed, 3 insertions(+), 11 deletions(-) delete mode 100644 smithy-templates.json diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index d5014bcaf..d6859878f 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -83,6 +83,9 @@ ENABLE_EXTERN_PROCESSING?= ENFORCE_DETERMINISM_OPTION := $(shell cd $(CODEGEN_CLI_ROOT); \ ./../gradlew run -q --args="if-dafny-at-least --dafny-version 4.8 --text --enforce-determinism") +# TODO: Remove parameter if possible +USE_DAFNY_STANDARD_LIBRARIES := 1 + # Verify the entire project verify:Z3_PROCESSES=$(shell echo $$(( $(CORES) >= 3 ? 2 : 1 ))) verify:DAFNY_PROCESSES=$(shell echo $$(( ($(CORES) - 1 ) / ($(CORES) >= 3 ? 2 : 1)))) diff --git a/smithy-templates.json b/smithy-templates.json deleted file mode 100644 index 87f09bc56..000000000 --- a/smithy-templates.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "smithy-dafny", - "templates": { - "polymorph-tutorial": { - "documentation": "Basic Polymorphing tutorial.", - "path": "examples/tutorial", - "include": [ - ] - } - } -} \ No newline at end of file From 0897353547283fa4253afe11e7306f7a91d60dbd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 24 Jun 2025 13:09:02 -0700 Subject: [PATCH 49/72] Add 4.10.0 --- .github/workflows/pull.yml | 4 ++-- .../DafnyStandardLibraries-smithy-dafny-subset.toml | 3 --- TestModels/dafny-dependencies/dafny | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 566c62f73..99f70da48 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -10,10 +10,10 @@ jobs: steps: - name: Populate Dafny versions list id: populate-dafny-versions-list - run: echo "dafny-versions-list=['4.7.0','4.9.1']" >> $GITHUB_OUTPUT + run: echo "dafny-versions-list=['4.7.0','4.9.1','4.10.0']" >> $GITHUB_OUTPUT - name: Populate Dafny versions list for "only new versions" languages (Python) id: populate-only-new-dafny-versions-list - run: echo "only-new-dafny-versions-list=['4.9.1']" >> $GITHUB_OUTPUT + run: echo "only-new-dafny-versions-list=['4.9.1','4.10.0']" >> $GITHUB_OUTPUT outputs: dafny-version-list: ${{ steps.populate-dafny-versions-list.outputs.dafny-versions-list }} only-new-dafny-version-list: ${{ steps.populate-only-new-dafny-versions-list.outputs.only-new-dafny-versions-list }} diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml index 1965faacf..723dc2f65 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml +++ b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml @@ -34,9 +34,6 @@ unicode-char = false # TODO: Required because this file isn't named dfyconfig.toml :P allow-warnings = true - -hidden-no-verify = true - # Options important for sustainable development # of the libraries themselves. resource-limit = 1000000 diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 1233be2d2..81dd55397 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 1233be2d26997531eccae1f12ad21f8180115e4d +Subproject commit 81dd553971d0a42f51acfb8745d0b59a02685004 From 9fe3c8ff9f320758455c615ba41a9abca366b4e1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 14:16:17 -0700 Subject: [PATCH 50/72] m --- SmithyDafnyMakefile.mk | 3 - TestModels/Streaming/Makefile | 5 +- .../Streaming/Model/SimpleStreamingTypes.dfy | 2 +- TestModels/Streaming/src/Index.dfy | 2 +- .../src/WrappedSimpleStreamingImpl.dfy | 2 +- .../StandardLibrary/src/Index.dfy | 3 +- .../StreamingSupport/Makefile | 10 ++ .../Model/streamingsupport.smithy | 11 ++ .../runtimes/java/build.gradle.kts | 91 ++++++++++++ .../java/Streams/ConsumerAsSubscriber.java | 0 .../Streams/DataStreamAsAsyncRequestBody.java | 0 .../java/Streams/DataStreamAsRequestBody.java | 0 .../java/Streams/InputStreamAsDataStream.java | 0 .../java/Streams/InputStreamAsProducer.java | 0 .../java/Streams/ProducerAsInputStream.java | 0 .../java/Streams/RequestBodyAsDataStream.java | 0 .../runtimes/python/.gitignore | 8 ++ .../runtimes/python/pyproject.toml | 22 +++ .../__init__.py | 10 ++ .../internaldafny/extern/streams.py | 131 ++++++++++++++++++ .../StreamingSupport/runtimes/python/tox.ini | 13 ++ .../StreamingSupport/src/Index.dfy | 4 + .../src/Streams.dfy | 0 .../src/dfyconfig.toml | 2 +- 24 files changed, 308 insertions(+), 11 deletions(-) create mode 100644 TestModels/dafny-dependencies/StreamingSupport/Makefile create mode 100644 TestModels/dafny-dependencies/StreamingSupport/Model/streamingsupport.smithy create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java (100%) create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/python/.gitignore create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/__init__.py create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py create mode 100644 TestModels/dafny-dependencies/StreamingSupport/runtimes/python/tox.ini create mode 100644 TestModels/dafny-dependencies/StreamingSupport/src/Index.dfy rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/src/Streams.dfy (100%) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/src/dfyconfig.toml (67%) diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index d6859878f..d5014bcaf 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -83,9 +83,6 @@ ENABLE_EXTERN_PROCESSING?= ENFORCE_DETERMINISM_OPTION := $(shell cd $(CODEGEN_CLI_ROOT); \ ./../gradlew run -q --args="if-dafny-at-least --dafny-version 4.8 --text --enforce-determinism") -# TODO: Remove parameter if possible -USE_DAFNY_STANDARD_LIBRARIES := 1 - # Verify the entire project verify:Z3_PROCESSES=$(shell echo $$(( $(CORES) >= 3 ? 2 : 1 ))) verify:DAFNY_PROCESSES=$(shell echo $$(( ($(CORES) - 1 ) / ($(CORES) >= 3 ? 2 : 1)))) diff --git a/TestModels/Streaming/Makefile b/TestModels/Streaming/Makefile index 0aca2882f..d3eb09653 100644 --- a/TestModels/Streaming/Makefile +++ b/TestModels/Streaming/Makefile @@ -20,8 +20,9 @@ SERVICE_DEPS_SimpleStreaming := SMITHY_DEPS=dafny-dependencies/Model/traits.smithy -# This project has no dependencies -# DEPENDENT-MODELS:= +PROJECT_DEPENDENCIES := + +PROJECT_INDEX := # Python diff --git a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy index c44e78368..c33a33e9b 100644 --- a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy +++ b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // Do not modify this file. This file is machine generated, and any changes to it will be overwritten. include "../../dafny-dependencies/StandardLibrary/src/Index.dfy" -module SimpleStreamingTypes +module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes { import opened Wrappers import opened StandardLibrary.UInt diff --git a/TestModels/Streaming/src/Index.dfy b/TestModels/Streaming/src/Index.dfy index 882a2e30c..6884eb35c 100644 --- a/TestModels/Streaming/src/Index.dfy +++ b/TestModels/Streaming/src/Index.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "SimpleStreamingImpl.dfy" -module {:extern "simple.streaming.internaldafny"} SimpleStreaming refines AbstractSimpleStreamingService { +module SimpleStreaming refines AbstractSimpleStreamingService { import Operations = SimpleStreamingImpl function method DefaultSimpleStreamingConfig(): SimpleStreamingConfig { diff --git a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy index 58bf8aed5..00827803f 100644 --- a/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy +++ b/TestModels/Streaming/src/WrappedSimpleStreamingImpl.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/SimpleStreamingTypesWrapped.dfy" -module {:extern "simple.streaming.internaldafny.wrapped"} WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { +module WrappedSimpleStreamingService refines WrappedAbstractSimpleStreamingService { import WrappedService = SimpleStreaming function method WrappedDefaultSimpleStreamingConfig(): SimpleStreamingConfig { SimpleStreamingConfig diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy b/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy index 55fab1f33..e9c061a82 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy +++ b/TestModels/dafny-dependencies/StandardLibrary/src/Index.dfy @@ -4,5 +4,4 @@ include "./WrappersInterop.dfy" include "./StandardLibrary.dfy" include "./UInt.dfy" -include "./UTF8.dfy" -include "./Streams.dfy" \ No newline at end of file +include "./UTF8.dfy" \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/Makefile b/TestModels/dafny-dependencies/StreamingSupport/Makefile new file mode 100644 index 000000000..1ce5cc44c --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/Makefile @@ -0,0 +1,10 @@ + +CORES=2 + +USE_DAFNY_STANDARD_LIBRARIES=1 + +include ../../SharedMakefile.mk + +transpile_java: | transpile_implementation_java transpile_test_java + +transpile_python: | transpile_implementation_python transpile_test_python \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/Model/streamingsupport.smithy b/TestModels/dafny-dependencies/StreamingSupport/Model/streamingsupport.smithy new file mode 100644 index 000000000..0054bef6e --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/Model/streamingsupport.smithy @@ -0,0 +1,11 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +namespace aws.polymorph + +// Dummy service just to keep polymorph happy. +service StreamingSupport { + version: "2021-11-01", + resources: [], + operations: [], + errors: [], +} diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts new file mode 100644 index 000000000..e8c54bdcd --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts @@ -0,0 +1,91 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import java.io.File +import java.io.FileInputStream +import java.util.Properties +import java.net.URI +import javax.annotation.Nullable + +plugins { + `java-library` + `maven-publish` +} + +var props = Properties().apply { + load(FileInputStream(File(rootProject.rootDir, "../../project.properties"))) +} +var dafnyVersion = props.getProperty("dafnyVersion") + +group = "software.amazon.cryptography" +version = "1.0-SNAPSHOT" +description = "StreamingSupport" + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(8)) + sourceSets["main"].java { + srcDir("src/main/java") + srcDir("src/main/dafny-generated") + } + sourceSets["test"].java { + srcDir("src/test/dafny-generated") + } +} + +var caUrl: URI? = null +@Nullable +val caUrlStr: String? = System.getenv("CODEARTIFACT_URL_JAVA_CONVERSION") +if (!caUrlStr.isNullOrBlank()) { + caUrl = URI.create(caUrlStr) +} + +var caPassword: String? = null +@Nullable +val caPasswordString: String? = System.getenv("CODEARTIFACT_AUTH_TOKEN") +if (!caPasswordString.isNullOrBlank()) { + caPassword = caPasswordString +} + +repositories { + mavenCentral() + mavenLocal() + if (caUrl != null && caPassword != null) { + maven { + name = "CodeArtifact" + url = caUrl!! + credentials { + username = "aws" + password = caPassword!! + } + } + } +} + +dependencies { + implementation("org.dafny:DafnyRuntime:${dafnyVersion}") + implementation("software.amazon.smithy.dafny:conversion:0.1.1") + api("software.amazon.awssdk:sdk-core:2.28.28") +} +publishing { + publications.create("mavenLocal") { + groupId = group as String? + artifactId = description + from(components["java"]) + } + publications.create("maven") { + groupId = "software.amazon.cryptography" + artifactId = "StandardLibrary" + from(components["java"]) + } + repositories { mavenLocal() } +} + +tasks.withType() { + options.encoding = "UTF-8" +} + +tasks { + register("runTests", JavaExec::class.java) { + mainClass.set("TestsFromDafny") + classpath = sourceSets["test"].runtimeClasspath + } +} diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/ConsumerAsSubscriber.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/DataStreamAsAsyncRequestBody.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/DataStreamAsRequestBody.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/InputStreamAsDataStream.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/InputStreamAsProducer.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/ProducerAsInputStream.java diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java rename to TestModels/dafny-dependencies/StreamingSupport/runtimes/java/src/main/java/Streams/RequestBodyAsDataStream.java diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/.gitignore b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/.gitignore new file mode 100644 index 000000000..3c31a7deb --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/.gitignore @@ -0,0 +1,8 @@ +# Python Artifacts +src/**.egg-info/ +.pytest_cache +.tox +build +poetry.lock +**/poetry.lock +dist \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml new file mode 100644 index 000000000..451bf0607 --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "smithy_dafny_streaming_support" +version = "0.1.0" +description = "" +authors = ["AWS "] +packages = [ + { include = "smithy_dafny_streaming_support", from = "src" }, +] +# Include all of the following .gitignored files in package distributions, +# even though it is not included in version control +include = ["**/internaldafny/generated/*.py"] + +[tool.poetry.dependencies] +python = "^3.11.0" +DafnyRuntimePython = "^4.7.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" + +[build-system] +requires = ["poetry-core<2.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/__init__.py b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/__init__.py new file mode 100644 index 000000000..12ef3d38a --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Boilerplate code for a Python package with internal Dafny code + +# Initialize generated Dafny +from .internaldafny.generated import module_ + +# Initialize externs +from .internaldafny import extern \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py new file mode 100644 index 000000000..e6e05eafa --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py @@ -0,0 +1,131 @@ + +from _dafny import Seq, Array as DafnyArray +from smithy_python.interfaces.blobs import ByteStream +from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream +from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_BatchValue, Batched_EndOfInput, BatchReader +from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer +from smithy_dafny_standard_library.internaldafny.generated.Std_Producers import Producer +from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None + +# Adaptor classes for wrapping up Python-native types as their +# corresponding Dafny interfaces, and vice-versa. +# These are the equivalent of type conversions, +# but avoiding having to load all data into memory at once. + +class DafnyDataStreamAsByteStream(ByteStream): + """Wrapper class adapting a Dafny DataStream as a native ByteStream.""" + + def __init__(self, data_stream): + self.data_stream = data_stream + self.reader = data_stream.Reader() + + def read(self, size: int = -1) -> bytes: + if size == -1: + writer = BatchSeqWriter() + writer.ctor__() + self.reader.ForEach(writer) + else: + writer = BatchArrayWriter() + writer.ctor__(DafnyArray(None, size)) + self.reader.Fill(writer) + + # TODO: Check for errors. Fine to ignore EOI though. + return bytes(writer.Values()) + + def tell(self) -> int: + return self.reader.ProducedCount() + + def seek(self, offset, whence=0): + # TODO: check whether invalid offsets must raise errors + # TODO: Need to -1 to account for EndOfInput + match whence: + case 0: + new_position = offset + case 1: + new_position = self.reader.ProducedCount() + offset + case 2: + new_position = self.data_stream.ContentLength().value + offset + + if new_position > self.reader.ProducedCount(): + consumer = IgnoreNConsumer() + consumer.ctor__(new_position - self.reader.ProducedCount()) + self.reader.Fill(consumer) + elif new_position < self.reader.ProducedCount(): + self.reader = self.data_stream.Reader() + consumer = IgnoreNConsumer() + consumer.ctor__(new_position) + self.reader.Fill(consumer) + + +class StreamingBlobAsDafnyDataStream(DataStream): + def __init__(self, streaming_blob): + self.streaming_blob = streaming_blob + self.read = False + + def Replayable(self): + False + + def Reader(self): + if self.read: + raise Exception("StreamingBlobAsDafnyDataStream.Reader() called twice") + self.read = True + return StreamingBlobAsDafnyProducer(self.streaming_blob) + + +# TODO: Missing some methods like Remaining() +class StreamingBlobAsDafnyProducer(Producer): + """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" + + def __init__(self, streaming_blob): + self.streaming_blob = streaming_blob + self.emitted_eoi = False + + def Next(self): + return Producer.Next(self) + + def Invoke(self, _) -> Option: + if self.emitted_eoi: + return Option_None() + + # TODO: error handling + next = self.streaming_blob.read(1) + if next: + return Option_Some(Batched_BatchValue(next[0])) + else: + self.emitted_eoi = True + return Option_Some(Batched_EndOfInput()) + + def ProducedCount(self): + return self.streaming_blob.position + + def ForEach(self, consumer): + if self.emitted_eoi: + return + + # TODO: error handling + while True: + next = self.streaming_blob.read(4096) + if not next: + break + batch = BatchReader() + batch.ctor__(Seq(next)) + batch.ForEach(consumer) + + consumer.Accept(Batched_EndOfInput()) + self.emitted_eoi = True + + def Fill(self, consumer): + if self.emitted_eoi: + return + + # TODO: error handling + size = consumer.Capacity() + next = self.streaming_blob.read(size) + if not next: + self.emitted_eoi = True + eoi = Batched_EndOfInput() + consumer.Accept(eoi) + else: + batch = BatchReader() + batch.ctor__(Seq(next)) + batch.Fill(consumer) diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/tox.ini b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/tox.ini new file mode 100644 index 000000000..b626d2496 --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/tox.ini @@ -0,0 +1,13 @@ +[tox] +isolated_build = True +envlist = + py{311,312} + +[testenv] +skip_install = true +allowlist_externals = poetry +commands_pre = + poetry lock + poetry install +commands = + poetry run pytest -s -v test/ \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/src/Index.dfy b/TestModels/dafny-dependencies/StreamingSupport/src/Index.dfy new file mode 100644 index 000000000..c57699c15 --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/src/Index.dfy @@ -0,0 +1,4 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "./Streams.dfy" \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy b/TestModels/dafny-dependencies/StreamingSupport/src/Streams.dfy similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/src/Streams.dfy rename to TestModels/dafny-dependencies/StreamingSupport/src/Streams.dfy diff --git a/TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml b/TestModels/dafny-dependencies/StreamingSupport/src/dfyconfig.toml similarity index 67% rename from TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml rename to TestModels/dafny-dependencies/StreamingSupport/src/dfyconfig.toml index da8b8f871..c1b0a5915 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/src/dfyconfig.toml +++ b/TestModels/dafny-dependencies/StreamingSupport/src/dfyconfig.toml @@ -6,5 +6,5 @@ unicode-char = false function-syntax = 3 library = [ - "../bin/DafnyStandardLibraries-smithy-dafny-subset.doo" + "../../StandardLibrary/bin/DafnyStandardLibraries-smithy-dafny-subset.doo" ] \ No newline at end of file From c04b81e4e762f625852fcb58205ec5df88fa8811 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 16:41:01 -0700 Subject: [PATCH 51/72] m --- SmithyDafnyMakefile.mk | 12 ++++-- ...StandardLibraries-smithy-dafny-subset.toml | 2 + .../software/amazon/polymorph/CodegenCli.java | 20 +++++---- .../amazon/polymorph/CodegenEngine.java | 40 ++++++++---------- .../smithydafny/DafnyApiCodegen.java | 16 ++++---- .../codegen/DafnyClientCodegenPlugin.java | 2 +- .../DafnyClientCodegenPluginSettings.java | 41 +++++++++++-------- .../smithydafny/DafnyApiCodegenTest.java | 2 +- 8 files changed, 73 insertions(+), 62 deletions(-) diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index d5014bcaf..47964a70e 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -359,7 +359,8 @@ polymorph_code_gen: _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -386,7 +387,8 @@ polymorph_dafny: _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_dafny: _polymorph dafny: polymorph_dafny verify @@ -406,7 +408,8 @@ polymorph_dotnet: _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_dotnet: _polymorph # Generates java code for all namespaces in this project @@ -424,7 +427,8 @@ polymorph_java: _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_java: _polymorph # Generates python code for all namespaces in this project diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml index 723dc2f65..7b7b61f73 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml +++ b/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml @@ -34,6 +34,8 @@ unicode-char = false # TODO: Required because this file isn't named dfyconfig.toml :P allow-warnings = true +hidden-no-verify = true + # Options important for sustainable development # of the libraries themselves. resource-limit = 1000000 diff --git a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java index b0368f882..c3b2706c9 100644 --- a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java +++ b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java @@ -139,7 +139,8 @@ public static void main(String[] args) { .withAwsSdkStyle(cliArguments.awsSdkStyle) .withDafnyVersion(cliArguments.dafnyVersion) .withUpdatePatchFiles(cliArguments.updatePatchFiles) - .withGenerationAspects(cliArguments.generationAspects); + .withGenerationAspects(cliArguments.generationAspects) + .withIncludeDafnyFiles(cliArguments.includeDafnyFiles); // Rust currently generates all code for all dependencies at once, // and the makefile structure makes it very difficult to avoid passing --local-service-test // when we don't actually want it for --aws-sdk style projects. @@ -159,9 +160,6 @@ public static void main(String[] args) { cliArguments.javaAwsSdkVersion.ifPresent( engineBuilder::withJavaAwsSdkVersion ); - cliArguments.includeDafnyFile.ifPresent( - engineBuilder::withIncludeDafnyFile - ); cliArguments.libraryName.ifPresent(engineBuilder::withLibraryName); cliArguments.patchFilesDir.ifPresent(engineBuilder::withPatchFilesDir); final CodegenEngine engine = engineBuilder.build(); @@ -519,7 +517,7 @@ private record CliArguments( Optional javaAwsSdkVersion, DafnyVersion dafnyVersion, Optional propertiesFile, - Optional includeDafnyFile, + List includeDafnyFiles, boolean awsSdkStyle, boolean localServiceTest, Optional patchFilesDir, @@ -664,9 +662,13 @@ static Optional parse(String[] args) throws ParseException { .ofNullable(commandLine.getOptionValue("properties-file")) .map(Paths::get); - Optional includeDafnyFile = Optional - .ofNullable(commandLine.getOptionValue("include-dafny")) - .map(Paths::get); + final String[] includeDafnyFileOptions = Optional + .ofNullable(commandLine.getOptionValues("include-dafny")) + .orElse(new String[0]); + final List includeDafnyFiles = Arrays + .stream(includeDafnyFileOptions) + .map(Paths::get) + .toList(); Optional patchFilesDir = Optional .ofNullable(commandLine.getOptionValue("patch-files-dir")) @@ -702,7 +704,7 @@ static Optional parse(String[] args) throws ParseException { javaAwsSdkVersion, dafnyVersion, propertiesFile, - includeDafnyFile, + includeDafnyFiles, awsSdkStyle, localServiceTest, patchFilesDir, diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java index 74c902cab..3d408be67 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java @@ -100,7 +100,7 @@ public class CodegenEngine { private final boolean updatePatchFiles; // refactor this to only be required if generating Java private final AwsSdkVersion javaAwsSdkVersion; - private final Optional includeDafnyFile; + private final List includeDafnyFiles; private final boolean awsSdkStyle; private final boolean localServiceTest; private final Set generationAspects; @@ -126,7 +126,7 @@ private CodegenEngine( final DafnyVersion dafnyVersion, final Optional propertiesFile, final AwsSdkVersion javaAwsSdkVersion, - final Optional includeDafnyFile, + final List includeDafnyFiles, final boolean awsSdkStyle, final boolean localServiceTest, final Set generationAspects, @@ -145,7 +145,7 @@ private CodegenEngine( this.dafnyVersion = dafnyVersion; this.propertiesFile = propertiesFile; this.javaAwsSdkVersion = javaAwsSdkVersion; - this.includeDafnyFile = includeDafnyFile; + this.includeDafnyFiles = includeDafnyFiles; this.awsSdkStyle = awsSdkStyle; this.localServiceTest = localServiceTest; this.generationAspects = generationAspects; @@ -242,12 +242,12 @@ private void generateProjectPropertiesFile(final Path outputPath) private void generateDafny(final Path outputDir) { // Validated by builder, but check again - assert this.includeDafnyFile.isPresent(); + assert !this.includeDafnyFiles.isEmpty(); final DafnyApiCodegen dafnyApiCodegen = new DafnyApiCodegen( model, serviceShape, outputDir, - this.includeDafnyFile.get(), + this.includeDafnyFiles, this.dependentModelPaths, this.awsSdkStyle ); @@ -364,7 +364,7 @@ private void generateDafnySkeleton(Path outputDir) { model, serviceShape, outputDir, - this.includeDafnyFile.get(), + this.includeDafnyFiles, this.dependentModelPaths, this.awsSdkStyle ); @@ -1014,18 +1014,17 @@ private static CommandResult runCommand(Path workingDir, String... args) { } private Path standardLibraryPath() { - final Path includeDafnyFile = - this.includeDafnyFile.orElseThrow(() -> - new IllegalStateException( - "includeDafnyFile required when generating additional aspects (--generate)" - ) + if (this.includeDafnyFiles.isEmpty()) { + throw new IllegalStateException( + "includeDafnyFile required when generating additional aspects (--generate)" ); + } - // Assumes that includeDafnyFile is at StandardLibrary/src/Index.dfy + // Assumes that includeDafnyFiles.get(0) is at StandardLibrary/src/Index.dfy // TODO be smarter about finding the StandardLibrary path return libraryRoot .resolve("runtimes/net") - .relativize(includeDafnyFile.resolve("../..")); + .relativize(includeDafnyFiles.get(0).resolve("../..")); } private void writeTemplatedFile( @@ -1062,7 +1061,7 @@ public static class Builder { private DafnyVersion dafnyVersion; private Path propertiesFile; private AwsSdkVersion javaAwsSdkVersion = AwsSdkVersion.V2; - private Path includeDafnyFile; + private List includeDafnyFiles; private boolean awsSdkStyle = false; private boolean localServiceTest = false; private Set generationAspects = Collections.emptySet(); @@ -1175,8 +1174,8 @@ public Builder withJavaAwsSdkVersion( /** * Sets a file to be included in the generated Dafny code. */ - public Builder withIncludeDafnyFile(final Path includeDafnyFile) { - this.includeDafnyFile = includeDafnyFile; + public Builder withIncludeDafnyFiles(final List includeDafnyFiles) { + this.includeDafnyFiles = includeDafnyFiles; return this; } @@ -1324,15 +1323,12 @@ public CodegenEngine build() { if ( targetLangOutputDirs.containsKey(TargetLanguage.DAFNY) && - this.includeDafnyFile == null + this.includeDafnyFiles.isEmpty() ) { throw new IllegalStateException( - "includeDafnyFile is required when generating Dafny code" + "includeDafnyFiles is required when generating Dafny code" ); } - final Optional includeDafnyFile = Optional - .ofNullable(this.includeDafnyFile) - .map(path -> path.toAbsolutePath().normalize()); if (this.awsSdkStyle && this.localServiceTest) { throw new IllegalStateException( @@ -1365,7 +1361,7 @@ public CodegenEngine build() { dafnyVersion, propertiesFile, javaAwsSdkVersion, - includeDafnyFile, + includeDafnyFiles, this.awsSdkStyle, this.localServiceTest, this.generationAspects, diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java index fda0789a4..eba95ad43 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyApiCodegen.java @@ -31,7 +31,7 @@ public class DafnyApiCodegen { private final ServiceShape serviceShape; private final DafnyNameResolver nameResolver; private final Path outputDir; - private final Path includeDafnyFile; + private final List includeDafnyFiles; private static final Logger LOGGER = LoggerFactory.getLogger( DafnyApiCodegen.class ); @@ -52,14 +52,14 @@ public DafnyApiCodegen( final Model model, final ServiceShape serviceShape, final Path outputDir, - final Path includeDafnyFile, + final List includeDafnyFiles, final Path[] dependentModelPaths, final boolean awsSdkRequest ) { this.model = model; this.serviceShape = serviceShape; this.outputDir = outputDir; - this.includeDafnyFile = includeDafnyFile; + this.includeDafnyFiles = includeDafnyFiles; this.nameResolver = new DafnyNameResolver( model, @@ -106,7 +106,7 @@ public Map generate() { .of( Stream .concat( - Stream.of(outputDir.relativize(includeDafnyFile)), + includeDafnyFiles.stream().map(outputDir::relativize), nameResolver .dependentModels() // nameResolve.dependentModels() filters dependentModelPaths @@ -237,9 +237,11 @@ public Map generateWrappedAbstractServiceModule( .of( Stream .concat( - Stream.of( - outputDir.relativize(includeDafnyFile), - outputDafny.relativize(outputDir.resolve("../src/Index.dfy")) + Stream.concat( + includeDafnyFiles.stream().map(outputDir::relativize), + Stream.of( + outputDafny.relativize(outputDir.resolve("../src/Index.dfy")) + ) ), nameResolver .dependentModels() diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPlugin.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPlugin.java index 5a1cc18d6..b63e3e91b 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPlugin.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPlugin.java @@ -85,7 +85,7 @@ public void execute(PluginContext context) { .withPropertiesFile(propertiesFile) .withTargetLangOutputDirs(outputDirs) .withAwsSdkStyle(true) // this plugin only generates AWS SDK-style code - .withIncludeDafnyFile(settings.includeDafnyFile) + .withIncludeDafnyFiles(settings.includeDafnyFiles) .withGenerationAspects( EnumSet.of( CodegenEngine.GenerationAspect.PROJECT_FILES, diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java index 4d44f39a1..172f05bd7 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java @@ -53,7 +53,7 @@ class DafnyClientCodegenPluginSettings { *

* TODO: replace this with something cleaner */ - public final Path includeDafnyFile; + public final List includeDafnyFiles; /** * The Dafny version to generate code compatible with. @@ -67,13 +67,13 @@ private DafnyClientCodegenPluginSettings( final DafnyClientCodegenEdition edition, final ShapeId serviceId, final Set targetLanguages, - final Path includeDafnyFile, + final List includeDafnyFiles, final DafnyVersion dafnyVersion ) { this.edition = edition; this.serviceId = serviceId; this.targetLanguages = targetLanguages; - this.includeDafnyFile = includeDafnyFile; + this.includeDafnyFiles = includeDafnyFiles; this.dafnyVersion = dafnyVersion; } @@ -128,20 +128,25 @@ static Optional fromObject( final Optional buildRoot = findSmithyBuildJson(manifest.getBaseDir()) .map(p -> p.getParent()); - final String includeDafnyFileStr = node - .expectStringMember("includeDafnyFile") - .getValue(); - final Path includeDafnyFile = Path.of(includeDafnyFileStr); - final Path includeDafnyFileNormalized = buildRoot.isPresent() && - !includeDafnyFile.isAbsolute() - ? buildRoot.get().resolve(includeDafnyFile).toAbsolutePath().normalize() - : includeDafnyFile; - if (Files.notExists(includeDafnyFileNormalized)) { - LOGGER.warn( - "Generated Dafny code may not compile because the includeDafnyFile could not be found: {}", - includeDafnyFileNormalized - ); - } + final List includeDafnyFilesNormalized = node + .expectArrayMember("includeDafnyFiles") + .getElementsAs(StringNode.class) + .stream() + .map(pathNode -> { + final Path includeDafnyFile = Path.of(pathNode.getValue()); + final Path includeDafnyFileNormalized = buildRoot.isPresent() && + !includeDafnyFile.isAbsolute() + ? buildRoot.get().resolve(includeDafnyFile).toAbsolutePath().normalize() + : includeDafnyFile; + if (Files.notExists(includeDafnyFileNormalized)) { + LOGGER.warn( + "Generated Dafny code may not compile because the includeDafnyFile could not be found: {}", + includeDafnyFileNormalized + ); + } + return includeDafnyFileNormalized; + }) + .toList(); // This is now optional since we can get it from dafny itself final DafnyVersion dafnyVersionString = node @@ -155,7 +160,7 @@ static Optional fromObject( edition, serviceId, targetLanguages, - includeDafnyFileNormalized, + includeDafnyFilesNormalized, dafnyVersionString ) ); diff --git a/codegen/smithy-dafny-codegen/src/test/java/software/amazon/polymorph/smithydafny/DafnyApiCodegenTest.java b/codegen/smithy-dafny-codegen/src/test/java/software/amazon/polymorph/smithydafny/DafnyApiCodegenTest.java index fbf8428b8..d19c98e96 100644 --- a/codegen/smithy-dafny-codegen/src/test/java/software/amazon/polymorph/smithydafny/DafnyApiCodegenTest.java +++ b/codegen/smithy-dafny-codegen/src/test/java/software/amazon/polymorph/smithydafny/DafnyApiCodegenTest.java @@ -35,7 +35,7 @@ private static DafnyApiCodegen setupCodegen( model, serviceShape, Path.of(""), - Path.of(""), + List.of(), new Path[0], false ); From 0e4fce63f7d3e5286713d50c5b09c3908a2a83be Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 16:43:30 -0700 Subject: [PATCH 52/72] m --- TestModels/SharedMakefile.mk | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index 2a1269bcb..d867998f5 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -16,7 +16,8 @@ include $(SMITHY_DAFNY_ROOT)/SmithyDafnyMakefile.mk _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -24,7 +25,8 @@ _polymorph_code_gen: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtime _polymorph_code_gen: _polymorph _polymorph_code_gen: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_code_gen: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) _polymorph_code_gen: _polymorph_wrapped @@ -45,17 +47,19 @@ polymorph_dafny: _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy _polymorph_dafny: _polymorph _polymorph_dafny: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_dafny: _polymorph_wrapped _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_java: _polymorph _polymorph_java: OUTPUT_JAVA_WRAPPED=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: _polymorph_wrapped @@ -63,7 +67,8 @@ _polymorph_java: _polymorph_wrapped _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) _polymorph_dotnet: _polymorph _polymorph_dotnet: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) @@ -77,7 +82,8 @@ _polymorph_python: _polymorph_wrapped _polymorph_rust: OUTPUT_RUST_WRAPPED=--output-rust $(LIBRARY_ROOT)/runtimes/rust _polymorph_rust: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/../StreamingSupport/src/Index.dfy, ) # For several TestModels we've just manually written the code generation target, # So we just want to ensure we can transpile and pass the tests in CI. # For those, make polymorph_rust should just be a no-op. From 9eafed94c680dbeebab91f61303c6af080c662cc Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 16:58:40 -0700 Subject: [PATCH 53/72] m --- TestModels/SharedMakefile.mk | 8 +++----- .../main/java/software/amazon/polymorph/CodegenCli.java | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index d867998f5..c196cea5f 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -25,8 +25,7 @@ _polymorph_code_gen: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtime _polymorph_code_gen: _polymorph _polymorph_code_gen: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy _polymorph_code_gen: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) _polymorph_code_gen: _polymorph_wrapped @@ -47,12 +46,11 @@ polymorph_dafny: _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy _polymorph_dafny: _polymorph _polymorph_dafny: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy _polymorph_dafny: _polymorph_wrapped _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated diff --git a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java index c3b2706c9..151e70d3b 100644 --- a/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java +++ b/codegen/smithy-dafny-codegen-cli/src/main/java/software/amazon/polymorph/CodegenCli.java @@ -333,7 +333,8 @@ private static Options getCliOptionsForBuild() { .builder() .longOpt("include-dafny") .desc(" files to be include in the generated Dafny") - .hasArg() + .hasArgs() + .valueSeparator(',') .build() ) .addOption( From 073ea8d48d9c879d630e81c480be6130304db53d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 17:08:17 -0700 Subject: [PATCH 54/72] m --- TestModels/SharedMakefile.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index c196cea5f..2c7ed5e9e 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -46,7 +46,8 @@ polymorph_dafny: _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ - --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy + --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ + --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy _polymorph_dafny: _polymorph _polymorph_dafny: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ From eb8bd46c329d36034a1580e23838767894501495 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Jun 2025 21:58:45 -0700 Subject: [PATCH 55/72] m --- .../Streaming/Model/SimpleStreamingTypes.dfy | 1 + .../Streaming/runtimes/python/pyproject.toml | 1 + .../internaldafny/extern/streams.py | 131 ------------------ .../internaldafny/extern/streams.py | 2 +- .../AwsSdkToDafnyShapeVisitor.java | 2 +- .../DafnyToAwsSdkShapeVisitor.java | 2 +- .../DafnyToLocalServiceShapeVisitor.java | 2 +- .../LocalServiceToDafnyShapeVisitor.java | 2 +- 8 files changed, 7 insertions(+), 136 deletions(-) delete mode 100644 TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py diff --git a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy index c33a33e9b..aed5b4754 100644 --- a/TestModels/Streaming/Model/SimpleStreamingTypes.dfy +++ b/TestModels/Streaming/Model/SimpleStreamingTypes.dfy @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // Do not modify this file. This file is machine generated, and any changes to it will be overwritten. include "../../dafny-dependencies/StandardLibrary/src/Index.dfy" +include "../../dafny-dependencies/StreamingSupport/src/Index.dfy" module {:extern "simple.streaming.internaldafny.types" } SimpleStreamingTypes { import opened Wrappers diff --git a/TestModels/Streaming/runtimes/python/pyproject.toml b/TestModels/Streaming/runtimes/python/pyproject.toml index b2a6f3351..c36504276 100644 --- a/TestModels/Streaming/runtimes/python/pyproject.toml +++ b/TestModels/Streaming/runtimes/python/pyproject.toml @@ -16,6 +16,7 @@ python = "^3.11.0" smithy-python = { path = "../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} smithy-dafny-standard-library = {path = "../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +smithy-dafny-streaming-support = {path = "../../../dafny-dependencies/StreamingSupport/runtimes/python", develop = false} DafnyRuntimePython = "^4.4.0" [tool.poetry.group.test.dependencies] diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py deleted file mode 100644 index e6e05eafa..000000000 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/extern/streams.py +++ /dev/null @@ -1,131 +0,0 @@ - -from _dafny import Seq, Array as DafnyArray -from smithy_python.interfaces.blobs import ByteStream -from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream -from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_BatchValue, Batched_EndOfInput, BatchReader -from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer -from smithy_dafny_standard_library.internaldafny.generated.Std_Producers import Producer -from smithy_dafny_standard_library.internaldafny.generated.Std_Wrappers import Option, Option_Some, Option_None - -# Adaptor classes for wrapping up Python-native types as their -# corresponding Dafny interfaces, and vice-versa. -# These are the equivalent of type conversions, -# but avoiding having to load all data into memory at once. - -class DafnyDataStreamAsByteStream(ByteStream): - """Wrapper class adapting a Dafny DataStream as a native ByteStream.""" - - def __init__(self, data_stream): - self.data_stream = data_stream - self.reader = data_stream.Reader() - - def read(self, size: int = -1) -> bytes: - if size == -1: - writer = BatchSeqWriter() - writer.ctor__() - self.reader.ForEach(writer) - else: - writer = BatchArrayWriter() - writer.ctor__(DafnyArray(None, size)) - self.reader.Fill(writer) - - # TODO: Check for errors. Fine to ignore EOI though. - return bytes(writer.Values()) - - def tell(self) -> int: - return self.reader.ProducedCount() - - def seek(self, offset, whence=0): - # TODO: check whether invalid offsets must raise errors - # TODO: Need to -1 to account for EndOfInput - match whence: - case 0: - new_position = offset - case 1: - new_position = self.reader.ProducedCount() + offset - case 2: - new_position = self.data_stream.ContentLength().value + offset - - if new_position > self.reader.ProducedCount(): - consumer = IgnoreNConsumer() - consumer.ctor__(new_position - self.reader.ProducedCount()) - self.reader.Fill(consumer) - elif new_position < self.reader.ProducedCount(): - self.reader = self.data_stream.Reader() - consumer = IgnoreNConsumer() - consumer.ctor__(new_position) - self.reader.Fill(consumer) - - -class StreamingBlobAsDafnyDataStream(DataStream): - def __init__(self, streaming_blob): - self.streaming_blob = streaming_blob - self.read = False - - def Replayable(self): - False - - def Reader(self): - if self.read: - raise Exception("StreamingBlobAsDafnyDataStream.Reader() called twice") - self.read = True - return StreamingBlobAsDafnyProducer(self.streaming_blob) - - -# TODO: Missing some methods like Remaining() -class StreamingBlobAsDafnyProducer(Producer): - """Wrapper class adapting a native StreamingBlob as a Dafny DataStream.""" - - def __init__(self, streaming_blob): - self.streaming_blob = streaming_blob - self.emitted_eoi = False - - def Next(self): - return Producer.Next(self) - - def Invoke(self, _) -> Option: - if self.emitted_eoi: - return Option_None() - - # TODO: error handling - next = self.streaming_blob.read(1) - if next: - return Option_Some(Batched_BatchValue(next[0])) - else: - self.emitted_eoi = True - return Option_Some(Batched_EndOfInput()) - - def ProducedCount(self): - return self.streaming_blob.position - - def ForEach(self, consumer): - if self.emitted_eoi: - return - - # TODO: error handling - while True: - next = self.streaming_blob.read(4096) - if not next: - break - batch = BatchReader() - batch.ctor__(Seq(next)) - batch.ForEach(consumer) - - consumer.Accept(Batched_EndOfInput()) - self.emitted_eoi = True - - def Fill(self, consumer): - if self.emitted_eoi: - return - - # TODO: error handling - size = consumer.Capacity() - next = self.streaming_blob.read(size) - if not next: - self.emitted_eoi = True - eoi = Batched_EndOfInput() - consumer.Accept(eoi) - else: - batch = BatchReader() - batch.ctor__(Seq(next)) - batch.Fill(consumer) diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py index e6e05eafa..e5475f736 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/src/smithy_dafny_streaming_support/internaldafny/extern/streams.py @@ -1,7 +1,7 @@ from _dafny import Seq, Array as DafnyArray from smithy_python.interfaces.blobs import ByteStream -from smithy_dafny_standard_library.internaldafny.generated.StandardLibrary_Streams import DataStream +from smithy_dafny_streaming_support.internaldafny.generated.StandardLibrary_Streams import DataStream from smithy_dafny_standard_library.internaldafny.generated.Std_BulkActions import BatchSeqWriter, BatchArrayWriter, Batched_BatchValue, Batched_EndOfInput, BatchReader from smithy_dafny_standard_library.internaldafny.generated.Std_Consumers import IgnoreNConsumer from smithy_dafny_standard_library.internaldafny.generated.Std_Producers import Producer diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java index 18686cb43..8202ba94f 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/AwsSdkToDafnyShapeVisitor.java @@ -77,7 +77,7 @@ protected String getDefault(Shape shape) { public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( - "smithy_dafny_standard_library.internaldafny.extern.streams", + "smithy_dafny_streaming_support.internaldafny.extern.streams", "StreamingBlobAsDafnyDataStream" ); return "StreamingBlobAsDafnyDataStream(%1$s)".formatted(dataSource); diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java index 67b182fd3..a0645cf29 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java @@ -77,7 +77,7 @@ protected String getDefault(Shape shape) { public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( - "smithy_dafny_standard_library.internaldafny.extern.streams", + "smithy_dafny_streaming_support.internaldafny.extern.streams", "DafnyDataStreamAsByteStream" ); return "DafnyDataStreamAsByteStream(%1$s)".formatted( diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java index b057fb16d..851afe529 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/DafnyToLocalServiceShapeVisitor.java @@ -95,7 +95,7 @@ protected String getDefault(Shape shape) { public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( - "smithy_dafny_standard_library.internaldafny.extern.streams", + "smithy_dafny_streaming_support.internaldafny.extern.streams", "DafnyDataStreamAsByteStream" ); return "DafnyDataStreamAsByteStream(%1$s)".formatted(dataSource); diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java index bdbf0c627..5fef44ad7 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/localservice/shapevisitor/LocalServiceToDafnyShapeVisitor.java @@ -96,7 +96,7 @@ protected String getDefault(Shape shape) { public String blobShape(BlobShape shape) { if (shape.hasTrait(StreamingTrait.class)) { writer.addStdlibImport( - "smithy_dafny_standard_library.internaldafny.extern.streams", + "smithy_dafny_streaming_support.internaldafny.extern.streams", "StreamingBlobAsDafnyDataStream" ); return "StreamingBlobAsDafnyDataStream(%1$s)".formatted(dataSource); From 52e7cd6f69b6531eb63f02843c6921b759a59dae Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 12:08:21 -0700 Subject: [PATCH 56/72] Java working again --- SmithyDafnyMakefile.mk | 13 ++++++++----- TestModels/SharedMakefile.mk | 10 +++++----- .../Streaming/runtimes/java/build.gradle.kts | 1 + .../StreamingSupport/Makefile | 18 +++++++++++++++++- .../runtimes/java/build.gradle.kts | 3 ++- .../runtimes/python/pyproject.toml | 1 + 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index 47964a70e..7db19cb31 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -265,6 +265,7 @@ transpile_test: # Transpile all other dependencies transpile_dependencies: $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) transpile_implementation_$(LANG) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport transpile_implementation_$(LANG) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) $(patsubst %, $(MAKE) -C $(PROJECT_ROOT)/% transpile_implementation_$(LANG);, $(PROJECT_DEPENDENCIES)) transpile_dependencies_test: @@ -337,7 +338,8 @@ _polymorph_wrapped: $(POLYMORPH_OPTIONS)"; _polymorph_dependencies: - $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/$(STD_LIBRARY) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) +# $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/$(STD_LIBRARY) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/dafny-dependencies/StreamingSupport USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) @$(foreach dependency, \ $(PROJECT_DEPENDENCIES), \ $(MAKE) -C $(PROJECT_ROOT)/$(dependency) polymorph_$(POLYMORPH_LANGUAGE_TARGET); \ @@ -360,7 +362,7 @@ _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -388,7 +390,7 @@ _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_dafny: _polymorph dafny: polymorph_dafny verify @@ -409,7 +411,7 @@ _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_dotnet: _polymorph # Generates java code for all namespaces in this project @@ -428,7 +430,7 @@ _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/mai _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_java: _polymorph # Generates python code for all namespaces in this project @@ -594,6 +596,7 @@ transpile_dependencies_java: transpile_dependencies # Locally deploy all other dependencies mvn_local_deploy_dependencies: $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) mvn_local_deploy, ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport mvn_local_deploy, ) $(patsubst %, $(MAKE) -C $(PROJECT_ROOT)/% mvn_local_deploy;, $(PROJECT_DEPENDENCIES)) # The Java MUST all exist already through the transpile step. diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index 2c7ed5e9e..ff5b07378 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -17,7 +17,7 @@ _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -47,7 +47,7 @@ _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_dafny: _polymorph _polymorph_dafny: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ @@ -58,7 +58,7 @@ _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/mai _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_java: _polymorph _polymorph_java: OUTPUT_JAVA_WRAPPED=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: _polymorph_wrapped @@ -67,7 +67,7 @@ _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ", $(PROJECT_ROOT)/StreamingSupport/src/Index.dfy", ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) _polymorph_dotnet: _polymorph _polymorph_dotnet: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) @@ -82,7 +82,7 @@ _polymorph_python: _polymorph_wrapped _polymorph_rust: OUTPUT_RUST_WRAPPED=--output-rust $(LIBRARY_ROOT)/runtimes/rust _polymorph_rust: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/../StreamingSupport/src/Index.dfy, ) + $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) # For several TestModels we've just manually written the code generation target, # So we just want to ensure we can transpile and pass the tests in CI. # For those, make polymorph_rust should just be a no-op. diff --git a/TestModels/Streaming/runtimes/java/build.gradle.kts b/TestModels/Streaming/runtimes/java/build.gradle.kts index 1aafd2631..c49dcab23 100644 --- a/TestModels/Streaming/runtimes/java/build.gradle.kts +++ b/TestModels/Streaming/runtimes/java/build.gradle.kts @@ -47,6 +47,7 @@ dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") + implementation("software.amazon.cryptography:StreamingSupport:1.0-SNAPSHOT") testImplementation("org.testng:testng:7.5") } diff --git a/TestModels/dafny-dependencies/StreamingSupport/Makefile b/TestModels/dafny-dependencies/StreamingSupport/Makefile index 1ce5cc44c..9f7864dc8 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/Makefile +++ b/TestModels/dafny-dependencies/StreamingSupport/Makefile @@ -5,6 +5,22 @@ USE_DAFNY_STANDARD_LIBRARIES=1 include ../../SharedMakefile.mk +PYTHON_MODULE_NAME=smithy_dafny_streaming_support + transpile_java: | transpile_implementation_java transpile_test_java -transpile_python: | transpile_implementation_python transpile_test_python \ No newline at end of file +transpile_python: | transpile_implementation_python transpile_test_python + +# Overriding this target just to generate the project.properties file. +polymorph_dafny: + cd $(CODEGEN_CLI_ROOT); \ + $(GRADLEW) run --args="\ + --library-root $(LIBRARY_ROOT) \ + --properties-file $(LIBRARY_ROOT)/project.properties \ + --model $(LIBRARY_ROOT)/Model \ + --namespace aws.polymorph \ + --dependent-model $(LIBRARY_ROOT)/../Model \ + "; + +_polymorph_dependencies: + echo "Skipping _polymorph_dependencies for StreamingSupport" \ No newline at end of file diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts index e8c54bdcd..e4a380c18 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/java/build.gradle.kts @@ -63,6 +63,7 @@ repositories { dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") + implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") api("software.amazon.awssdk:sdk-core:2.28.28") } publishing { @@ -73,7 +74,7 @@ publishing { } publications.create("maven") { groupId = "software.amazon.cryptography" - artifactId = "StandardLibrary" + artifactId = "StreamingSupport" from(components["java"]) } repositories { mavenLocal() } diff --git a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml index 451bf0607..870acd975 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml +++ b/TestModels/dafny-dependencies/StreamingSupport/runtimes/python/pyproject.toml @@ -13,6 +13,7 @@ include = ["**/internaldafny/generated/*.py"] [tool.poetry.dependencies] python = "^3.11.0" DafnyRuntimePython = "^4.7.0" +smithy-dafny-standard-library = {path = "../../../StandardLibrary/runtimes/python", develop = false} [tool.poetry.group.test.dependencies] pytest = "^7.4.0" From 4805d70d0d3b375205031bce56867ea2749c9b1a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 14:11:56 -0700 Subject: [PATCH 57/72] Fix python --- TestModels/aws-sdks/s3/Makefile | 1 + TestModels/dafny-dependencies/StreamingSupport/Makefile | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/TestModels/aws-sdks/s3/Makefile b/TestModels/aws-sdks/s3/Makefile index ddbb526b3..0ee9ac3f3 100644 --- a/TestModels/aws-sdks/s3/Makefile +++ b/TestModels/aws-sdks/s3/Makefile @@ -34,4 +34,5 @@ INDEX_FILE_WITH_EXTERN_STRING="module {:extern \"software.amazon.cryptography.se INDEX_FILE_WITHOUT_EXTERN_STRING="module Com.Amazonaws.S3 refines AbstractComAmazonawsS3Service {" TRANSLATION_RECORD_PYTHON := \ + --translation-record ../../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr \ --translation-record ../../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr diff --git a/TestModels/dafny-dependencies/StreamingSupport/Makefile b/TestModels/dafny-dependencies/StreamingSupport/Makefile index 9f7864dc8..ed0586c98 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/Makefile +++ b/TestModels/dafny-dependencies/StreamingSupport/Makefile @@ -7,6 +7,10 @@ include ../../SharedMakefile.mk PYTHON_MODULE_NAME=smithy_dafny_streaming_support +TRANSLATION_RECORD_PYTHON := \ + --translation-record ../StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr + + transpile_java: | transpile_implementation_java transpile_test_java transpile_python: | transpile_implementation_python transpile_test_python From 50f9596483db2e482c554362b7565e35c9f2c9d2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 14:47:51 -0700 Subject: [PATCH 58/72] s3 works for Python --- TestModels/aws-sdks/s3/Makefile | 1 - TestModels/aws-sdks/s3/runtimes/python/pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/TestModels/aws-sdks/s3/Makefile b/TestModels/aws-sdks/s3/Makefile index 0ee9ac3f3..2762e7214 100644 --- a/TestModels/aws-sdks/s3/Makefile +++ b/TestModels/aws-sdks/s3/Makefile @@ -35,4 +35,3 @@ INDEX_FILE_WITHOUT_EXTERN_STRING="module Com.Amazonaws.S3 refines AbstractComAma TRANSLATION_RECORD_PYTHON := \ --translation-record ../../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr \ - --translation-record ../../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr diff --git a/TestModels/aws-sdks/s3/runtimes/python/pyproject.toml b/TestModels/aws-sdks/s3/runtimes/python/pyproject.toml index fecc98501..6058c169b 100644 --- a/TestModels/aws-sdks/s3/runtimes/python/pyproject.toml +++ b/TestModels/aws-sdks/s3/runtimes/python/pyproject.toml @@ -14,6 +14,7 @@ include = ["**/smithygenerated/**/*.py", "**/internaldafny/generated/*.py"] python = "^3.11.0" boto3 = "^1.28.38" smithy-dafny-standard-library = {path = "../../../../dafny-dependencies/StandardLibrary/runtimes/python", develop = false} +smithy-dafny-streaming-support = {path = "../../../../dafny-dependencies/StreamingSupport/runtimes/python", develop = false} # TODO: Depend on PyPi once Smithy-Python publishes their Python package there smithy-python = { path = "../../../../../codegen/smithy-dafny-codegen-modules/smithy-python/python-packages/smithy-python", develop = false} From ab2f6d1ba7242f3b87673e31ccddc25f6ba02e1f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 15:02:01 -0700 Subject: [PATCH 59/72] Patch Java codegen gaps --- .../s3/codegen-patches/java/dafny-4.9.0.patch | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 TestModels/aws-sdks/s3/codegen-patches/java/dafny-4.9.0.patch diff --git a/TestModels/aws-sdks/s3/codegen-patches/java/dafny-4.9.0.patch b/TestModels/aws-sdks/s3/codegen-patches/java/dafny-4.9.0.patch new file mode 100644 index 000000000..624d1e978 --- /dev/null +++ b/TestModels/aws-sdks/s3/codegen-patches/java/dafny-4.9.0.patch @@ -0,0 +1,315 @@ +diff --git b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +index 87bdb89d9..bc7fb494d 100644 +--- b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java ++++ a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/Shim.java +@@ -3,9 +3,13 @@ + // Do not modify this file. This file is machine generated, and any changes to it will be overwritten. + package software.amazon.cryptography.services.s3.internaldafny; + ++import Streams.DataStreamAsRequestBody; + import Wrappers_Compile.Result; + import java.lang.Override; + import java.lang.String; ++ ++import software.amazon.awssdk.core.ResponseInputStream; ++import software.amazon.awssdk.core.sync.RequestBody; + import software.amazon.awssdk.services.s3.S3Client; + import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; + import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +@@ -107,7 +111,7 @@ public class Shim implements IS3Client { + software.amazon.awssdk.services.s3.model.GetObjectRequest converted = + ToNative.GetObjectRequest(input); + try { +- GetObjectResponse result = _impl.getObject(converted); ++ ResponseInputStream result = _impl.getObject(converted); + GetObjectOutput dafnyResponse = ToDafny.GetObjectOutput(result); + return Result.create_Success( + GetObjectOutput._typeDescriptor(), +@@ -145,8 +149,9 @@ public class Shim implements IS3Client { + public Result PutObject(PutObjectRequest input) { + software.amazon.awssdk.services.s3.model.PutObjectRequest converted = + ToNative.PutObjectRequest(input); ++ RequestBody body = DataStreamAsRequestBody.of(input._Body.dtor_value()); + try { +- PutObjectResponse result = _impl.putObject(converted); ++ PutObjectResponse result = _impl.putObject(converted, body); + PutObjectOutput dafnyResponse = ToDafny.PutObjectOutput(result); + return Result.create_Success( + PutObjectOutput._typeDescriptor(), +diff --git b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +index 8cfc49346..3746cdacc 100644 +--- b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java ++++ a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToDafny.java +@@ -4,6 +4,8 @@ + package software.amazon.cryptography.services.s3.internaldafny; + + import StandardLibrary_Compile.Streams_Compile.DataStream; ++import Streams.InputStreamAsDataStream; ++import Streams.RequestBodyAsDataStream; + import Wrappers_Compile.Option; + import dafny.DafnyMap; + import dafny.DafnySequence; +@@ -19,6 +21,9 @@ import java.lang.String; + import java.util.List; + import java.util.Map; + import java.util.Objects; ++ ++import software.amazon.awssdk.core.ResponseInputStream; ++import software.amazon.awssdk.core.sync.RequestBody; + import software.amazon.awssdk.services.s3.S3Client; + import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; + import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +@@ -352,7 +357,7 @@ public class ToDafny { + } + + public static DafnySequence Errors( +- List nativeValue ++ List nativeValue + ) { + return software.amazon.smithy.dafny.conversion.ToDafny.Aggregate.GenericToSequence( + nativeValue, +@@ -362,7 +367,7 @@ public class ToDafny { + } + + public static ErrorShape ErrorShape( +- software.amazon.awssdk.services.s3.model.ErrorShape nativeValue ++ software.amazon.awssdk.services.s3.model.S3Error nativeValue + ) { + Option> key; + key = +@@ -415,16 +420,13 @@ public class ToDafny { + return new ErrorShape(key, versionId, code, message); + } + +- public static GetObjectOutput GetObjectOutput(GetObjectResponse nativeValue) { ++ public static GetObjectOutput GetObjectOutput(ResponseInputStream responseInputStream) { ++ GetObjectResponse nativeValue = responseInputStream.response(); + Option> body; + body = +- Objects.nonNull(nativeValue.body()) +- ? Option.create_Some( +- DafnySequence._typeDescriptor(TypeDescriptor.BYTE), +- ToDafny.DataStream(nativeValue.body().asByteArray()) +- ) +- : Option.create_None( +- DafnySequence._typeDescriptor(TypeDescriptor.BYTE) ++ Option.create_Some( ++ null, ++ new InputStreamAsDataStream(Error._typeDescriptor(), responseInputStream, Error::create_Opaque) + ); + Option deleteMarker; + deleteMarker = +@@ -716,11 +718,11 @@ public class ToDafny { + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = +- Objects.nonNull(nativeValue.sseKMSKeyId()) ++ Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +- nativeValue.sseKMSKeyId() ++ nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( +@@ -1252,11 +1254,11 @@ public class ToDafny { + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = +- Objects.nonNull(nativeValue.sseKMSKeyId()) ++ Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +- nativeValue.sseKMSKeyId() ++ nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( +@@ -1264,11 +1266,11 @@ public class ToDafny { + ); + Option> sSEKMSEncryptionContext; + sSEKMSEncryptionContext = +- Objects.nonNull(nativeValue.sseKMSEncryptionContext()) ++ Objects.nonNull(nativeValue.ssekmsEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +- nativeValue.sseKMSEncryptionContext() ++ nativeValue.ssekmsEncryptionContext() + ) + ) + : Option.create_None( +@@ -1309,7 +1311,8 @@ public class ToDafny { + } + + public static PutObjectRequest PutObjectRequest( +- software.amazon.awssdk.services.s3.model.PutObjectRequest nativeValue ++ software.amazon.awssdk.services.s3.model.PutObjectRequest nativeValue, ++ RequestBody nativeBody + ) { + Option aCL; + aCL = +@@ -1319,16 +1322,10 @@ public class ToDafny { + ToDafny.ObjectCannedACL(nativeValue.acl()) + ) + : Option.create_None(ObjectCannedACL._typeDescriptor()); +- Option> body; +- body = +- Objects.nonNull(nativeValue.body()) +- ? Option.create_Some( +- DafnySequence._typeDescriptor(TypeDescriptor.BYTE), +- ToDafny.DataStream(nativeValue.body().asByteArray()) +- ) +- : Option.create_None( +- DafnySequence._typeDescriptor(TypeDescriptor.BYTE) +- ); ++ Option> body = Option.create_Some( ++ null, ++ new RequestBodyAsDataStream( ++ null, nativeBody, Error::create_Opaque )); + DafnySequence bucket; + bucket = + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +@@ -1620,11 +1617,11 @@ public class ToDafny { + ); + Option> sSEKMSKeyId; + sSEKMSKeyId = +- Objects.nonNull(nativeValue.sseKMSKeyId()) ++ Objects.nonNull(nativeValue.ssekmsKeyId()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +- nativeValue.sseKMSKeyId() ++ nativeValue.ssekmsKeyId() + ) + ) + : Option.create_None( +@@ -1632,11 +1629,11 @@ public class ToDafny { + ); + Option> sSEKMSEncryptionContext; + sSEKMSEncryptionContext = +- Objects.nonNull(nativeValue.sseKMSEncryptionContext()) ++ Objects.nonNull(nativeValue.ssekmsEncryptionContext()) + ? Option.create_Some( + DafnySequence._typeDescriptor(TypeDescriptor.CHAR), + software.amazon.smithy.dafny.conversion.ToDafny.Simple.CharacterSequence( +- nativeValue.sseKMSEncryptionContext() ++ nativeValue.ssekmsEncryptionContext() + ) + ) + : Option.create_None( +diff --git b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +index e02646327..433c3f16c 100644 +--- b/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java ++++ a/TestModels/aws-sdks/s3/runtimes/java/src/main/smithy-generated/software/amazon/cryptography/services/s3/internaldafny/ToNative.java +@@ -22,7 +22,6 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; + import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; + import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; + import software.amazon.awssdk.services.s3.model.DeletedObject; +-import software.amazon.awssdk.services.s3.model.ErrorShape; + import software.amazon.awssdk.services.s3.model.GetObjectRequest; + import software.amazon.awssdk.services.s3.model.GetObjectResponse; + import software.amazon.awssdk.services.s3.model.IntelligentTieringAccessTier; +@@ -38,6 +37,7 @@ import software.amazon.awssdk.services.s3.model.PutObjectResponse; + import software.amazon.awssdk.services.s3.model.ReplicationStatus; + import software.amazon.awssdk.services.s3.model.RequestCharged; + import software.amazon.awssdk.services.s3.model.RequestPayer; ++import software.amazon.awssdk.services.s3.model.S3Error; + import software.amazon.awssdk.services.s3.model.S3Exception; + import software.amazon.awssdk.services.s3.model.ServerSideEncryption; + import software.amazon.awssdk.services.s3.model.StorageClass; +@@ -269,7 +269,7 @@ public class ToNative { + return builder.build(); + } + +- public static List Errors( ++ public static List Errors( + DafnySequence< + ? extends software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape + > dafnyValue +@@ -280,10 +280,10 @@ public class ToNative { + ); + } + +- public static ErrorShape ErrorShape( ++ public static S3Error ErrorShape( + software.amazon.cryptography.services.s3.internaldafny.types.ErrorShape dafnyValue + ) { +- ErrorShape.Builder builder = ErrorShape.builder(); ++ S3Error.Builder builder = S3Error.builder(); + if (dafnyValue.dtor_Code().is_Some()) { + builder.code( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( +@@ -324,13 +324,6 @@ public class ToNative { + ) + ); + } +- if (dafnyValue.dtor_Body().is_Some()) { +- builder.body( +- SdkBytes.fromByteArray( +- (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) +- ) +- ); +- } + if (dafnyValue.dtor_BucketKeyEnabled().is_Some()) { + builder.bucketKeyEnabled( + (dafnyValue.dtor_BucketKeyEnabled().dtor_value()) +@@ -511,7 +504,7 @@ public class ToNative { + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { +- builder.sseKMSKeyId( ++ builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) +@@ -866,14 +859,14 @@ public class ToNative { + ); + } + if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { +- builder.sseKMSEncryptionContext( ++ builder.ssekmsEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { +- builder.sseKMSKeyId( ++ builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) +@@ -896,13 +889,6 @@ public class ToNative { + if (dafnyValue.dtor_ACL().is_Some()) { + builder.acl(ToNative.ObjectCannedACL(dafnyValue.dtor_ACL().dtor_value())); + } +- if (dafnyValue.dtor_Body().is_Some()) { +- builder.body( +- SdkBytes.fromByteArray( +- (byte[]) (dafnyValue.dtor_Body().dtor_value().toRawArray()) +- ) +- ); +- } + builder.bucket( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_Bucket() +@@ -1098,14 +1084,14 @@ public class ToNative { + ); + } + if (dafnyValue.dtor_SSEKMSEncryptionContext().is_Some()) { +- builder.sseKMSEncryptionContext( ++ builder.ssekmsEncryptionContext( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSEncryptionContext().dtor_value() + ) + ); + } + if (dafnyValue.dtor_SSEKMSKeyId().is_Some()) { +- builder.sseKMSKeyId( ++ builder.ssekmsKeyId( + software.amazon.smithy.dafny.conversion.ToNative.Simple.String( + dafnyValue.dtor_SSEKMSKeyId().dtor_value() + ) From 076653efe2d10ba74cf0c1abdb9aa560be3c8b9e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 15:11:10 -0700 Subject: [PATCH 60/72] Fix S3 Java --- TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts index 5fe73ed04..093eb548e 100644 --- a/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts +++ b/TestModels/aws-sdks/s3/runtimes/java/build.gradle.kts @@ -47,6 +47,7 @@ dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") implementation("software.amazon.cryptography:StandardLibrary:1.0-SNAPSHOT") + implementation("software.amazon.cryptography:StreamingSupport:1.0-SNAPSHOT") implementation(platform("software.amazon.awssdk:bom:2.31.30")) implementation("software.amazon.awssdk:s3") } From 0dbfea584ece1d19b8da989a6b34248d2af9cbe6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 15:22:01 -0700 Subject: [PATCH 61/72] Test on Dafny 4.10 iff streaming --- .../software/amazon/polymorph/TestModelTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index 40f5cea6e..3bfe3d068 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -91,14 +91,13 @@ protected Path getTestModelPath(String relativeTestModelPath) { protected void testModels(String relativeTestModelPath) { // The @streaming support depends on our subset of the Dafny standard libraries // which cannot be built for old versions of Dafny. - if ( - relativeTestModelPath.endsWith("Streaming") || - relativeTestModelPath.endsWith("s3") - ) { - DafnyVersion dafnyVersion = CodegenEngine.getDafnyVersionFromDafny(); - if (dafnyVersion.compareTo(DafnyVersion.parse("4.9.0")) < 0) { - Assumptions.assumeTrue(false); - } + // We also don't want to burn a lot of CI cycles testing all models against Java 4.10 yet. + DafnyVersion dafnyVersion = CodegenEngine.getDafnyVersionFromDafny(); + boolean needsStreaming = relativeTestModelPath.endsWith("Streaming") || relativeTestModelPath.endsWith("s3"); + if (dafnyVersion.compareTo(DafnyVersion.parse("4.10.0")) < 0) { + Assumptions.assumeFalse(needsStreaming); + } else { + Assumptions.assumeTrue(needsStreaming); } } } From 9c4b46d789c52b890fec8e9ef6a26099435196d0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 16:06:34 -0700 Subject: [PATCH 62/72] Move doo file to StreamingSupport, rename variable --- SmithyDafnyMakefile.mk | 27 ++++++++++--------- TestModels/SharedMakefile.mk | 11 ++++---- TestModels/Streaming/Makefile | 2 +- TestModels/aws-sdks/s3/Makefile | 2 +- .../StandardLibrary/Makefile | 8 ++---- ...StandardLibraries-smithy-dafny-subset.toml | 0 .../StreamingSupport/Makefile | 8 ++++-- 7 files changed, 30 insertions(+), 28 deletions(-) rename TestModels/dafny-dependencies/{StandardLibrary => StreamingSupport}/DafnyStandardLibraries-smithy-dafny-subset.toml (100%) diff --git a/SmithyDafnyMakefile.mk b/SmithyDafnyMakefile.mk index 7db19cb31..e8388e5c0 100644 --- a/SmithyDafnyMakefile.mk +++ b/SmithyDafnyMakefile.mk @@ -30,6 +30,7 @@ # of Model directories # AWS_SDK_CMD -- the `--aws-sdk` command to generate AWS SDK style interfaces. # STD_LIBRARY -- path from this file to the StandardLibrary Dafny project. +# STREAMING_SUPPORT -- path from this file to the StreamingSupport Dafny project. # SMITHY_DEPS -- path from this file to smithy dependencies, such as custom traits. # GRADLEW -- the gradlew to use when building Java runtimes. # defaults to $(SMITHY_DAFNY_ROOT)/codegen/gradlew @@ -98,7 +99,7 @@ verify: --log-format csv \ --verification-time-limit $(VERIFY_TIMEOUT) \ --resource-limit $(MAX_RESOURCE_COUNT) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USES_STREAMING), --library:$(PROJECT_ROOT)/$(STREAMING_SUPPORT)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(DAFNY_OPTIONS) \ % @@ -115,7 +116,7 @@ verify_single: --log-format text \ --verification-time-limit $(VERIFY_TIMEOUT) \ --resource-limit $(MAX_RESOURCE_COUNT) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USES_STREAMING), --library:$(PROJECT_ROOT)/$(STREAMING_SUPPORT)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(DAFNY_OPTIONS) \ $(if ${PROC},-proc:*$(PROC)*,) \ $(FILE) @@ -215,7 +216,7 @@ transpile_implementation: $(DAFNY_OTHER_FILES) \ $(TRANSPILE_MODULE_NAME) \ $(if $(strip $(STD_LIBRARY)) , --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, ) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES) , --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USES_STREAMING) , --library:$(PROJECT_ROOT)/$(STREAMING_SUPPORT)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(TRANSLATION_RECORD) \ $(TRANSPILE_DEPENDENCIES) @@ -255,7 +256,7 @@ transpile_test: $(DAFNY_OPTIONS) \ $(DAFNY_OTHER_FILES) \ $(if $(strip $(STD_LIBRARY)) , --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy, ) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES) , --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USES_STREAMING) , --library:$(PROJECT_ROOT)/$(STREAMING_SUPPORT)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(TRANSLATION_RECORD) \ $(SOURCE_TRANSLATION_RECORD) \ $(TRANSPILE_MODULE_NAME) \ @@ -264,8 +265,8 @@ transpile_test: # If we are not the StandardLibrary, transpile the StandardLibrary. # Transpile all other dependencies transpile_dependencies: - $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) transpile_implementation_$(LANG) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) - $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport transpile_implementation_$(LANG) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) + $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) transpile_implementation_$(LANG) USES_STREAMING=$(USES_STREAMING), ) + $(if $(USES_STREAMING), $(MAKE) -C $(PROJECT_ROOT)/$(STREAMING_SUPPORT) transpile_implementation_$(LANG) USES_STREAMING=$(USES_STREAMING), ) $(patsubst %, $(MAKE) -C $(PROJECT_ROOT)/% transpile_implementation_$(LANG);, $(PROJECT_DEPENDENCIES)) transpile_dependencies_test: @@ -338,8 +339,8 @@ _polymorph_wrapped: $(POLYMORPH_OPTIONS)"; _polymorph_dependencies: -# $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/$(STD_LIBRARY) USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) - $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/dafny-dependencies/StreamingSupport USE_DAFNY_STANDARD_LIBRARIES=$(USE_DAFNY_STANDARD_LIBRARIES), ) + $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/$(STD_LIBRARY) USES_STREAMING=$(USES_STREAMING), ) + $(if $(USES_STREAMING), $(MAKE) -C $(PROJECT_ROOT)/$(STREAMING_SUPPORT) polymorph_$(POLYMORPH_LANGUAGE_TARGET) LIBRARY_ROOT=$(PROJECT_ROOT)/$(STREAMING_SUPPORT) USES_STREAMING=$(USES_STREAMING), ) @$(foreach dependency, \ $(PROJECT_DEPENDENCIES), \ $(MAKE) -C $(PROJECT_ROOT)/$(dependency) polymorph_$(POLYMORPH_LANGUAGE_TARGET); \ @@ -362,7 +363,7 @@ _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -390,7 +391,7 @@ _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_dafny: _polymorph dafny: polymorph_dafny verify @@ -411,7 +412,7 @@ _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_dotnet: _polymorph # Generates java code for all namespaces in this project @@ -430,7 +431,7 @@ _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/mai _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_java: _polymorph # Generates python code for all namespaces in this project @@ -596,7 +597,7 @@ transpile_dependencies_java: transpile_dependencies # Locally deploy all other dependencies mvn_local_deploy_dependencies: $(if $(strip $(STD_LIBRARY)), $(MAKE) -C $(PROJECT_ROOT)/$(STD_LIBRARY) mvn_local_deploy, ) - $(if $(USE_DAFNY_STANDARD_LIBRARIES), $(MAKE) -C $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport mvn_local_deploy, ) + $(if $(USES_STREAMING), $(MAKE) -C $(PROJECT_ROOT)/$(STREAMING_SUPPORT) mvn_local_deploy, ) $(patsubst %, $(MAKE) -C $(PROJECT_ROOT)/% mvn_local_deploy;, $(PROJECT_DEPENDENCIES)) # The Java MUST all exist already through the transpile step. diff --git a/TestModels/SharedMakefile.mk b/TestModels/SharedMakefile.mk index ff5b07378..71523876e 100644 --- a/TestModels/SharedMakefile.mk +++ b/TestModels/SharedMakefile.mk @@ -7,6 +7,7 @@ PROJECT_ROOT := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) SMITHY_DAFNY_ROOT := $(PROJECT_ROOT)/.. STD_LIBRARY := dafny-dependencies/StandardLibrary +STREAMING_SUPPORT := dafny-dependencies/StreamingSupport include $(SMITHY_DAFNY_ROOT)/SmithyDafnyMakefile.mk @@ -17,7 +18,7 @@ _polymorph_code_gen: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_code_gen: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_code_gen: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_code_gen: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated @@ -47,7 +48,7 @@ _polymorph_dafny: OUTPUT_DAFNY=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) _polymorph_dafny: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_dafny: _polymorph _polymorph_dafny: OUTPUT_DAFNY_WRAPPED=\ --output-dafny $(if $(DIR_STRUCTURE_V2), $(LIBRARY_ROOT)/dafny/$(SERVICE)/Model, $(LIBRARY_ROOT)/Model) \ @@ -58,7 +59,7 @@ _polymorph_java: OUTPUT_JAVA=--output-java $(LIBRARY_ROOT)/runtimes/java/src/mai _polymorph_java: OUTPUT_JAVA_TEST=--output-java-test $(LIBRARY_ROOT)/runtimes/java/src/test/smithy-generated _polymorph_java: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_java: _polymorph _polymorph_java: OUTPUT_JAVA_WRAPPED=--output-java $(LIBRARY_ROOT)/runtimes/java/src/main/smithy-generated _polymorph_java: _polymorph_wrapped @@ -67,7 +68,7 @@ _polymorph_dotnet: OUTPUT_DOTNET=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/) _polymorph_dotnet: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) _polymorph_dotnet: _polymorph _polymorph_dotnet: OUTPUT_DOTNET_WRAPPED=\ $(if $(DIR_STRUCTURE_V2), --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped/$(SERVICE)/, --output-dotnet $(LIBRARY_ROOT)/runtimes/net/Generated/Wrapped) @@ -82,7 +83,7 @@ _polymorph_python: _polymorph_wrapped _polymorph_rust: OUTPUT_RUST_WRAPPED=--output-rust $(LIBRARY_ROOT)/runtimes/rust _polymorph_rust: INPUT_DAFNY=\ --include-dafny $(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), --include-dafny $(PROJECT_ROOT)/dafny-dependencies/StreamingSupport/src/Index.dfy, ) + $(if $(USES_STREAMING), --include-dafny $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/src/Index.dfy, ) # For several TestModels we've just manually written the code generation target, # So we just want to ensure we can transpile and pass the tests in CI. # For those, make polymorph_rust should just be a no-op. diff --git a/TestModels/Streaming/Makefile b/TestModels/Streaming/Makefile index d3eb09653..b120520f6 100644 --- a/TestModels/Streaming/Makefile +++ b/TestModels/Streaming/Makefile @@ -3,7 +3,7 @@ CORES=2 -USE_DAFNY_STANDARD_LIBRARIES=1 +USES_STREAMING=1 ENABLE_EXTERN_PROCESSING=1 diff --git a/TestModels/aws-sdks/s3/Makefile b/TestModels/aws-sdks/s3/Makefile index 2762e7214..3f1fb9669 100644 --- a/TestModels/aws-sdks/s3/Makefile +++ b/TestModels/aws-sdks/s3/Makefile @@ -3,7 +3,7 @@ ENABLE_EXTERN_PROCESSING=1 CORES=2 -USE_DAFNY_STANDARD_LIBRARIES=1 +USES_STREAMING=1 include ../../SharedMakefile.mk diff --git a/TestModels/dafny-dependencies/StandardLibrary/Makefile b/TestModels/dafny-dependencies/StandardLibrary/Makefile index a34ac61df..9e209eaa6 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/Makefile +++ b/TestModels/dafny-dependencies/StandardLibrary/Makefile @@ -25,7 +25,7 @@ else endif # Overriding this target just to generate the project.properties file. -polymorph_dafny: $(if $(USE_DAFNY_STANDARD_LIBRARIES), build_dafny_stdlibs, ) +polymorph_dafny: cd $(CODEGEN_CLI_ROOT); \ $(GRADLEW) run --args="\ --library-root $(LIBRARY_ROOT) \ @@ -108,15 +108,11 @@ transpile_implementation: --unicode-char:false \ --function-syntax:3 \ --output $(OUT) \ - $(if $(USE_DAFNY_STANDARD_LIBRARIES), ./bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ + $(if $(USES_STREAMING), $(PROJECT_ROOT)/$(STREAMING_SUPPORT)/bin/DafnyStandardLibraries-smithy-dafny-subset.doo, ) \ $(DAFNY_OPTIONS) \ $(DAFNY_OTHER_FILES) \ $(TRANSPILE_MODULE_NAME) -build_dafny_stdlibs: - dafny build -t:lib --output bin/DafnyStandardLibraries-smithy-dafny-subset.doo DafnyStandardLibraries-smithy-dafny-subset.toml - rm -rf DafnyStandardLibraries-smithy-dafny-subset-lib - # Override SharedMakefile's build_java to not install # StandardLibrary as a dependency build_java: transpile_java diff --git a/TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml similarity index 100% rename from TestModels/dafny-dependencies/StandardLibrary/DafnyStandardLibraries-smithy-dafny-subset.toml rename to TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml diff --git a/TestModels/dafny-dependencies/StreamingSupport/Makefile b/TestModels/dafny-dependencies/StreamingSupport/Makefile index ed0586c98..7284aa1e4 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/Makefile +++ b/TestModels/dafny-dependencies/StreamingSupport/Makefile @@ -1,7 +1,7 @@ CORES=2 -USE_DAFNY_STANDARD_LIBRARIES=1 +USES_STREAMING=1 include ../../SharedMakefile.mk @@ -16,7 +16,7 @@ transpile_java: | transpile_implementation_java transpile_test_java transpile_python: | transpile_implementation_python transpile_test_python # Overriding this target just to generate the project.properties file. -polymorph_dafny: +polymorph_dafny: build_dafny_stdlibs cd $(CODEGEN_CLI_ROOT); \ $(GRADLEW) run --args="\ --library-root $(LIBRARY_ROOT) \ @@ -26,5 +26,9 @@ polymorph_dafny: --dependent-model $(LIBRARY_ROOT)/../Model \ "; +build_dafny_stdlibs: + dafny build -t:lib --output bin/DafnyStandardLibraries-smithy-dafny-subset.doo DafnyStandardLibraries-smithy-dafny-subset.toml + rm -rf DafnyStandardLibraries-smithy-dafny-subset-lib + _polymorph_dependencies: echo "Skipping _polymorph_dependencies for StreamingSupport" \ No newline at end of file From 221ce1f13979c799a49e98dabd916de1301af861 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 16:24:13 -0700 Subject: [PATCH 63/72] Revert smithy-build.json change --- .gitmodules | 1 - .../amazon/polymorph/TestModelTest.java | 4 ++- .../amazon/polymorph/CodegenEngine.java | 6 ++-- .../smithydafny/DafnyNameResolver.java | 5 ++- .../smithyjava/generator/ToNative.java | 5 ++- .../generator/library/ToDafnyLibrary.java | 12 ++++--- .../generator/library/ToNativeLibrary.java | 1 - .../smithyjava/nameresolver/Constants.java | 14 +++----- .../smithyjava/nameresolver/Native.java | 1 - .../DafnyToAwsSdkShapeVisitor.java | 4 +-- .../DafnyClientCodegenPluginSettings.java | 36 +++++++++---------- 11 files changed, 44 insertions(+), 45 deletions(-) diff --git a/.gitmodules b/.gitmodules index bc8908a19..7ff076388 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,4 +4,3 @@ [submodule "TestModels/dafny-dependencies/dafny"] path = TestModels/dafny-dependencies/dafny url = git@github.com:dafny-lang/dafny.git - branch = more-streaming-libs diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index 3bfe3d068..44cc56979 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -93,7 +93,9 @@ protected void testModels(String relativeTestModelPath) { // which cannot be built for old versions of Dafny. // We also don't want to burn a lot of CI cycles testing all models against Java 4.10 yet. DafnyVersion dafnyVersion = CodegenEngine.getDafnyVersionFromDafny(); - boolean needsStreaming = relativeTestModelPath.endsWith("Streaming") || relativeTestModelPath.endsWith("s3"); + boolean needsStreaming = + relativeTestModelPath.endsWith("Streaming") || + relativeTestModelPath.endsWith("s3"); if (dafnyVersion.compareTo(DafnyVersion.parse("4.10.0")) < 0) { Assumptions.assumeFalse(needsStreaming); } else { diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java index 3d408be67..5bf18bf96 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/CodegenEngine.java @@ -1015,9 +1015,9 @@ private static CommandResult runCommand(Path workingDir, String... args) { private Path standardLibraryPath() { if (this.includeDafnyFiles.isEmpty()) { - throw new IllegalStateException( - "includeDafnyFile required when generating additional aspects (--generate)" - ); + throw new IllegalStateException( + "includeDafnyFile required when generating additional aspects (--generate)" + ); } // Assumes that includeDafnyFiles.get(0) is at StandardLibrary/src/Index.dfy diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java index 82c8b3d8c..329d0c123 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithydafny/DafnyNameResolver.java @@ -386,7 +386,10 @@ public static Stream modulePreludeStandardImports( .stream() .anyMatch(s -> s.hasTrait(StreamingTrait.class)) ) { - return Streams.concat(basics, Stream.of("import opened StandardLibrary.Streams")); + return Streams.concat( + basics, + Stream.of("import opened StandardLibrary.Streams") + ); } else { return basics; } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java index 186622f5a..1ea196a70 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/ToNative.java @@ -411,7 +411,10 @@ protected MethodReference conversionMethodReference(Shape shape) { ); } if (shape.isBlobShape() && shape.hasTrait(StreamingTrait.class)) { - return new MethodReference(software.amazon.polymorph.smithyjava.nameresolver.Constants.DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME, "of"); + return new MethodReference( + software.amazon.polymorph.smithyjava.nameresolver.Constants.DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME, + "of" + ); } // If the target is simple, use SIMPLE_CONVERSION_METHOD_FROM_SHAPE_TYPE if (ModelUtils.isSmithyApiOrSimpleShape(shape)) { diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java index 89226b1df..8f2c61192 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java @@ -19,7 +19,6 @@ import java.util.Set; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; - import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithyjava.MethodReference; import software.amazon.polymorph.smithyjava.generator.ToDafny; @@ -275,7 +274,8 @@ MethodSpec collectionError() { } MethodSpec dataStream() { - ClassName dafnyDataStream = software.amazon.polymorph.smithyjava.nameresolver.Constants.DAFNY_DATA_STREAM_CLASS_NAME; + ClassName dafnyDataStream = + software.amazon.polymorph.smithyjava.nameresolver.Constants.DAFNY_DATA_STREAM_CLASS_NAME; TypeName dafnyDataStreamParameterized = ParameterizedTypeName.get( dafnyDataStream, ClassName.get(Byte.class), @@ -286,11 +286,15 @@ MethodSpec dataStream() { .returns(dafnyDataStreamParameterized) .addModifiers(PUBLIC_STATIC) .addParameter(ClassName.get(RequestBody.class), VAR_INPUT) - .addStatement("return new $T(\n" + + .addStatement( + "return new $T(\n" + " Error._typeDescriptor(),\n" + " $L,\n" + " Error::create_Opaque\n" + - ")", software.amazon.polymorph.smithyjava.nameresolver.Constants.REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME, VAR_INPUT) + ")", + software.amazon.polymorph.smithyjava.nameresolver.Constants.REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME, + VAR_INPUT + ) .build(); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java index 54487c042..80e06b186 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java @@ -20,7 +20,6 @@ import java.util.Set; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; - import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithydafny.DafnyNameResolver; import software.amazon.polymorph.smithyjava.MethodReference; diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java index 2cf8fff40..d8948941c 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Constants.java @@ -6,7 +6,6 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Set; - import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.smithy.model.shapes.ShapeId; import software.amazon.smithy.model.shapes.ShapeType; @@ -63,13 +62,8 @@ public class Constants { "StandardLibrary_Compile.Streams_Compile", "DataStream" ); - public static final ClassName REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME = ClassName.get( - "Streams", - "RequestBodyAsDataStream" - ); - public static final ClassName DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME = ClassName.get( - "Streams", - "DataStreamAsRequestBody" - ); - + public static final ClassName REQUEST_BODY_AS_DATA_STREAM_CLASS_NAME = + ClassName.get("Streams", "RequestBodyAsDataStream"); + public static final ClassName DATA_STREAM_AS_REQUEST_BODY_CLASS_NAME = + ClassName.get("Streams", "DataStreamAsRequestBody"); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java index cdf4c9c85..2f93fb596 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/nameresolver/Native.java @@ -18,7 +18,6 @@ import java.util.Optional; import java.util.Set; import javax.annotation.Nullable; - import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithydafny.DafnyNameResolver; import software.amazon.polymorph.smithyjava.NamespaceHelper; diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java index a0645cf29..aa2834b58 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithypython/awssdk/shapevisitor/DafnyToAwsSdkShapeVisitor.java @@ -80,9 +80,7 @@ public String blobShape(BlobShape shape) { "smithy_dafny_streaming_support.internaldafny.extern.streams", "DafnyDataStreamAsByteStream" ); - return "DafnyDataStreamAsByteStream(%1$s)".formatted( - dataSource - ); + return "DafnyDataStreamAsByteStream(%1$s)".formatted(dataSource); } else { return "bytes(%1$s)".formatted(dataSource); } diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java index 172f05bd7..f33464597 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/smithy/dafny/codegen/DafnyClientCodegenPluginSettings.java @@ -128,25 +128,23 @@ static Optional fromObject( final Optional buildRoot = findSmithyBuildJson(manifest.getBaseDir()) .map(p -> p.getParent()); - final List includeDafnyFilesNormalized = node - .expectArrayMember("includeDafnyFiles") - .getElementsAs(StringNode.class) - .stream() - .map(pathNode -> { - final Path includeDafnyFile = Path.of(pathNode.getValue()); - final Path includeDafnyFileNormalized = buildRoot.isPresent() && - !includeDafnyFile.isAbsolute() - ? buildRoot.get().resolve(includeDafnyFile).toAbsolutePath().normalize() - : includeDafnyFile; - if (Files.notExists(includeDafnyFileNormalized)) { - LOGGER.warn( - "Generated Dafny code may not compile because the includeDafnyFile could not be found: {}", - includeDafnyFileNormalized - ); - } - return includeDafnyFileNormalized; - }) - .toList(); + final String includeDafnyFileStr = node + .expectStringMember("includeDafnyFile") + .getValue(); + final Path includeDafnyFile = Path.of(includeDafnyFileStr); + final Path includeDafnyFileNormalized = buildRoot.isPresent() && + !includeDafnyFile.isAbsolute() + ? buildRoot.get().resolve(includeDafnyFile).toAbsolutePath().normalize() + : includeDafnyFile; + if (Files.notExists(includeDafnyFileNormalized)) { + LOGGER.warn( + "Generated Dafny code may not compile because the includeDafnyFile could not be found: {}", + includeDafnyFileNormalized + ); + } + final List includeDafnyFilesNormalized = List.of( + includeDafnyFileNormalized + ); // This is now optional since we can get it from dafny itself final DafnyVersion dafnyVersionString = node From 85d293bc44effc7360a73a9465491cab63d88357 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Jun 2025 16:42:15 -0700 Subject: [PATCH 64/72] m --- .../src/test/java/software/amazon/polymorph/TestModelTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index 44cc56979..c6c90c9c9 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -95,7 +95,8 @@ protected void testModels(String relativeTestModelPath) { DafnyVersion dafnyVersion = CodegenEngine.getDafnyVersionFromDafny(); boolean needsStreaming = relativeTestModelPath.endsWith("Streaming") || - relativeTestModelPath.endsWith("s3"); + relativeTestModelPath.endsWith("s3") || + relativeTestModelPath.endsWith("StreamingSupport"); if (dafnyVersion.compareTo(DafnyVersion.parse("4.10.0")) < 0) { Assumptions.assumeFalse(needsStreaming); } else { From cfcd919131b492f901f2cf54ec44102b44c184c5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 27 Jun 2025 13:03:28 -0700 Subject: [PATCH 65/72] More conditional code --- .../software/amazon/polymorph/smithyjava/JavaTestModels.java | 1 - .../smithyjava/generator/library/ToDafnyLibrary.java | 4 +++- .../smithyjava/generator/library/ToNativeLibrary.java | 5 +---- .../java/software/amazon/polymorph/utils/ModelUtils.java | 5 +++++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java index 99e38b79b..6d5601353 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/smithyjava/JavaTestModels.java @@ -42,7 +42,6 @@ class JavaTestModels extends TestModelTest { DISABLED_TESTS.add("SimpleTypes/SimpleDocument"); // Not supported yet DISABLED_TESTS.add("SimpleTypes/SimpleFloat"); // Not supported yet DISABLED_TESTS.add("SimpleTypes/SimpleShort"); // Not supported yet - DISABLED_TESTS.add("Streaming"); // Not supported yet // These are commented out because they should work // They are left here because it can be useful // to have these here so that it is easy to only run a single test locally. diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java index 8f2c61192..9144372f0 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToDafnyLibrary.java @@ -90,7 +90,9 @@ TypeSpec toDafny() { // CollectionError toDafnyMethods.add(collectionError()); // DataStreams - toDafnyMethods.add(dataStream()); + if (ModelUtils.usesStreaming(subject.model)) { + toDafnyMethods.add(dataStream()); + } // Structures subject .getStructuresInServiceNamespace() diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java index 80e06b186..b9031713a 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/smithyjava/generator/library/ToNativeLibrary.java @@ -10,7 +10,6 @@ import com.squareup.javapoet.CodeBlock; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; -import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; import java.util.ArrayList; @@ -20,11 +19,9 @@ import java.util.Set; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; -import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.polymorph.smithydafny.DafnyNameResolver; import software.amazon.polymorph.smithyjava.MethodReference; import software.amazon.polymorph.smithyjava.generator.ToNative; -import software.amazon.polymorph.smithyjava.nameresolver.Constants; import software.amazon.polymorph.smithyjava.nameresolver.Dafny; import software.amazon.polymorph.smithyjava.nameresolver.Native; import software.amazon.polymorph.smithyjava.unmodeled.CollectionOfErrors; @@ -141,7 +138,7 @@ TypeSpec toNative() { .stream() .map(this::modeledResource) .forEachOrdered(toNativeMethods::add); - // The Service, it's self + // The Service itself toNativeMethods.add(modeledService(subject.serviceShape)); return TypeSpec .classBuilder(thisClassName) diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java index 0b8469b54..3a3a9e70c 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java @@ -26,6 +26,7 @@ import software.amazon.smithy.model.traits.EnumTrait; import software.amazon.smithy.model.traits.ErrorTrait; import software.amazon.smithy.model.traits.RequiredTrait; +import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.model.transform.ModelTransformer; @@ -791,4 +792,8 @@ public static StructureShape getConfigShape( StructureShape.class ); } + + public static boolean usesStreaming(final Model model) { + return model.toSet().stream().anyMatch(shape -> shape.hasTrait(StreamingTrait.class)); + } } From db46c44c6302cd718d2280339e2323285a3b3514 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 27 Jun 2025 15:46:41 -0700 Subject: [PATCH 66/72] Typos --- .../main/resources/templates/src/$forLocalService;LIndex.dfy | 2 +- .../main/resources/templates/src/Wrapped$service;LImpl.dfy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy index 9afda92a7..0f4c46470 100644 --- a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy +++ b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/$forLocalService;LIndex.dfy @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 include "$sdkID:LImpl.dfy" -module {:extern "$namespace:L.internaldafny"} $sdkID:L refines Abstract$dafnyModuleName:LService { +module {:extern "$namespace:L.internaldafny" } $sdkID:L refines Abstract$dafnyModuleName:LService { import Operations = $service:LImpl function method Default$serviceConfig:L(): $serviceConfig:L { diff --git a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy index 88f8014ad..85ccdd15d 100644 --- a/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy +++ b/codegen/smithy-dafny-codegen/src/main/resources/templates/src/Wrapped$service;LImpl.dfy @@ -2,9 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 include "../Model/$service:LTypesWrapped.dfy" -module {:extern "$namespace:L.internaldafny.wrapped"} Wrapped$service:LService refines WrappedAbstract$service:LService { +module {:options "--function-syntax:4"} {:extern "$namespace:L.internaldafny.wrapped"} Wrapped$service:LService refines WrappedAbstract$service:LService { import WrappedService = $sdkID:L - function method WrappedDefault$serviceConfig:L(): $serviceConfig:L { + function WrappedDefault$serviceConfig:L(): $serviceConfig:L { $serviceConfig:L } } From 54c962888c7d4e7ba1193be5bfa1269367f1245d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 28 Jun 2025 10:56:45 -0700 Subject: [PATCH 67/72] m --- .../StandardLibrary/runtimes/java/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts index ad167cdef..02ce47a00 100644 --- a/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts +++ b/TestModels/dafny-dependencies/StandardLibrary/runtimes/java/build.gradle.kts @@ -63,7 +63,6 @@ repositories { dependencies { implementation("org.dafny:DafnyRuntime:${dafnyVersion}") implementation("software.amazon.smithy.dafny:conversion:0.1.1") - api("software.amazon.awssdk:sdk-core:2.28.28") } publishing { publications.create("mavenLocal") { From edc311b69845a85bb536dc3f1800fd0583d779b0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 29 Jun 2025 08:33:13 -0700 Subject: [PATCH 68/72] Use compatible Dafny branch --- TestModels/dafny-dependencies/dafny | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TestModels/dafny-dependencies/dafny b/TestModels/dafny-dependencies/dafny index 81dd55397..332fa44fe 160000 --- a/TestModels/dafny-dependencies/dafny +++ b/TestModels/dafny-dependencies/dafny @@ -1 +1 @@ -Subproject commit 81dd553971d0a42f51acfb8745d0b59a02685004 +Subproject commit 332fa44fe38cd6209ac1210ccfef036929ab157b From 2e4017997d9f8c751ab509e877fde7828fe37b95 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Jun 2025 08:33:39 -0700 Subject: [PATCH 69/72] Use prerelease --- .github/workflows/pull.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 99f70da48..d40b93653 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -10,10 +10,10 @@ jobs: steps: - name: Populate Dafny versions list id: populate-dafny-versions-list - run: echo "dafny-versions-list=['4.7.0','4.9.1','4.10.0']" >> $GITHUB_OUTPUT + run: echo "dafny-versions-list=['4.7.0','4.9.1','nightly-2025-06-29-f5ca47b']" >> $GITHUB_OUTPUT - name: Populate Dafny versions list for "only new versions" languages (Python) id: populate-only-new-dafny-versions-list - run: echo "only-new-dafny-versions-list=['4.9.1','4.10.0']" >> $GITHUB_OUTPUT + run: echo "only-new-dafny-versions-list=['4.9.1','nightly-2025-06-29-f5ca47b']" >> $GITHUB_OUTPUT outputs: dafny-version-list: ${{ steps.populate-dafny-versions-list.outputs.dafny-versions-list }} only-new-dafny-version-list: ${{ steps.populate-only-new-dafny-versions-list.outputs.only-new-dafny-versions-list }} From d535c113265673d6f1118ff2b7486141760e952c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 11 Jul 2025 07:29:37 -0700 Subject: [PATCH 70/72] m --- .../test/java/software/amazon/polymorph/TestModelTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index c6c90c9c9..6cfa61239 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -102,5 +102,10 @@ protected void testModels(String relativeTestModelPath) { } else { Assumptions.assumeTrue(needsStreaming); } + + // StreamingSupport isn't directly testable + if (relativeTestModelPath.endsWith("StreamingSupport")) { + Assumptions.assumeTrue(false); + } } } From 17050c00733ec3a3e61de2402e49e05486a333f6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 11 Jul 2025 09:42:41 -0700 Subject: [PATCH 71/72] Formatting --- .../test/java/software/amazon/polymorph/TestModelTest.java | 2 +- .../java/software/amazon/polymorph/utils/ModelUtils.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index 6cfa61239..18664993a 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -102,7 +102,7 @@ protected void testModels(String relativeTestModelPath) { } else { Assumptions.assumeTrue(needsStreaming); } - + // StreamingSupport isn't directly testable if (relativeTestModelPath.endsWith("StreamingSupport")) { Assumptions.assumeTrue(false); diff --git a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java index 3a3a9e70c..1eb72c52f 100644 --- a/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java +++ b/codegen/smithy-dafny-codegen/src/main/java/software/amazon/polymorph/utils/ModelUtils.java @@ -794,6 +794,9 @@ public static StructureShape getConfigShape( } public static boolean usesStreaming(final Model model) { - return model.toSet().stream().anyMatch(shape -> shape.hasTrait(StreamingTrait.class)); + return model + .toSet() + .stream() + .anyMatch(shape -> shape.hasTrait(StreamingTrait.class)); } } From 6b51bf19f6a0714fc16779b216277909cec1eb37 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 23 Jul 2025 13:56:14 -0700 Subject: [PATCH 72/72] Cleanup --- .gitmodules | 2 +- TestModels/Streaming/src/Chunker.dfy | 2 +- ...StandardLibraries-smithy-dafny-subset.toml | 44 ------------------- .../dfyconfig.toml | 39 ++++++++++++++++ .../StreamingSupport/Makefile | 3 +- .../amazon/polymorph/TestModelTest.java | 2 +- 6 files changed, 43 insertions(+), 49 deletions(-) delete mode 100644 TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml create mode 100644 TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset/dfyconfig.toml diff --git a/.gitmodules b/.gitmodules index 7ff076388..0cd030a60 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/smithy-lang/smithy-rs.git [submodule "TestModels/dafny-dependencies/dafny"] path = TestModels/dafny-dependencies/dafny - url = git@github.com:dafny-lang/dafny.git + url = https://github.com/dafny-lang/dafny.git diff --git a/TestModels/Streaming/src/Chunker.dfy b/TestModels/Streaming/src/Chunker.dfy index 14b2dc568..20057c6c1 100644 --- a/TestModels/Streaming/src/Chunker.dfy +++ b/TestModels/Streaming/src/Chunker.dfy @@ -309,7 +309,7 @@ module {:options "--function-syntax:4"} Chunker { } function BinaryOfNumber(x: int32): seq { - // TODO: Actually compute the binary + // Don't actually need to compute the binary properly here [12 as uint8, 34, 56] } diff --git a/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml b/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml deleted file mode 100644 index 7b7b61f73..000000000 --- a/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset.toml +++ /dev/null @@ -1,44 +0,0 @@ -includes = [ - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Actions.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/BulkActions.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Producers.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Consumers.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Actions/GenericAction.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/BoundedInts.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Seq.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Set.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Multiset.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/DynamicArray.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Frames.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Functions.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Math.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Ordinal.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Relations.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Termination.dfy", - "../dafny/Source/DafnyStandardLibraries/src/Std/Wrappers.dfy" -] - -[options] -# Options that help support more values of consuming context options -# (mostly turning on any more restrictive verification) -track-print-effects = true -enforce-determinism = true -reads-clauses-on-methods = true -type-system-refresh = true -general-traits = "datatype" -general-newtypes = true - -# Options that differ from Dafny's -unicode-char = false - -# TODO: Required because this file isn't named dfyconfig.toml :P -allow-warnings = true - -hidden-no-verify = true - -# Options important for sustainable development -# of the libraries themselves. -resource-limit = 1000000 -# These give too many false positives right now, but should be enabled in the future -warn-redundant-assumptions = false -warn-contradictory-assumptions = false diff --git a/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset/dfyconfig.toml b/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset/dfyconfig.toml new file mode 100644 index 000000000..69ec290de --- /dev/null +++ b/TestModels/dafny-dependencies/StreamingSupport/DafnyStandardLibraries-smithy-dafny-subset/dfyconfig.toml @@ -0,0 +1,39 @@ +includes = [ + "../../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Actions.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Actions/BulkActions.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Producers.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Actions/Consumers.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Actions/GenericAction.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/BoundedInts.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Seq.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Set.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Collections/Multiset.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/DynamicArray.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Frames.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Functions.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Math.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Ordinal.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Relations.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Termination.dfy", + "../../dafny/Source/DafnyStandardLibraries/src/Std/Wrappers.dfy" +] + +[options] +# Options that help support more values of consuming context options +# (mostly turning on any more restrictive verification) +track-print-effects = true +enforce-determinism = true +reads-clauses-on-methods = true +type-system-refresh = true +general-traits = "datatype" +general-newtypes = true + +# Options that differ from Dafny's +unicode-char = false + +# Options important for sustainable development +# of the libraries themselves. +resource-limit = 1000000 +# These give too many false positives right now, but should be enabled in the future +warn-redundant-assumptions = false +warn-contradictory-assumptions = false diff --git a/TestModels/dafny-dependencies/StreamingSupport/Makefile b/TestModels/dafny-dependencies/StreamingSupport/Makefile index 7284aa1e4..18d6dee0a 100644 --- a/TestModels/dafny-dependencies/StreamingSupport/Makefile +++ b/TestModels/dafny-dependencies/StreamingSupport/Makefile @@ -27,8 +27,7 @@ polymorph_dafny: build_dafny_stdlibs "; build_dafny_stdlibs: - dafny build -t:lib --output bin/DafnyStandardLibraries-smithy-dafny-subset.doo DafnyStandardLibraries-smithy-dafny-subset.toml - rm -rf DafnyStandardLibraries-smithy-dafny-subset-lib + dafny build -t:lib --output bin/DafnyStandardLibraries-smithy-dafny-subset.doo DafnyStandardLibraries-smithy-dafny-subset/dfyconfig.toml _polymorph_dependencies: echo "Skipping _polymorph_dependencies for StreamingSupport" \ No newline at end of file diff --git a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java index 18664993a..dbd275d85 100644 --- a/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java +++ b/codegen/smithy-dafny-codegen-test/src/test/java/software/amazon/polymorph/TestModelTest.java @@ -91,7 +91,7 @@ protected Path getTestModelPath(String relativeTestModelPath) { protected void testModels(String relativeTestModelPath) { // The @streaming support depends on our subset of the Dafny standard libraries // which cannot be built for old versions of Dafny. - // We also don't want to burn a lot of CI cycles testing all models against Java 4.10 yet. + // We also don't want to burn a lot of CI cycles testing all models against Dafny 4.10 yet. DafnyVersion dafnyVersion = CodegenEngine.getDafnyVersionFromDafny(); boolean needsStreaming = relativeTestModelPath.endsWith("Streaming") ||