From 7841969b8a2af662144aaaf002026759b161bf7c Mon Sep 17 00:00:00 2001 From: Sam Bisciglia Date: Mon, 15 Dec 2025 15:03:24 -0500 Subject: [PATCH] updates to the S3 Encryption Client --------- Co-authored-by: Andy Jewell Co-authored-by: Ryan Emery --- .../core/utils/crypto/ContentCryptoMaterial.h | 92 +++ .../core/utils/crypto/ContentCryptoScheme.h | 7 +- .../utils/crypto/ContentCryptoMaterial.cpp | 7 + .../utils/crypto/ContentCryptoScheme.cpp | 14 +- .../source/utils/crypto/KeyWrapAlgorithm.cpp | 12 +- .../SUPPORT_POLICY.rst | 34 + .../aws/s3-encryption/CryptoConfiguration.h | 93 ++- .../include/aws/s3-encryption/HKDF.h | 49 ++ .../aws/s3-encryption/S3EncryptionClient.h | 94 ++- .../aws/s3-encryption/handlers/DataHandler.h | 57 +- .../handlers/InstructionFileHandler.h | 12 +- .../s3-encryption/handlers/MetadataHandler.h | 13 +- .../aws/s3-encryption/modules/CryptoModule.h | 10 +- .../s3-encryption/CryptoConfiguration.cpp | 63 +- .../source/s3-encryption/HKDF.cpp | 103 ++++ .../s3-encryption/S3EncryptionClient.cpp | 163 ++++- .../s3-encryption/handlers/DataHandler.cpp | 191 +++++- .../handlers/InstructionFileHandler.cpp | 91 ++- .../handlers/MetadataHandler.cpp | 70 ++- .../materials/KMSEncryptionMaterials.cpp | 73 ++- .../materials/SimpleEncryptionMaterials.cpp | 35 +- .../s3-encryption/modules/CryptoModule.cpp | 104 +++- .../LiveClientTests.cpp | 580 +++++++++++++++++- .../DataHandlersTest.cpp | 139 ++++- 24 files changed, 1971 insertions(+), 135 deletions(-) create mode 100644 src/aws-cpp-sdk-s3-encryption/SUPPORT_POLICY.rst create mode 100644 src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/HKDF.h create mode 100644 src/aws-cpp-sdk-s3-encryption/source/s3-encryption/HKDF.cpp diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoMaterial.h b/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoMaterial.h index ea997988539..45f18535ef6 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoMaterial.h +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoMaterial.h @@ -35,6 +35,11 @@ namespace Aws */ ContentCryptoMaterial(const Aws::Utils::CryptoBuffer& cek, ContentCryptoScheme contentCryptoScheme); + /* + Initialize in the error state. + */ + ContentCryptoMaterial(const char * msg); + /** * Gets the underlying content encryption key. */ @@ -59,6 +64,30 @@ namespace Aws return m_iv; } + /** + * Gets the underlying encryption context + */ + inline const Aws::Map& GetEncryptionContext() const + { + return m_encryptionContext; + } + + /** + * Gets the underlying key commitment + */ + inline const Aws::Utils::CryptoBuffer& GetKeyCommitment() const + { + return m_keyCommitment; + } + + /** + * Gets the underlying message ID + */ + inline const Aws::Utils::CryptoBuffer& GetMessageID() const + { + return m_messageId; + } + /** * Gets the underlying crypto tag length */ @@ -123,6 +152,43 @@ namespace Aws m_iv = iv; } + /** + * Sets the underlying iv to 12 bytes of zero, as needed for V3 encoding + */ + inline void SetV3IV() + { + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + //# The IV's total length MUST match the IV length defined by the algorithm suite. + unsigned char iv[12] = {0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}; + CryptoBuffer iv2(&iv[0], 12); + SetIV(iv2); + } + + /** + * Sets the underlying encryption context. Copies from parameter encryptionContext. + */ + inline void SetEncryptionContext(const Aws::Map& encryptionContext) + { + m_encryptionContext = encryptionContext; + } + + /** + * Sets the underlying key commitment. Copies from parameter keyCommitment. + */ + inline void SetKeyCommitment(const Aws::Utils::CryptoBuffer& keyCommitment) + { + m_keyCommitment = keyCommitment; + } + + /** + * Sets the underlying message ID. Copies from parameter messageId. + */ + inline void SetMessageID(const Aws::Utils::CryptoBuffer& messageId) + { + m_messageId = messageId; + } + /** * Sets the underlying crypto Tag Length. Copies from parameter cryptoTagLength. */ @@ -223,6 +289,28 @@ namespace Aws return m_finalCEK; } + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=implication + //# The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + //= type=implication + //# The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + inline const Aws::Utils::CryptoBuffer GetAAD() const + { + if (m_contentCryptoScheme == ContentCryptoScheme::GCM_COMMIT) { + // Algorithm Suite 0x0073 as bytes + static const uint8_t gcmAAD[2] = {0, 0x73}; + return Aws::Utils::CryptoBuffer(gcmAAD, 2); + } else { + return Aws::Utils::CryptoBuffer(); + } + } + + inline bool Ok() const {return m_error.empty();} + inline bool Fail() const {return !m_error.empty();} + inline const Aws::String & Error() const {return m_error;} + private: Aws::Utils::CryptoBuffer m_contentEncryptionKey; Aws::Utils::CryptoBuffer m_encryptedContentEncryptionKey; @@ -234,10 +322,14 @@ namespace Aws Aws::Utils::CryptoBuffer m_cekIV; Aws::Utils::CryptoBuffer m_gcmAAD; Aws::Utils::CryptoBuffer m_cekGCMTag; + Aws::Map m_encryptionContext; + Aws::Utils::CryptoBuffer m_keyCommitment; + Aws::Utils::CryptoBuffer m_messageId; size_t m_cryptoTagLength; Aws::Map m_materialsDescription; KeyWrapAlgorithm m_keyWrapAlgorithm; ContentCryptoScheme m_contentCryptoScheme; + Aws::String m_error; }; } } diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoScheme.h b/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoScheme.h index 605ad4e3757..ca619a32dec 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoScheme.h +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/crypto/ContentCryptoScheme.h @@ -17,9 +17,12 @@ namespace Aws CBC, CTR, GCM, + GCM_COMMIT, NONE }; - + inline bool IsGCM(ContentCryptoScheme scheme) { + return scheme == ContentCryptoScheme::GCM || scheme == ContentCryptoScheme::GCM_COMMIT; + } namespace ContentCryptoSchemeMapper { AWS_CORE_API ContentCryptoScheme GetContentCryptoSchemeForName(const Aws::String& name); @@ -29,4 +32,4 @@ namespace Aws } //namespace Crypto }//namespace Utils -}//namespace Aws \ No newline at end of file +}//namespace Aws diff --git a/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoMaterial.cpp b/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoMaterial.cpp index 3036bd70eb0..6a0b7dfca8e 100644 --- a/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoMaterial.cpp +++ b/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoMaterial.cpp @@ -18,6 +18,13 @@ namespace Aws { } + ContentCryptoMaterial::ContentCryptoMaterial(const char * msg) : + m_cryptoTagLength(0), m_keyWrapAlgorithm(KeyWrapAlgorithm::NONE), m_contentCryptoScheme(ContentCryptoScheme::NONE), + m_error(msg) + { + AWS_LOGSTREAM_ERROR("DataHandler", msg); + } + ContentCryptoMaterial::ContentCryptoMaterial(ContentCryptoScheme contentCryptoScheme) : m_contentEncryptionKey(SymmetricCipher::GenerateKey()), m_cryptoTagLength(0), m_keyWrapAlgorithm(KeyWrapAlgorithm::NONE), m_contentCryptoScheme(contentCryptoScheme) { diff --git a/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoScheme.cpp b/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoScheme.cpp index f39a75df2c8..f009ea78927 100644 --- a/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoScheme.cpp +++ b/src/aws-cpp-sdk-core/source/utils/crypto/ContentCryptoScheme.cpp @@ -20,6 +20,8 @@ namespace Aws static const int cryptoScheme_CBC_HASH = HashingUtils::HashString("AES/CBC/PKCS5Padding"); static const int cryptoScheme_CTR_HASH = HashingUtils::HashString("AES/CTR/NoPadding"); static const int cryptoScheme_GCM_HASH = HashingUtils::HashString("AES/GCM/NoPadding"); + // "115" represents ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (0x0073) + static const int cryptoScheme_COMMIT_HASH = HashingUtils::HashString("115"); ContentCryptoScheme GetContentCryptoSchemeForName(const Aws::String& name) { @@ -36,8 +38,12 @@ namespace Aws { return ContentCryptoScheme::GCM; } - assert(0); - return ContentCryptoScheme::NONE; + else if (hashcode == cryptoScheme_COMMIT_HASH) + { + return ContentCryptoScheme::GCM_COMMIT; + } + // Return NONE for unrecognized schemes instead of asserting + return ContentCryptoScheme::NONE; } Aws::String GetNameForContentCryptoScheme(ContentCryptoScheme enumValue) @@ -50,6 +56,8 @@ namespace Aws return "AES/CTR/NoPadding"; case ContentCryptoScheme::GCM: return "AES/GCM/NoPadding"; + case ContentCryptoScheme::GCM_COMMIT: + return "115"; default: assert(0); return ""; @@ -58,4 +66,4 @@ namespace Aws }//namespace ContentCryptoSchemeMapper } //namespace Crypto }//namespace Utils -}//namespace Aws \ No newline at end of file +}//namespace Aws diff --git a/src/aws-cpp-sdk-core/source/utils/crypto/KeyWrapAlgorithm.cpp b/src/aws-cpp-sdk-core/source/utils/crypto/KeyWrapAlgorithm.cpp index b9e098775c9..6ff7cb33398 100644 --- a/src/aws-cpp-sdk-core/source/utils/crypto/KeyWrapAlgorithm.cpp +++ b/src/aws-cpp-sdk-core/source/utils/crypto/KeyWrapAlgorithm.cpp @@ -37,12 +37,12 @@ namespace Aws { return KeyWrapAlgorithm::AES_KEY_WRAP; } - else if (hashcode == keyWrapAlgorithm_AES_GCM_HASH) - { - return KeyWrapAlgorithm::AES_GCM; - } - assert(0); - return KeyWrapAlgorithm::NONE; + else if (hashcode == keyWrapAlgorithm_AES_GCM_HASH) + { + return KeyWrapAlgorithm::AES_GCM; + } + // Return NONE for unrecognized algorithms instead of asserting + return KeyWrapAlgorithm::NONE; } Aws::String GetNameForKeyWrapAlgorithm(KeyWrapAlgorithm enumValue) diff --git a/src/aws-cpp-sdk-s3-encryption/SUPPORT_POLICY.rst b/src/aws-cpp-sdk-s3-encryption/SUPPORT_POLICY.rst new file mode 100644 index 00000000000..c88528667b9 --- /dev/null +++ b/src/aws-cpp-sdk-s3-encryption/SUPPORT_POLICY.rst @@ -0,0 +1,34 @@ +Overview +======== +This page describes the support policy for the S3EncryptionClient. +We regularly provide the S3EncryptionClient with updates that may contain support for new or updated APIs, new features, enhancements, bug fixes, security patches, or documentation updates. Updates may also address changes with dependencies, and operating systems. + +We recommend users to stay up-to-date with S3EncryptionClient to keep up with the latest features, security updates, and underlying dependencies. Continued use of an unsupported S3EncryptionClient version is not recommended and is done at the user’s discretion + + +Major Version Lifecycle +======================== +The S3 Encryption Client uses separately named classes for new major versions. + +Version Support Matrix +=============================== +This table describes the current support status of each major version of the S3EncryptionClient. It also shows the next status each major version will transition to, and the date at which that transition will happen. + +.. list-table:: + :widths: 30 50 50 + :header-rows: 1 + + * - Major version + - Current status + - Next status + * - S3EncryptionClient + - End Of Support + - + * - S3EncryptionClientV2 + - Maintenance + - End Of Support + * - S3EncryptionClientV3 + - Generally Available + - + +.. _AWS SDKs and Tools Maintenance Policy: https://docs.aws.amazon.com/sdkref/latest/guide/maint-policy.html#version-life-cycle diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/CryptoConfiguration.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/CryptoConfiguration.h index c17bcb4329d..661f980afdd 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/CryptoConfiguration.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/CryptoConfiguration.h @@ -43,16 +43,34 @@ namespace Aws enum class SecurityProfile { - V2, // Client only decrypt objects encrypted using best practice key wrap algorithms (KMS_CONTEXT and AES_GCM ) and best practice content crypto schemes (AES_GCM) - V2_AND_LEGACY, // Client will try to decrypt objects encrypted using all key wrap algorithms (KMS_CONTEXT, KMS, AES_KeyWrap, AES_GCM) and content crypto schemes (AES_GCM, AES_CBC). + V2, // Client decrypt objects encrypted using best practice key wrap algorithms (KMS_CONTEXT and AES_GCM ) and best practice content crypto schemes (AES_GCM), but does not require Key Commitment. + V2_AND_LEGACY // Client will try to decrypt objects encrypted using all key wrap algorithms (KMS_CONTEXT, KMS, AES_KeyWrap, AES_GCM) and content crypto schemes (AES_GCM, AES_CBC), and does not require Key Commitment. }; + enum class CommitmentPolicy { + FORBID_ENCRYPT_ALLOW_DECRYPT, // Encrypt as V2, decrypt any + REQUIRE_ENCRYPT_ALLOW_DECRYPT, // Encrypt as V3, decrypt any + REQUIRE_ENCRYPT_REQUIRE_DECRYPT // Encrypt as V3, decrypt only V3 + }; + + //= ../specification/s3-encryption/decryption.md#ranged-gets + //= type=implication + //# The S3EC MAY support the "range" parameter on GetObject which specifies a subset of bytes to download and decrypt. enum class RangeGetMode { DISABLED, // Range get is not allowed ALL, // Range get is allowed }; + enum class AlgorithmSuite { + AES_GCM, + // AES_GCM_WITH_COMMITMENT is AES_GCM, but with key commitment. + // That is, the encrypted key is used to derive both an encryption key and a commitment key + // The commitment key is stored in the meta data. + // On decrypt, commitment key is derived and must match the stored commitment key. + AES_GCM_WITH_COMMITMENT + }; + class AWS_S3ENCRYPTION_API CryptoConfiguration { public: @@ -92,6 +110,22 @@ namespace Aws return m_cryptoMode; } + /** + * Gets the underlying AlgorithmSuite. + */ + inline AlgorithmSuite GetEncryptionAlgorithm() const + { + return m_algorithmSuite; + } + + /** + * Gets the underlying CommitmentPolicy. + */ + inline CommitmentPolicy GetCommitmentPolicy() const + { + return m_commitmentPolicy; + } + /** * Sets the underlying storage method. Copies from parameter storageMethod. */ @@ -113,14 +147,19 @@ namespace Aws inline void SetUnAuthenticatedRangeGet(RangeGetMode mode) { m_unAuthenticatedRangeGet = mode; } inline RangeGetMode GetUnAuthenticatedRangeGet() const { return m_unAuthenticatedRangeGet; } inline SecurityProfile GetSecurityProfile() const { return m_securityProfile; } + inline void SetCommitmentPolicy(CommitmentPolicy commitmentPolicy) { m_commitmentPolicy = commitmentPolicy; } + inline void SetEncryptionAlgorithm(AlgorithmSuite algorithmSuite) { m_algorithmSuite = algorithmSuite; } StorageMethod m_storageMethod; CryptoMode m_cryptoMode; RangeGetMode m_unAuthenticatedRangeGet; SecurityProfile m_securityProfile; + AlgorithmSuite m_algorithmSuite; + CommitmentPolicy m_commitmentPolicy; friend class S3EncryptionClientBase; friend class S3EncryptionClientV2; + friend class S3EncryptionClientV3; }; class AWS_S3ENCRYPTION_API CryptoConfigurationV2 @@ -137,12 +176,62 @@ namespace Aws inline RangeGetMode GetUnAuthenticatedRangeGet() const { return m_unAuthenticatedRangeGet; } inline StorageMethod GetStorageMethod() const { return m_storageMethod; } std::shared_ptr GetEncryptionMaterials() const { return m_encryptionMaterials; } + private: + StorageMethod m_storageMethod; + RangeGetMode m_unAuthenticatedRangeGet; + SecurityProfile m_securityProfile; + std::shared_ptr m_encryptionMaterials; + }; + + //= ../specification/s3-encryption/client.md#aws-sdk-compatibility + //= type=implication + //# The S3EC MUST provide a different set of configuration options than the conventional S3 client. + class AWS_S3ENCRYPTION_API CryptoConfigurationV3 + { + public: + CryptoConfigurationV3(const std::shared_ptr& materials); + CryptoConfigurationV3(const std::shared_ptr& materials); + + inline void AllowLegacy(bool allow = true) { m_securityProfile = (allow ? SecurityProfile::V2_AND_LEGACY : SecurityProfile::V2); } + inline void SetCommitmentPolicy(CommitmentPolicy commitmentPolicy) { m_commitmentPolicy = commitmentPolicy; } + inline void SetUnAuthenticatedRangeGet(RangeGetMode mode) { m_unAuthenticatedRangeGet = mode; } + inline void SetStorageMethod(StorageMethod storageMethod) { m_storageMethod = storageMethod; } + + inline bool GetAllowLegacy() const { return m_securityProfile == SecurityProfile::V2_AND_LEGACY; } + inline RangeGetMode GetUnAuthenticatedRangeGet() const { return m_unAuthenticatedRangeGet; } + inline StorageMethod GetStorageMethod() const { return m_storageMethod; } + inline CommitmentPolicy GetCommitmentPolicy() const { return m_commitmentPolicy; } + inline SecurityProfile GetSecurityProfile() const { return m_securityProfile; } + std::shared_ptr GetEncryptionMaterials() const { return m_encryptionMaterials; } private: + //= ../specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms + //= type=implication + //# The S3EC MUST support the option to enable or disable legacy wrapping algorithms. + + //= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + //= type=implication + //# The S3EC MUST support the option to enable or disable legacy unauthenticated modes (content encryption algorithms). + + //= ../specification/s3-encryption/client.md#enable-delayed-authentication + //= type=implication + //# The S3EC MUST support the option to enable or disable Delayed Authentication mode. + + //= ../specification/s3-encryption/client.md#instruction-file-configuration + //= type=implication + //# The S3EC MAY support the option to provide Instruction File Configuration during its initialization. + //# If the S3EC in a given language supports Instruction Files, then it MUST accept Instruction File Configuration during its initialization. + //# In this case, the Instruction File Configuration SHOULD be optional, such that its default configuration is used when none is provided. + + //= ../specification/s3-encryption/client.md#key-commitment + //= type=implication + //# The S3EC MUST support configuration of the [Key Commitment policy](./key-commitment.md) during its initialization. + StorageMethod m_storageMethod; RangeGetMode m_unAuthenticatedRangeGet; SecurityProfile m_securityProfile; std::shared_ptr m_encryptionMaterials; + CommitmentPolicy m_commitmentPolicy; }; } } diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/HKDF.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/HKDF.h new file mode 100644 index 00000000000..94270b29385 --- /dev/null +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/HKDF.h @@ -0,0 +1,49 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once +#include +#include + +#include + +namespace Aws +{ + namespace S3Encryption + { + constexpr size_t MESSAGE_ID_BYTES = 28; + constexpr size_t COMMITMENT_KEY_BYTES = 28; + constexpr size_t ENCRYPTION_KEY_BYTES = 32; + + extern const char *ENCRYPTION_KEY_INFO; + extern const char *COMMITMENT_KEY_INFO; + extern const size_t ENCRYPTION_KEY_INFO_LEN; + extern const size_t COMMITMENT_KEY_INFO_LEN; + + /** + Derive an encryption key from a given data key and message ID using HKDF-SHA512. + @param data_key Input data key (symmetric key material). + @param message_id Unique per-message identifier. + @param output Output buffer to receive the derived key. Must be pre-sized appropriately. + @return true on success, false on failure. + */ + AWS_S3ENCRYPTION_API bool derive_encryption_key(const Aws::Utils::CryptoBuffer &data_key, const Aws::Utils::CryptoBuffer &message_id, + Aws::Utils::CryptoBuffer &output); + + + /** + Derive an encryption key from a given data key and message ID using HKDF-SHA512. + @param data_key Input data key (symmetric key material). + @param message_id Unique per-message identifier. + @param output Output buffer to receive the derived key. Must be pre-sized appropriately. + @return true on success, false on failure. + */ + AWS_S3ENCRYPTION_API bool derive_commitment_key(const Aws::Utils::CryptoBuffer &data_key, const Aws::Utils::CryptoBuffer &message_id, + Aws::Utils::CryptoBuffer &output); + + // Compare bytes for equality, in time O(len) regardless of the contents of x and y + // Returns false if the buffers have different lengths + AWS_S3ENCRYPTION_API bool constant_time_equal(const Aws::Utils::CryptoBuffer& x, const Aws::Utils::CryptoBuffer& y); + } +} diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/S3EncryptionClient.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/S3EncryptionClient.h index 7b97acf4c53..32c9a9a4f6e 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/S3EncryptionClient.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/S3EncryptionClient.h @@ -82,6 +82,10 @@ namespace Aws * For KMSWithContext encryption materials, you can provide a context map as the KMS context for encrypting the CEK. * For other encryption materials, this context map must be an empty map. */ + //= ../specification/s3-encryption/client.md#required-api-operations + //= type=implication + //# - PutObject MUST be implemented by the S3EC. + //# - PutObject MUST encrypt its input data before it is uploaded to S3. S3EncryptionPutObjectOutcome PutObject(const Aws::S3::Model::PutObjectRequest& request, const Aws::Map& contextMap) const; /* @@ -90,6 +94,10 @@ namespace Aws * Range gets using this method are deprecated. Please see * for more information */ + //= ../specification/s3-encryption/client.md#required-api-operations + //= type=implication + //# - GetObject MUST be implemented by the S3EC. + //# - GetObject MUST decrypt data received from the S3 server and return it as plaintext. S3EncryptionGetObjectOutcome GetObject(const Aws::S3::Model::GetObjectRequest& request) const; /* @@ -103,6 +111,10 @@ namespace Aws inline bool MultipartUploadSupported() const { return false; } protected: + bool ValidateStorageMethod(StorageMethod s); + bool ValidateSecurityProfile(SecurityProfile s); + bool ValidateCommitmentPolicy(CommitmentPolicy s); + bool ValidateRangeGetMode(RangeGetMode s); /* * GetObject with optional contextMap. * Fail if contextMap is supplied and does not exactly match stored Materials Description @@ -118,13 +130,14 @@ namespace Aws Aws::S3Encryption::Modules::CryptoModuleFactory m_cryptoModuleFactory; std::shared_ptr m_encryptionMaterials; Aws::S3Encryption::CryptoConfiguration m_cryptoConfig; + Aws::String m_error; }; /** - * @deprecated This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV2. + * @deprecated This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV3. */ class - AWS_DEPRECATED("This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV2. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information.") + AWS_DEPRECATED("This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV3. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information.") AWS_S3ENCRYPTION_API S3EncryptionClient : public S3EncryptionClientBase { public: @@ -165,7 +178,12 @@ namespace Aws * S3EncryptionClientV2 enforce the secure postures for customers. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html * Examples: https://github.com/awsdocs/aws-doc-sdk-examples/blob/master/cpp/example_code/s3encryption/s3Encryption.cpp */ - class AWS_S3ENCRYPTION_API S3EncryptionClientV2 : public S3EncryptionClientBase + /** + * @deprecated This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV3. + */ + class + AWS_DEPRECATED("This class is in the maintenance mode, no new updates will be released, use S3EncryptionClientV3. Please see https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html for more information.") + AWS_S3ENCRYPTION_API S3EncryptionClientV2 : public S3EncryptionClientBase { public: /* @@ -219,5 +237,75 @@ namespace Aws private: void Init(const Aws::S3Encryption::CryptoConfigurationV2& cryptoConfig); }; + + class AWS_S3ENCRYPTION_API S3EncryptionClientV3 : public S3EncryptionClientBase + { + public: + /* + * Initialize the S3 Encryption Client V3 with crypto configuration v3, and a client configuration. If no client configuration is supplied, + * the default client configuration will be used. + */ + //= ../specification/s3-encryption/client.md#cryptographic-materials + //= type=implication + //# The S3EC MAY accept key material directly. + + //= ../specification/s3-encryption/client.md#inherited-sdk-configuration + //= type=implication + //# The S3EC MAY support directly configuring the wrapped SDK clients through its initialization. + //# For example, the S3EC MAY accept a credentials provider instance during its initialization. + //# If the S3EC accepts SDK client configuration, the configuration MUST be applied to all wrapped S3 clients. + // There's only one client, so it's safe to say "all" + S3EncryptionClientV3(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig, + const Aws::Client::ClientConfiguration& clientConfig = Aws::Client::ClientConfiguration()) + : S3EncryptionClientBase(cryptoConfig.GetEncryptionMaterials(), CryptoConfiguration(), clientConfig) + { + Init(cryptoConfig); + } + + + /* + * Initialize the S3 Encryption Client V3 with crypto configuration v3, AWS credentials and a client configuration. If no client configuration is supplied, + * the default client configuration will be used. + */ + S3EncryptionClientV3(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig, const Aws::Auth::AWSCredentials& credentials, + const Aws::Client::ClientConfiguration& clientConfig = Aws::Client::ClientConfiguration()) + : S3EncryptionClientBase(cryptoConfig.GetEncryptionMaterials(), CryptoConfiguration(), credentials, clientConfig) + { + Init(cryptoConfig); + } + + /* + * Initialize the S3 Encryption Client V3 with crypto configuration v3, AWS credentials provider and a client configuration. If no client configuration is supplied, + * the default client configuration will be used. + */ + S3EncryptionClientV3(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig, const std::shared_ptr& credentialsProvider, + const Aws::Client::ClientConfiguration& clientConfig = Aws::Client::ClientConfiguration()) + : S3EncryptionClientBase(cryptoConfig.GetEncryptionMaterials(), CryptoConfiguration(), credentialsProvider, clientConfig) + { + Init(cryptoConfig); + } + + /* + * Initialize the S3 Encryption Client V3 with crypto configuration v3, and a s3 client factory. + * The factory will be used to create the underlying S3 Client. + */ + //= ../specification/s3-encryption/client.md#wrapped-s3-client-s + //= type=implication + //# The S3EC MUST support the option to provide an SDK S3 client instance during its initialization. + //# The S3EC MUST NOT support use of S3EC as the provided S3 client during its initialization; it MUST throw an exception in this case. + // The S3EncryptionClientV3 is not an S3Client, so this can't happen. + S3EncryptionClientV3(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig, + const std::function ()>& s3ClientFactory) + : S3EncryptionClientBase(cryptoConfig.GetEncryptionMaterials(), CryptoConfiguration(), s3ClientFactory) + { + Init(cryptoConfig); + } + + S3EncryptionClientV3(const S3EncryptionClientV3&) = delete; + S3EncryptionClientV3& operator=(const S3EncryptionClientV3&) = delete; + + private: + void Init(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig); + }; } } diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/DataHandler.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/DataHandler.h index e4c1da9d7ec..8ddaec54232 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/DataHandler.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/DataHandler.h @@ -23,6 +23,52 @@ namespace Aws static const char* const CRYPTO_TAG_LENGTH_HEADER = "x-amz-tag-len"; static const char* const KEY_WRAP_ALGORITHM = "x-amz-wrap-alg"; static const char* const INSTRUCTION_FILE_HEADER = "x-amz-crypto-instr-file"; + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-c") SHOULD be represented by a constant named "CONTENT_CIPHER_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-3") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-m") SHOULD be represented by a constant named "MAT_DESC_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-t") SHOULD be represented by a constant named "ENCRYPTION_CONTEXT_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-w") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_ALGORITHM_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-d") SHOULD be represented by a constant named "KEY_COMMITMENT_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - This mapkey ("x-amz-i") SHOULD be represented by a constant named "MESSAGE_ID_V3" or similar in the implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# The "x-amz-meta-" prefix is automatically added by the S3 server and MUST NOT be included in implementation code. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# The "x-amz-" prefix denotes that the metadata is owned by an Amazon product and MUST be prepended to all S3EC metadata mapkeys. + + + static const char* const CONTENT_CIPHER_V3 = "x-amz-c"; // same as CONTENT_CRYPTO_SCHEME_HEADER + static const char* const ENCRYPTED_DATA_KEY_V3 = "x-amz-3"; // same as DEPRECATED_CONTENT_KEY_HEADER or CONTENT_KEY_HEADER + static const char* const MAT_DESC_V3 = "x-amz-m"; // same as MATERIALS_DESCRIPTION_HEADER + static const char* const ENCRYPTION_CONTEXT_V3 = "x-amz-t"; + static const char* const ENCRYPTED_DATA_KEY_ALGORITHM_V3 = "x-amz-w"; // same as KEY_WRAP_ALGORITHM, but different encoding + static const char* const KEY_COMMITMENT_V3 = "x-amz-d"; + static const char* const MESSAGE_ID_V3 = "x-amz-i"; + static const size_t AES_GCM_IV_BYTES = 12; static const size_t AES_GCM_KEY_BYTES = 32; static const size_t AES_GCM_TAG_BYTES = 16; @@ -30,6 +76,7 @@ namespace Aws namespace Handlers { + Aws::String V2ToV3Alg(const Aws::String & v2); /* Data handler class will be responsible for reading and writing metadata and instruction files to and from S3 object using a Put object request or a Get object result. @@ -37,16 +84,6 @@ namespace Aws class AWS_S3ENCRYPTION_API DataHandler { public: - /* - Override this function to write content crypto material data to S3 object request. - */ - virtual void PopulateRequest(Aws::S3::Model::PutObjectRequest& request, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial) = 0; - - /* - Override this function to read data from an S3 object and return a Content Crypto Material object. - */ - virtual Aws::Utils::Crypto::ContentCryptoMaterial ReadContentCryptoMaterial(Aws::S3::Model::GetObjectResult& result) = 0; - /* Function to json serialize a map containing pairs of strings. */ diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/InstructionFileHandler.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/InstructionFileHandler.h index 8786667a72d..4841abaedb0 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/InstructionFileHandler.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/InstructionFileHandler.h @@ -11,6 +11,11 @@ namespace Aws { namespace Handlers { + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //= type=exception + //# The S3EC MUST NOT support providing a custom Instruction File suffix on ordinary writes; custom suffixes MUST only be used during re-encryption. + // We do not support a custom Instruction File suffix under any circumstances. + static const char* const DEFAULT_INSTRUCTION_FILE_SUFFIX = ".instruction"; /* Instruction file handler will be responsible for reading and writing instruction files to and from S3 object using a Put object @@ -22,12 +27,15 @@ namespace Aws /* Write ContentCryptoMaterial data to an instruction file object which is passed as an argument for this function. */ - void PopulateRequest(Aws::S3::Model::PutObjectRequest& request, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial) override; + void PopulateRequest(Aws::S3::Model::PutObjectRequest& objRequest, Aws::S3::Model::PutObjectRequest& ifRequest, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial); /* Read data from an instruction file object and return a Content Crypto Material object. */ - Aws::Utils::Crypto::ContentCryptoMaterial ReadContentCryptoMaterial(Aws::S3::Model::GetObjectResult& result) override; + Aws::Utils::Crypto::ContentCryptoMaterial ReadContentCryptoMaterial(const Aws::S3::Model::GetObjectResult& instruction_file, const Aws::S3::Model::HeadObjectResult& object_head); + private: + // Helper for writing V3-format instruction files. + void PopulateRequestV3(Aws::S3::Model::PutObjectRequest& objRequest, Aws::S3::Model::PutObjectRequest& ifRequest, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial); }; } } diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/MetadataHandler.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/MetadataHandler.h index e17b80677f7..04e5cca35fd 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/MetadataHandler.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/handlers/MetadataHandler.h @@ -4,7 +4,6 @@ */ #pragma once -#include #include namespace Aws @@ -23,17 +22,17 @@ namespace Aws /* * Write ContentCryptoMaterial data to a put object request. This occurs in place. */ - void PopulateRequest(Aws::S3::Model::PutObjectRequest& request, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial) override; - - /* - * Read the metadata of a GetObject result and store into a ContentCryptoMaterial Object. - */ - Aws::Utils::Crypto::ContentCryptoMaterial ReadContentCryptoMaterial(Aws::S3::Model::GetObjectResult& result) override; + void PopulateRequest(Aws::S3::Model::PutObjectRequest& objRequest, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial); /* * Read the metadata of a HeadObject result and store into a ContentCryptoMaterialObject. */ Aws::Utils::Crypto::ContentCryptoMaterial ReadContentCryptoMaterial(const Aws::S3::Model::HeadObjectResult& result); + private: + /* + * Write V3 ContentCryptoMaterial data to a put object request. This occurs in place. + */ + void PopulateRequestV3(Aws::S3::Model::PutObjectRequest& objRequest, const Aws::Utils::Crypto::ContentCryptoMaterial& contentCryptoMaterial); }; } } diff --git a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/modules/CryptoModule.h b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/modules/CryptoModule.h index 7622a412fa7..747a8388ffb 100644 --- a/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/modules/CryptoModule.h +++ b/src/aws-cpp-sdk-s3-encryption/include/aws/s3-encryption/modules/CryptoModule.h @@ -85,7 +85,7 @@ namespace Aws /** * This function initializes the cipher for decryption with the content encryption key, iv and tag. */ - virtual void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer()) = 0; + virtual void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer(), const Aws::Utils::CryptoBuffer& aad = Aws::Utils::CryptoBuffer()) = 0; /* * This function populates the content crypto material with the module specific details for encryption. @@ -141,7 +141,7 @@ namespace Aws /** *Function to initialize the cipher for decryption using the crypto content material. */ - void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer()) override; + void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer(), const Aws::Utils::CryptoBuffer& aad = Aws::Utils::CryptoBuffer()) override; /* * Function to get the crypto tag according to the module. @@ -185,7 +185,7 @@ namespace Aws /** *Function to initialize the cipher for decryption using the crypto content material. */ - void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer()) override; + void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer(), const Aws::Utils::CryptoBuffer& aad = Aws::Utils::CryptoBuffer()) override; /* * Function to get the crypto tag according to the module. @@ -229,7 +229,7 @@ namespace Aws /** *Function to initialize the cipher for decryption using the crypto content material. */ - void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer()) override; + void InitDecryptionCipher(int64_t rangeStart = 0, int64_t rangeEnd = 0, const Aws::Utils::CryptoBuffer& tag = Aws::Utils::CryptoBuffer(), const Aws::Utils::CryptoBuffer& aad = Aws::Utils::CryptoBuffer()) override; /* * Function to get the crypto tag according to the module. @@ -257,7 +257,7 @@ namespace Aws /** * This will create a GCM cipher under the hood, passing the specified key. */ - AES_GCM_AppendedTag(const Aws::Utils::CryptoBuffer& key); + AES_GCM_AppendedTag(const Aws::Utils::CryptoBuffer& key, const Aws::Utils::CryptoBuffer& iv, const Aws::Utils::CryptoBuffer& aad); operator bool() const override; diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/CryptoConfiguration.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/CryptoConfiguration.cpp index c9ec6cbf485..5747ab96a84 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/CryptoConfiguration.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/CryptoConfiguration.cpp @@ -12,7 +12,10 @@ namespace Aws m_storageMethod(StorageMethod::METADATA), m_cryptoMode(CryptoMode::AUTHENTICATED_ENCRYPTION), m_unAuthenticatedRangeGet(RangeGetMode::ALL), - m_securityProfile(SecurityProfile::V2_AND_LEGACY) + m_securityProfile(SecurityProfile::V2_AND_LEGACY), + m_algorithmSuite(AlgorithmSuite::AES_GCM), + m_commitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT) + { } @@ -20,7 +23,9 @@ namespace Aws m_storageMethod(storageMethod), m_cryptoMode(CryptoMode::AUTHENTICATED_ENCRYPTION), m_unAuthenticatedRangeGet(RangeGetMode::ALL), - m_securityProfile(SecurityProfile::V2_AND_LEGACY) + m_securityProfile(SecurityProfile::V2_AND_LEGACY), + m_algorithmSuite(AlgorithmSuite::AES_GCM), + m_commitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT) { } @@ -28,7 +33,9 @@ namespace Aws m_storageMethod(StorageMethod::METADATA), m_cryptoMode(cryptoMode), m_unAuthenticatedRangeGet(RangeGetMode::ALL), - m_securityProfile(SecurityProfile::V2_AND_LEGACY) + m_securityProfile(SecurityProfile::V2_AND_LEGACY), + m_algorithmSuite(AlgorithmSuite::AES_GCM), + m_commitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT) { } @@ -36,7 +43,9 @@ namespace Aws m_storageMethod(storageMode), m_cryptoMode(cryptoMode), m_unAuthenticatedRangeGet(RangeGetMode::ALL), - m_securityProfile(SecurityProfile::V2_AND_LEGACY) + m_securityProfile(SecurityProfile::V2_AND_LEGACY), + m_algorithmSuite(AlgorithmSuite::AES_GCM), + m_commitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT) { } @@ -55,5 +64,51 @@ namespace Aws m_encryptionMaterials(materials) { } + + CryptoConfigurationV3::CryptoConfigurationV3(const std::shared_ptr& materials) : + m_storageMethod(StorageMethod::METADATA), + m_unAuthenticatedRangeGet(RangeGetMode::DISABLED), + m_securityProfile(SecurityProfile::V2), + m_encryptionMaterials(materials), + m_commitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + { + } + + CryptoConfigurationV3::CryptoConfigurationV3(const std::shared_ptr& materials) : + //= ../specification/s3-encryption/data-format/metadata-strategy.md#object-metadata + //= type=implication + //# By default, the S3EC MUST store content metadata in the S3 Object Metadata. + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //= type=implication + //# Instruction File writes MUST NOT be enabled by default. + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //= type=implication + //# Instruction File writes MUST be optionally configured during client creation or on each PutObject request. + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //= type=implication + //# The S3EC MUST support writing some or all (depending on format) content metadata to an Instruction File. + + //= ../specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms + //= type=implication + //# The option to enable legacy wrapping algorithms MUST be set to false by default. + + //= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + //= type=implication + //# The option to enable legacy unauthenticated modes MUST be set to false by default. + + //= ../specification/s3-encryption/client.md#enable-delayed-authentication + //= type=implication + //# Delayed Authentication mode MUST be set to false by default. + + m_storageMethod(StorageMethod::METADATA), + m_unAuthenticatedRangeGet(RangeGetMode::DISABLED), + m_securityProfile(SecurityProfile::V2), + m_encryptionMaterials(materials), + m_commitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + { + } } } diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/HKDF.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/HKDF.cpp new file mode 100644 index 00000000000..d4e3cc51514 --- /dev/null +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/HKDF.cpp @@ -0,0 +1,103 @@ +#include +#include +#include + +namespace Aws { +namespace S3Encryption { + +const size_t ENCRYPTION_KEY_INFO_LEN = 11; +const size_t COMMITMENT_KEY_INFO_LEN = 11; +const char *ENCRYPTION_KEY_INFO = "\x00\x73" "DERIVEKEY"; +const char *COMMITMENT_KEY_INFO = "\x00\x73" "COMMITKEY"; + +bool derive_key(const Aws::Utils::CryptoBuffer &data_key, const Aws::Utils::CryptoBuffer &message_id, Aws::Crt::ByteCursor & info, + Aws::Utils::CryptoBuffer &output) { + auto data_key_cursor = Aws::Crt::ByteCursorFromArray(data_key.GetUnderlyingData(), data_key.GetLength()); + auto message_id_cursor = Aws::Crt::ByteCursorFromArray(message_id.GetUnderlyingData(), message_id.GetLength()); + auto out = Aws::Crt::ByteBufFromEmptyArray(output.GetUnderlyingData(), output.GetLength()); + + return Aws::Crt::Crypto::DeriveSHA512HMACHKDF(Aws::Crt::ApiAllocator(), + data_key_cursor, + message_id_cursor, + info, + out, output.GetLength()); +} + +bool derive_encryption_key(const Aws::Utils::CryptoBuffer &data_key, const Aws::Utils::CryptoBuffer &message_id, + Aws::Utils::CryptoBuffer &output) { + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The salt MUST be the Message ID with the length defined in the algorithm suite. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The hash function MUST be specified by the algorithm suite commitment settings. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=implication + //# - The DEK input pseudorandom key MUST be the output from the extract step. + auto info_cursor = Aws::Crt::ByteCursorFromArray((const uint8_t *)ENCRYPTION_KEY_INFO, ENCRYPTION_KEY_INFO_LEN); + return derive_key(data_key, message_id, info_cursor, output); +} + +//= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key +//= type=implication +//# The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). +bool derive_commitment_key(const Aws::Utils::CryptoBuffer &data_key, const Aws::Utils::CryptoBuffer &message_id, + Aws::Utils::CryptoBuffer &output) { + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The salt MUST be the Message ID with the length defined in the algorithm suite. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //# - The hash function MUST be specified by the algorithm suite commitment settings. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=implication + //# - The CK input pseudorandom key MUST be the output from the extract step. + auto info_cursor = Aws::Crt::ByteCursorFromArray((const uint8_t *)COMMITMENT_KEY_INFO, COMMITMENT_KEY_INFO_LEN); + return derive_key(data_key, message_id, info_cursor, output); +} + +// The cal-c people promise to surface a "real" implementation of this at some point. +bool constant_time_equal(const Aws::Utils::CryptoBuffer& x, const Aws::Utils::CryptoBuffer& y) +{ + // If lengths don't match, they're not equal + if (x.GetLength() != y.GetLength()) { + return false; + } + + const uint8_t* x_data = x.GetUnderlyingData(); + const uint8_t* y_data = y.GetUnderlyingData(); + + volatile uint8_t val = 0; + for (size_t i=0; iAppendToUserAgent("ft/S3CryptoV1n"); } + bool S3EncryptionClientBase::ValidateStorageMethod(StorageMethod s) + { + if (s != StorageMethod::METADATA && s != StorageMethod::INSTRUCTION_FILE) { + m_error = "Invalid StorageMethod used."; + return true; + } + return false; + } + + bool S3EncryptionClientBase::ValidateSecurityProfile(SecurityProfile s) + { + if (s != SecurityProfile::V2 && s != SecurityProfile::V2_AND_LEGACY) { + m_error = "Invalid SecurityProfile used."; + return true; + } + return false; + } + + bool S3EncryptionClientBase::ValidateCommitmentPolicy(CommitmentPolicy s) + { + if (s != CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT && s != CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT && s != CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT) { + m_error = "Invalid CommitmentPolicy used."; + return true; + } + return false; + } + + bool S3EncryptionClientBase::ValidateRangeGetMode(RangeGetMode s) + { + if (s != RangeGetMode::DISABLED && s != RangeGetMode::ALL) { + m_error = "Invalid RangeGetMode used."; + return true; + } + return false; + } + S3EncryptionPutObjectOutcome S3EncryptionClientBase::PutObject(const Aws::S3::Model::PutObjectRequest& request, const Aws::Map& contextMap) const { + if (!m_error.empty()) { + return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError( + S3Errors::INVALID_PARAMETER_VALUE, "MaterialsDescriptionMismatch", + m_error, false/*not retryable*/))); + } + //= ../specification/s3-encryption/encryption.md#content-encryption + //= type=implication + //# The S3EC MUST use the encryption algorithm configured during [client](./client.md) initialization. auto module = m_cryptoModuleFactory.FetchCryptoModule(m_encryptionMaterials, m_cryptoConfig); auto putObjectFunction = [this](const Aws::S3::Model::PutObjectRequest& putRequest) { return m_s3Client->PutObject(putRequest); }; return module->PutObjectSecurely(request, putObjectFunction, contextMap); @@ -65,7 +109,7 @@ namespace Aws bool MapsEqual(const Aws::Map& passed, const Aws::Map& stored) { - // everything in passed must be in stored, withthe same value. + // everything in passed must be in stored, with the same value. auto stored_end = stored.end(); auto passed_end = passed.end(); for (const auto& pair : passed) { @@ -96,6 +140,11 @@ namespace Aws } S3EncryptionGetObjectOutcome S3EncryptionClientBase::GetObjectInner(const Aws::S3::Model::GetObjectRequest & request, const Aws::Map * contextMap) const { + if (!m_error.empty()) { + return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError( + S3Errors::INVALID_PARAMETER_VALUE, "MaterialsDescriptionMismatch", + m_error, false/*not retryable*/))); + } Aws::S3::Model::HeadObjectRequest headRequest; headRequest.WithBucket(request.GetBucket()); headRequest.WithKey(request.GetKey()); @@ -108,12 +157,17 @@ namespace Aws return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(headOutcome.GetError())); } + //= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + //= type=implication + //# If the object matches none of the V1/V2/V3 formats, the S3EC MUST attempt to get the instruction file. + auto headMetadata = headOutcome.GetResult().GetMetadata(); auto metadataEnd = headMetadata.end(); CryptoConfiguration decryptionCryptoConfig; - headMetadata.find(CONTENT_KEY_HEADER) != metadataEnd && headMetadata.find(IV_HEADER) != metadataEnd - ? decryptionCryptoConfig.SetStorageMethod(StorageMethod::METADATA) - : decryptionCryptoConfig.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + headMetadata.find(CONTENT_KEY_HEADER) == metadataEnd && headMetadata.find(DEPRECATED_CONTENT_KEY_HEADER) == metadataEnd + && headMetadata.find(ENCRYPTED_DATA_KEY_V3) == metadataEnd + ? decryptionCryptoConfig.SetStorageMethod(StorageMethod::INSTRUCTION_FILE) + : decryptionCryptoConfig.SetStorageMethod(StorageMethod::METADATA); ContentCryptoMaterial contentCryptoMaterial; if (decryptionCryptoConfig.GetStorageMethod() == StorageMethod::INSTRUCTION_FILE) @@ -124,13 +178,17 @@ namespace Aws return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(instructionOutcome.GetError())); } Handlers::InstructionFileHandler handler; - contentCryptoMaterial = handler.ReadContentCryptoMaterial(instructionOutcome.GetResult()); + contentCryptoMaterial = handler.ReadContentCryptoMaterial(instructionOutcome.GetResult(), headOutcome.GetResult()); } else { Handlers::MetadataHandler handler; contentCryptoMaterial = handler.ReadContentCryptoMaterial(headOutcome.GetResult()); } + if (contentCryptoMaterial.Fail()) { + return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError( + S3Errors::VALIDATION, "BadMetaData", contentCryptoMaterial.Error(), false/*not retryable*/))); + } if (contextMap && !MapsEqual(*contextMap, contentCryptoMaterial.GetMaterialsDescription())) { return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError( @@ -138,7 +196,10 @@ namespace Aws "Provided encryption context does not match information retrieved from S3", false/*not retryable*/))); } - // security check + //= ../specification/s3-encryption/client.md#enable-delayed-authentication + //# When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. + //# When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + if (request.RangeHasBeenSet() && m_cryptoConfig.GetUnAuthenticatedRangeGet() == RangeGetMode::DISABLED) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Unable to perform range get request: Range get support has been disabled. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html"); @@ -148,6 +209,12 @@ namespace Aws if (m_cryptoConfig.GetSecurityProfile() == SecurityProfile::V2) { + //= ../specification/s3-encryption/decryption.md#legacy-decryption + //# The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites unless specifically configured to do so. + + //= ../specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms + //# When enabled, the S3EC MUST be able to decrypt objects encrypted with all supported wrapping algorithms (both legacy and fully supported). + //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy wrapping algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy wrapping algorithm. if (contentCryptoMaterial.GetKeyWrapAlgorithm() != KeyWrapAlgorithm::AES_GCM && contentCryptoMaterial.GetKeyWrapAlgorithm() != KeyWrapAlgorithm::KMS_CONTEXT) { @@ -156,7 +223,13 @@ namespace Aws "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration securityProfile=V2. Retry with V2_AND_LEGACY enabled or re-encrypt the object.", false/*not retryable*/))); } - if (contentCryptoMaterial.GetContentCryptoScheme() != ContentCryptoScheme::GCM) + //= ../specification/s3-encryption/decryption.md#legacy-decryption + //# If the S3EC is not configured to enable legacy unauthenticated content decryption, the client MUST throw an exception when attempting to decrypt an object encrypted with a legacy unauthenticated algorithm suite. + + //= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + //# When enabled, the S3EC MUST be able to decrypt objects encrypted with all content encryption algorithms (both legacy and fully supported). + //# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy content encryption algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy content encryption algorithm. + if (!IsGCM(contentCryptoMaterial.GetContentCryptoScheme())) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration securityProfile=V2. Retry with V2_AND_LEGACY enabled or re-encrypt the object"); return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError(S3Errors::INVALID_ACTION, "DecryptV1EncryptSchemaFailed", @@ -164,12 +237,33 @@ namespace Aws } } + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. + + //= ../specification/s3-encryption/decryption.md#key-commitment + //# The S3EC MUST validate the algorithm suite used for decryption against the key commitment policy before attempting to decrypt the content ciphertext. + + //= ../specification/s3-encryption/decryption.md#key-commitment + //# If the commitment policy requires decryption using a committing algorithm suite, and the algorithm suite associated with the object does not support key commitment, then the S3EC MUST throw an exception. + if (m_cryptoConfig.GetCommitmentPolicy() == CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT + && contentCryptoMaterial.GetContentCryptoScheme() != ContentCryptoScheme::GCM_COMMIT) { + AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "The requested object is encrypted with V2 encryption schemas that have been disabled by client configuration CommitmentPolicy=REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Retry with REQUIRE_ENCRYPT_ALLOW_DECRYPT enabled or re-encrypt the object"); + return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError(S3Errors::INVALID_ACTION, "DecryptV2EncryptSchemaFailed", + "The requested object is encrypted with V2 encryption schemas that have been disabled by client configuration CommitmentPolicy=REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Retry with REQUIRE_ENCRYPT_ALLOW_DECRYPT enabled or re-encrypt the object.", false/*not retryable*/))); + } + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::CBC) { decryptionCryptoConfig.SetCryptoMode(CryptoMode::ENCRYPTION_ONLY); } else if (m_cryptoConfig.GetCryptoMode() != CryptoMode::STRICT_AUTHENTICATED_ENCRYPTION && - contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM) + IsGCM(contentCryptoMaterial.GetContentCryptoScheme())) { decryptionCryptoConfig.SetCryptoMode(CryptoMode::AUTHENTICATED_ENCRYPTION); } @@ -202,14 +296,61 @@ namespace Aws void S3EncryptionClientV2::Init(const Aws::S3Encryption::CryptoConfigurationV2& cryptoConfig) { + // This validation could be, in a bizarre set of circumstances, a breaking change + // if (ValidateStorageMethod(cryptoConfig.GetStorageMethod())) return; + // if (ValidateRangeGetMode(cryptoConfig.GetUnAuthenticatedRangeGet())) return; + // if (ValidateSecurityProfile(cryptoConfig.GetSecurityProfile())) return; + m_cryptoConfig.SetSecurityProfile(cryptoConfig.GetSecurityProfile()); m_cryptoConfig.SetUnAuthenticatedRangeGet(cryptoConfig.GetUnAuthenticatedRangeGet()); + m_cryptoConfig.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + m_cryptoConfig.SetEncryptionAlgorithm(AlgorithmSuite::AES_GCM); + m_cryptoConfig.SetStorageMethod(cryptoConfig.GetStorageMethod()); + m_s3Client->AppendToUserAgent("ft/S3CryptoV2"); - - if (cryptoConfig.GetSecurityProfile() == SecurityProfile::V2_AND_LEGACY) - { + if (cryptoConfig.GetSecurityProfile() == SecurityProfile::V2_AND_LEGACY) { AWS_LOGSTREAM_WARN(ALLOCATION_TAG, "The S3 Encryption Client is configured to read encrypted data with legacy encryption modes. If you don't have objects encrypted with these legacy modes, you should disable support for them to enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html"); } } + void S3EncryptionClientV3::Init(const Aws::S3Encryption::CryptoConfigurationV3& cryptoConfig) + { + if (ValidateStorageMethod(cryptoConfig.GetStorageMethod())) return; + if (ValidateRangeGetMode(cryptoConfig.GetUnAuthenticatedRangeGet())) return; + if (ValidateSecurityProfile(cryptoConfig.GetSecurityProfile())) return; + if (ValidateCommitmentPolicy(cryptoConfig.GetCommitmentPolicy())) return; + + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + + //= ../specification/s3-encryption/key-commitment.md#commitment-policy + //# When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + + if (cryptoConfig.GetCommitmentPolicy() == CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT) { + m_cryptoConfig.SetEncryptionAlgorithm(AlgorithmSuite::AES_GCM); + } else { + m_cryptoConfig.SetEncryptionAlgorithm(AlgorithmSuite::AES_GCM_WITH_COMMITMENT); + } + if (cryptoConfig.GetCommitmentPolicy() == CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT) { + m_cryptoConfig.SetSecurityProfile(SecurityProfile::V2); + } else { + m_cryptoConfig.SetSecurityProfile(cryptoConfig.GetSecurityProfile()); + } + m_cryptoConfig.SetUnAuthenticatedRangeGet(cryptoConfig.GetUnAuthenticatedRangeGet()); + m_cryptoConfig.SetCommitmentPolicy(cryptoConfig.GetCommitmentPolicy()); + m_cryptoConfig.SetStorageMethod(cryptoConfig.GetStorageMethod()); + + m_s3Client->AppendToUserAgent("ft/S3CryptoV3"); + + if (cryptoConfig.GetSecurityProfile() == SecurityProfile::V2_AND_LEGACY) { + if (cryptoConfig.GetCommitmentPolicy() == CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT) { + AWS_LOGSTREAM_WARN(ALLOCATION_TAG, "The S3 Encryption Client is configured with both CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT and AllowLegacy. AllowLegacy will be ignored. Objects stored with legacy encryption will not be decrypted."); + } else { + AWS_LOGSTREAM_WARN(ALLOCATION_TAG, "The S3 Encryption Client is configured to read encrypted data with legacy encryption modes. If you don't have objects encrypted with these legacy modes, you should disable support for them to enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html"); + } + } + } } } diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/DataHandler.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/DataHandler.cpp index 3839a28828d..d72915c3836 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/DataHandler.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/DataHandler.cpp @@ -50,39 +50,168 @@ namespace Aws } } + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + + Aws::String V3ToV2Alg(const Aws::String & v3) { + if (v3 == "01") return "AESWrap"; + if (v3 == "02") return "AES/GCM"; + if (v3 == "11") return "kms"; + if (v3 == "12") return "kms+context"; + if (v3 == "21") return "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"; + if (v3 == "22") return "RSA-OAEP-SHA1"; + return ""; + } + Aws::String V2ToV3Alg(const Aws::String & v2) { + if (v2 == "AESWrap") return "01"; + if (v2 == "AES/GCM") return "02"; + if (v2 == "kms") return "11"; + if (v2 == "kms+context") return "12"; + if (v2 == "RSA/ECB/OAEPWithSHA-256AndMGF1Padding") return "21"; + if (v2 == "RSA-OAEP-SHA1") return "22"; + return ""; + } + ContentCryptoMaterial DataHandler::ReadMetadata(const Aws::Map& metadata) { - auto keyIterator = metadata.find(CONTENT_KEY_HEADER); + auto total_keys = 0; auto deprecatedKeyIterator = metadata.find(DEPRECATED_CONTENT_KEY_HEADER); - auto ivIterator = metadata.find(IV_HEADER); + if (deprecatedKeyIterator != metadata.end()) ++total_keys; auto materialsDescriptionIterator = metadata.find(MATERIALS_DESCRIPTION_HEADER); + auto ivIterator = metadata.find(IV_HEADER); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - The mapkey "x-amz-key" MUST be present for V1 format objects. + // implication, because V1 format is defined as the presence of x-amz-key + + //= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + //= type=implication + //# - If the metadata contains "x-amz-iv" and "x-amz-key" then the object MUST be considered as an S3EC-encrypted object using the V1 format. + + if (deprecatedKeyIterator != metadata.end()) { + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - The mapkey "x-amz-matdesc" MUST be present for V1 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - The mapkey "x-amz-iv" MUST be present for V1 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=exception + //# - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects. + // exception because it's either present or not present + + if (materialsDescriptionIterator == metadata.end() || ivIterator == metadata.end()) { + return ContentCryptoMaterial("One or more metadata fields do not exist for V1 decryption."); + } + } + + auto keyIterator = metadata.find(CONTENT_KEY_HEADER); + if (keyIterator != metadata.end()) ++total_keys; + auto v3KeyIterator = metadata.find(ENCRYPTED_DATA_KEY_V3); + if (v3KeyIterator != metadata.end()) ++total_keys; + + if (total_keys == 0) { + return ContentCryptoMaterial("Encrypted S3 Object is not in any known format."); + } + //= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + //= type=implication + //# If there are multiple mapkeys which are meant to be exclusive, such as "x-amz-key", "x-amz-key-v2", and "x-amz-3" then the S3EC SHOULD throw an exception. + //# In general, if there is any deviation from the above format, with the exception of additional unrelated mapkeys, then the S3EC SHOULD throw an exception. + if (total_keys != 1) { + return ContentCryptoMaterial("Encrypted S3 Object seems to be in multiple formats."); + } + auto schemeIterator = metadata.find(CONTENT_CRYPTO_SCHEME_HEADER); auto keyWrapIterator = metadata.find(KEY_WRAP_ALGORITHM); - // C++ SDK never writes x-amz-key but adding this check so as to be capable of reading other language SDK encrypted objects. - // This fits into both old and new clients. - if ((keyIterator == metadata.end() && deprecatedKeyIterator == metadata.end()) || ivIterator == metadata.end() || - materialsDescriptionIterator == metadata.end() || schemeIterator == metadata.end() || - keyIterator == metadata.end()) - { - AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "One or more metadata fields do not exist for decryption."); - return ContentCryptoMaterial(); + if (schemeIterator == metadata.end()) schemeIterator = metadata.find(CONTENT_CIPHER_V3); + if (materialsDescriptionIterator == metadata.end()) materialsDescriptionIterator = metadata.find(MAT_DESC_V3); + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#object-metadata + //= type=implication + //# If the S3EC does not support decoding the S3 Server's "double encoding" then it MUST return the content metadata untouched. + auto encryptionContext = metadata.find(ENCRYPTION_CONTEXT_V3); + auto keyCommitment = metadata.find(KEY_COMMITMENT_V3); + auto messageId = metadata.find(MESSAGE_ID_V3); + if (messageId != metadata.end()) ivIterator = messageId; + if (materialsDescriptionIterator == metadata.end()) materialsDescriptionIterator = encryptionContext; + + Aws::String keyWrapAlgorithmAsString; + if (keyWrapIterator != metadata.end()) { + keyWrapAlgorithmAsString = keyWrapIterator->second; + } else { + keyWrapIterator = metadata.find(ENCRYPTED_DATA_KEY_ALGORITHM_V3); + if (keyWrapIterator != metadata.end()) { + keyWrapAlgorithmAsString = V3ToV2Alg(keyWrapIterator->second); + } + } + + if (v3KeyIterator == metadata.end()) { + //= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + //= type=implication + //# - If the metadata contains "x-amz-iv" and "x-amz-metadata-x-amz-key-v2" then the object MUST be considered as an S3EC-encrypted object using the V2 format. + if ((keyIterator == metadata.end() && deprecatedKeyIterator == metadata.end()) + || ivIterator == metadata.end() || materialsDescriptionIterator == metadata.end() + || schemeIterator == metadata.end() || keyIterator == metadata.end()) + { + return ContentCryptoMaterial("One or more metadata fields do not exist for decryption."); + } + } else { + //= ../specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + //= type=implication + //# - If the metadata contains "x-amz-3" and "x-amz-d" and "x-amz-i" then the object MUST be considered an S3EC-encrypted object using the V3 format. + if (keyCommitment == metadata.end() || messageId == metadata.end()) { + return ContentCryptoMaterial("One or more V3 metadata fields do not exist for decryption."); + } } ContentCryptoMaterial contentCryptoMaterial; - Aws::String keyWrapAlgorithmAsString = keyWrapIterator->second; - contentCryptoMaterial.SetKeyWrapAlgorithm(KeyWrapAlgorithmMapper::GetKeyWrapAlgorithmForName(keyWrapAlgorithmAsString)); + KeyWrapAlgorithm keyWrapAlgorithm = KeyWrapAlgorithmMapper::GetKeyWrapAlgorithmForName(keyWrapAlgorithmAsString); + + // Check if the key wrap algorithm is valid + if (keyWrapAlgorithm == KeyWrapAlgorithm::NONE) { + Aws::OStringStream msg; + msg << "Invalid or unrecognized key wrap algorithm: " << keyWrapAlgorithmAsString; + return ContentCryptoMaterial(msg.str().c_str()); + } + + contentCryptoMaterial.SetKeyWrapAlgorithm(keyWrapAlgorithm); + + if (v3KeyIterator != metadata.end()) { + if (encryptionContext != metadata.end()) { + contentCryptoMaterial.SetEncryptionContext(DeserializeMap(encryptionContext->second)); + } + contentCryptoMaterial.SetMessageID(Aws::Utils::HashingUtils::Base64Decode(messageId->second)); + contentCryptoMaterial.SetKeyCommitment(Aws::Utils::HashingUtils::Base64Decode(keyCommitment->second)); + } // if the key wrap algorithm is AES_GCM, we need to split 12 bytes IV and 16 bytes tag out of it. - CryptoBuffer finalCEK = Aws::Utils::HashingUtils::Base64Decode((keyIterator != metadata.end() ? keyIterator->second : deprecatedKeyIterator->second)); + CryptoBuffer finalCEK = Aws::Utils::HashingUtils::Base64Decode( + keyIterator != metadata.end() ? keyIterator->second : + v3KeyIterator != metadata.end() ? v3KeyIterator->second : + deprecatedKeyIterator->second); + contentCryptoMaterial.SetFinalCEK(finalCEK); if (contentCryptoMaterial.GetKeyWrapAlgorithm() == KeyWrapAlgorithm::AES_GCM) { size_t expectedLength = AES_GCM_IV_BYTES + AES_GCM_KEY_BYTES + AES_GCM_TAG_BYTES; if (finalCEK.GetLength() != expectedLength) { - AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "S3 Encryption Client get unexpected AES GCM key wrap final CEK length: " << finalCEK.GetLength() << " expected: " << expectedLength); - return ContentCryptoMaterial(); + Aws::OStringStream msg; + msg << "S3 Encryption Client get unexpected AES GCM key wrap final CEK length: " << finalCEK.GetLength() << " expected: " << expectedLength; + return ContentCryptoMaterial(msg.str().c_str()); } contentCryptoMaterial.SetCekIV(CryptoBuffer(finalCEK.GetUnderlyingData(), AES_GCM_IV_BYTES)); contentCryptoMaterial.SetEncryptedContentEncryptionKey(CryptoBuffer(finalCEK.GetUnderlyingData() + AES_GCM_IV_BYTES, AES_GCM_KEY_BYTES)); @@ -92,24 +221,44 @@ namespace Aws { if (contentCryptoMaterial.GetKeyWrapAlgorithm() == KeyWrapAlgorithm::AES_KEY_WRAP && finalCEK.GetLength() != AES_KEY_WRAP_ENCRYPTED_CEK_BYTES) { - AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "S3 Encryption Client get unexpected AES Key Wrap final CEK length: " << finalCEK.GetLength() << " expected: " << AES_KEY_WRAP_ENCRYPTED_CEK_BYTES); - return ContentCryptoMaterial(); + Aws::OStringStream msg; + msg << "S3 Encryption Client get unexpected AES Key Wrap final CEK length: " << finalCEK.GetLength() << " expected: " << AES_KEY_WRAP_ENCRYPTED_CEK_BYTES; + return ContentCryptoMaterial(msg.str().c_str()); } contentCryptoMaterial.SetEncryptedContentEncryptionKey(finalCEK); contentCryptoMaterial.SetCEKGCMTag(CryptoBuffer()); } + if (v3KeyIterator == metadata.end()) { + contentCryptoMaterial.SetIV(Aws::Utils::HashingUtils::Base64Decode(ivIterator->second)); + } else { + contentCryptoMaterial.SetV3IV(); + } - contentCryptoMaterial.SetIV(Aws::Utils::HashingUtils::Base64Decode(ivIterator->second)); - contentCryptoMaterial.SetMaterialsDescription(DeserializeMap(materialsDescriptionIterator->second)); + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# If the mapkey x-amz-t is not present, the default Material Description value MUST be set to an empty map (`{}`). + // If not set, it is empty. + if (materialsDescriptionIterator != metadata.end()) { + contentCryptoMaterial.SetMaterialsDescription(DeserializeMap(materialsDescriptionIterator->second)); + } Aws::String schemeAsString = schemeIterator->second; - contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoSchemeMapper::GetContentCryptoSchemeForName(schemeAsString)); + ContentCryptoScheme scheme = ContentCryptoSchemeMapper::GetContentCryptoSchemeForName(schemeAsString); + + // Check if the crypto scheme is valid + if (scheme == ContentCryptoScheme::NONE) { + Aws::OStringStream msg; + msg << "Invalid or unrecognized content crypto scheme: " << schemeAsString; + return ContentCryptoMaterial(msg.str().c_str()); + } + + contentCryptoMaterial.SetContentCryptoScheme(scheme); // value of x-amz-cek-alg is used as AES/GCM AAD info for CEK encryption/decryption contentCryptoMaterial.SetGCMAAD(CryptoBuffer((const unsigned char*)schemeAsString.c_str(), schemeAsString.size())); // Ignore CRYPTO_TAG_LENGTH_HEADER for new client, this change is still compatible with old client, which only accept GCM and CBC for data encryption. - if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM) + if (IsGCM(contentCryptoMaterial.GetContentCryptoScheme())) { contentCryptoMaterial.SetCryptoTagLength(AES_GCM_TAG_BYTES * 8UL); } diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/InstructionFileHandler.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/InstructionFileHandler.cpp index 2c185659596..816fb94e64d 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/InstructionFileHandler.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/InstructionFileHandler.cpp @@ -22,13 +22,77 @@ namespace Aws static const char* const ALLOCATION_TAG = "InstructionFileHandler"; static const char* const INSTRUCTION_HEADER_VALUE = "default instruction file header"; - void InstructionFileHandler::PopulateRequest(Aws::S3::Model::PutObjectRequest & request, const ContentCryptoMaterial & contentCryptoMaterial) + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# In the V3 format, the mapkeys "x-amz-c", "x-amz-d", and "x-amz-i" MUST be stored exclusively in the Object Metadata. + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + //# - The V3 message format MUST store the mapkey "x-amz-c" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-d" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-i" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-3" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-w" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-m" and its value (when present in the content metadata) in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-t" and its value (when present in the content metadata) in the Instruction File. + + void InstructionFileHandler::PopulateRequestV3(Aws::S3::Model::PutObjectRequest& request, Aws::S3::Model::PutObjectRequest& ifRequest, const ContentCryptoMaterial& contentCryptoMaterial) + { + // Meta Data for object + + ContentCryptoScheme scheme = contentCryptoMaterial.GetContentCryptoScheme(); + request.AddMetadata(CONTENT_CIPHER_V3, GetNameForContentCryptoScheme(scheme)); + + Aws::String encodedKeyCommitment = HashingUtils::Base64Encode(contentCryptoMaterial.GetKeyCommitment()); + request.AddMetadata(KEY_COMMITMENT_V3, encodedKeyCommitment); + + Aws::String encodedMessageId = HashingUtils::Base64Encode(contentCryptoMaterial.GetMessageID()); + request.AddMetadata(MESSAGE_ID_V3, encodedMessageId); + + + // Meta Data for instruction file + + ifRequest.SetKey(request.GetKey() + DEFAULT_INSTRUCTION_FILE_SUFFIX); + + Aws::Map instructionMetadata; + instructionMetadata[INSTRUCTION_FILE_HEADER] = INSTRUCTION_HEADER_VALUE; + ifRequest.SetMetadata(instructionMetadata); + + Aws::Map contentCryptoMap; + + auto encryptionContextMap = contentCryptoMaterial.GetMaterialsDescription(); + if (!encryptionContextMap.empty()) { + contentCryptoMap[ENCRYPTION_CONTEXT_V3] = SerializeMap(encryptionContextMap); + } + contentCryptoMap[ENCRYPTED_DATA_KEY_ALGORITHM_V3] = V2ToV3Alg(GetNameForKeyWrapAlgorithm(contentCryptoMaterial.GetKeyWrapAlgorithm())); + contentCryptoMap[ENCRYPTED_DATA_KEY_V3] = HashingUtils::Base64Encode(contentCryptoMaterial.GetFinalCEK()); + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //# The content metadata stored in the Instruction File MUST be serialized to a JSON string. + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //# The serialized JSON string MUST be the only contents of the Instruction File. + Aws::String jsonCryptoMap = SerializeMap(contentCryptoMap); + std::shared_ptr streamPtr = Aws::MakeShared(ALLOCATION_TAG, jsonCryptoMap); + ifRequest.SetBody(streamPtr); + } + + void InstructionFileHandler::PopulateRequest(Aws::S3::Model::PutObjectRequest& objRequest, Aws::S3::Model::PutObjectRequest& ifRequest, const ContentCryptoMaterial& contentCryptoMaterial) { - request.SetKey(request.GetKey() + DEFAULT_INSTRUCTION_FILE_SUFFIX); + if (contentCryptoMaterial.GetKeyCommitment().GetLength() != 0) + { + return PopulateRequestV3(objRequest, ifRequest, contentCryptoMaterial); + } + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + //# In the V1/V2 message format, all of the content metadata MUST be stored in the Instruction File. + + ifRequest.SetKey(ifRequest.GetKey() + DEFAULT_INSTRUCTION_FILE_SUFFIX); Aws::Map instructionMetadata; instructionMetadata[INSTRUCTION_FILE_HEADER] = INSTRUCTION_HEADER_VALUE; - request.SetMetadata(instructionMetadata); + ifRequest.SetMetadata(instructionMetadata); Aws::Map contentCryptoMap; contentCryptoMap[CONTENT_KEY_HEADER] = HashingUtils::Base64Encode(contentCryptoMaterial.GetFinalCEK()); @@ -40,15 +104,30 @@ namespace Aws Aws::String jsonCryptoMap = SerializeMap(contentCryptoMap); std::shared_ptr streamPtr = Aws::MakeShared(ALLOCATION_TAG, jsonCryptoMap); - request.SetBody(streamPtr); + ifRequest.SetBody(streamPtr); } - ContentCryptoMaterial InstructionFileHandler::ReadContentCryptoMaterial(Aws::S3::Model::GetObjectResult & result) + static ContentCryptoMaterial InvalidKey(const char * key) { + Aws::OStringStream msg; + msg << "Instruction File must not contain: " << key; + return ContentCryptoMaterial(msg.str().c_str()); + } + ContentCryptoMaterial InstructionFileHandler::ReadContentCryptoMaterial(const Aws::S3::Model::GetObjectResult & instruction_file, const Aws::S3::Model::HeadObjectResult & object_head) { - IOStream& stream = result.GetBody(); + IOStream& stream = instruction_file.GetBody(); Aws::String jsonString; stream >> jsonString; Aws::Map cryptoContentMap = DeserializeMap(jsonString); + + if (cryptoContentMap.find(CONTENT_CIPHER_V3) != cryptoContentMap.end()) + return InvalidKey(CONTENT_CIPHER_V3); + if (cryptoContentMap.find(KEY_COMMITMENT_V3) != cryptoContentMap.end()) + return InvalidKey(KEY_COMMITMENT_V3); + if (cryptoContentMap.find(MESSAGE_ID_V3) != cryptoContentMap.end()) + return InvalidKey(MESSAGE_ID_V3); + + Aws::Map metadata = object_head.GetMetadata(); + cryptoContentMap.insert(metadata.begin(), metadata.end()); return ReadMetadata(cryptoContentMap); } diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/MetadataHandler.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/MetadataHandler.cpp index 0fb978922c6..b6575750a90 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/MetadataHandler.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/handlers/MetadataHandler.cpp @@ -20,33 +20,87 @@ namespace Aws { namespace Handlers { - void MetadataHandler::PopulateRequest(Aws::S3::Model::PutObjectRequest& request, const ContentCryptoMaterial& contentCryptoMaterial) + //= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility + //= type=implication + //# Objects encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY MUST use the V3 message format version only. + void MetadataHandler::PopulateRequestV3(Aws::S3::Model::PutObjectRequest& request, const ContentCryptoMaterial& contentCryptoMaterial) { + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-c" MUST be present for V3 format objects. + ContentCryptoScheme scheme = contentCryptoMaterial.GetContentCryptoScheme(); + request.AddMetadata(CONTENT_CIPHER_V3, GetNameForContentCryptoScheme(scheme)); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-3" MUST be present for V3 format objects. + Aws::String encodedCEK = HashingUtils::Base64Encode(contentCryptoMaterial.GetFinalCEK()); + request.AddMetadata(ENCRYPTED_DATA_KEY_V3, encodedCEK); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=implication + //# - The mapkey "x-amz-t" SHOULD be present for V3 format objects that use KMS Encryption Context. + Aws::Map encryptionContextMap = contentCryptoMaterial.GetMaterialsDescription(); + if (!encryptionContextMap.empty()) { + request.AddMetadata(ENCRYPTION_CONTEXT_V3, SerializeMap(encryptionContextMap)); + } + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-w" MUST be present for V3 format objects. + KeyWrapAlgorithm keyWrapAlgorithm = contentCryptoMaterial.GetKeyWrapAlgorithm(); + request.AddMetadata(ENCRYPTED_DATA_KEY_ALGORITHM_V3, V2ToV3Alg(GetNameForKeyWrapAlgorithm(keyWrapAlgorithm))); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-d" MUST be present for V3 format objects. + Aws::String encodedKeyCommitment = HashingUtils::Base64Encode(contentCryptoMaterial.GetKeyCommitment()); + request.AddMetadata(KEY_COMMITMENT_V3, encodedKeyCommitment); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-i" MUST be present for V3 format objects. + Aws::String encodedMessageId = HashingUtils::Base64Encode(contentCryptoMaterial.GetMessageID()); + request.AddMetadata(MESSAGE_ID_V3, encodedMessageId); + } + void MetadataHandler::PopulateRequest(Aws::S3::Model::PutObjectRequest& request, const ContentCryptoMaterial& contentCryptoMaterial) + { + if (contentCryptoMaterial.GetKeyCommitment().GetLength() != 0) + { + return PopulateRequestV3(request, contentCryptoMaterial); + } + + //= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility + //= type=implication + //# Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. + //# Objects encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF MUST use the V2 message format version only. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-key-v2" MUST be present for V2 format objects. Aws::String encodedCEK = HashingUtils::Base64Encode(contentCryptoMaterial.GetFinalCEK()); request.AddMetadata(CONTENT_KEY_HEADER, encodedCEK); + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-iv" MUST be present for V2 format objects. Aws::String encodedIV = HashingUtils::Base64Encode(contentCryptoMaterial.GetIV()); request.AddMetadata(IV_HEADER, encodedIV); + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-matdesc" MUST be present for V2 format objects. Aws::Map materialsDescriptionMap = contentCryptoMaterial.GetMaterialsDescription(); request.AddMetadata(MATERIALS_DESCRIPTION_HEADER, SerializeMap(materialsDescriptionMap)); + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-cek-alg" MUST be present for V2 format objects. ContentCryptoScheme scheme = contentCryptoMaterial.GetContentCryptoScheme(); request.AddMetadata(CONTENT_CRYPTO_SCHEME_HEADER, GetNameForContentCryptoScheme(scheme)); + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-tag-len" MUST be present for V2 format objects. request.AddMetadata(CRYPTO_TAG_LENGTH_HEADER, StringUtils::to_string(contentCryptoMaterial.GetCryptoTagLength())); + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //# - The mapkey "x-amz-wrap-alg" MUST be present for V2 format objects. KeyWrapAlgorithm keyWrapAlgorithm = contentCryptoMaterial.GetKeyWrapAlgorithm(); request.AddMetadata(KEY_WRAP_ALGORITHM, GetNameForKeyWrapAlgorithm(keyWrapAlgorithm)); } - ContentCryptoMaterial MetadataHandler::ReadContentCryptoMaterial(Aws::S3::Model::GetObjectResult& result) - { - Aws::Map metadata = result.GetMetadata(); - return ReadMetadata(metadata); - } - - ContentCryptoMaterial MetadataHandler::ReadContentCryptoMaterial(const Aws::S3::Model::HeadObjectResult & result) + ContentCryptoMaterial MetadataHandler::ReadContentCryptoMaterial(const Aws::S3::Model::HeadObjectResult & result) { Aws::Map metadata = result.GetMetadata(); return ReadMetadata(metadata); diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/KMSEncryptionMaterials.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/KMSEncryptionMaterials.cpp index f0fac7f450f..5ffc93219c3 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/KMSEncryptionMaterials.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/KMSEncryptionMaterials.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using namespace Aws; using namespace Aws::Utils; @@ -68,13 +69,25 @@ namespace Aws contentCryptoMaterial.SetKeyWrapAlgorithm(KeyWrapAlgorithm::KMS); contentCryptoMaterial.SetEncryptedContentEncryptionKey(result.GetCiphertextBlob()); contentCryptoMaterial.SetFinalCEK(result.GetCiphertextBlob()); + + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM_COMMIT) { + contentCryptoMaterial.SetV3IV(); + contentCryptoMaterial.SetMessageID(SymmetricCipher::GenerateIV(MESSAGE_ID_BYTES, false)); + + CryptoBuffer CommitmentKey(COMMITMENT_KEY_BYTES); + Aws::S3Encryption::derive_commitment_key(contentCryptoMaterial.GetContentEncryptionKey(), contentCryptoMaterial.GetMessageID(), CommitmentKey); + contentCryptoMaterial.SetKeyCommitment(CommitmentKey); + + CryptoBuffer EncryptionKey(ENCRYPTION_KEY_BYTES); + Aws::S3Encryption::derive_encryption_key(contentCryptoMaterial.GetContentEncryptionKey(), contentCryptoMaterial.GetMessageID(), EncryptionKey); + contentCryptoMaterial.SetContentEncryptionKey(EncryptionKey); + } return CryptoOutcome(Aws::NoResult()); } CryptoOutcome KMSEncryptionMaterialsBase::DecryptCEK(ContentCryptoMaterial &contentCryptoMaterial) { auto errorOutcome = CryptoOutcome(AWSError(CryptoErrors::DECRYPT_CONTENT_ENCRYPTION_KEY_FAILED, "DecryptContentEncryptionKeyFailed", "Failed to decrypt content encryption key(CEK)", false/*not retryable*/)); - if (m_customerMasterKeyID.empty() && IsKMSDecryptWithAnyCMKAllowed() == false) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Failed to decrypt content encryption key(CEK): KMS CMK is not provided and CMK Any is not allowed."); @@ -112,7 +125,20 @@ namespace Aws } DecryptResult result = outcome.GetResult(); - contentCryptoMaterial.SetContentEncryptionKey(CryptoBuffer(result.GetPlaintext())); + auto decryptedKey = CryptoBuffer(result.GetPlaintext()); + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM_COMMIT) { + CryptoBuffer CommitmentKey(COMMITMENT_KEY_BYTES); + derive_commitment_key(decryptedKey, contentCryptoMaterial.GetMessageID(), CommitmentKey); + if (!constant_time_equal(CommitmentKey, contentCryptoMaterial.GetKeyCommitment())) { + AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Commitment Key does not match."); + return errorOutcome; + } + CryptoBuffer EncryptionKey(ENCRYPTION_KEY_BYTES); + derive_encryption_key(decryptedKey, contentCryptoMaterial.GetMessageID(), EncryptionKey); + contentCryptoMaterial.SetContentEncryptionKey(EncryptionKey); + } else { + contentCryptoMaterial.SetContentEncryptionKey(decryptedKey); + } if (contentCryptoMaterial.GetContentEncryptionKey().GetLength() == 0u) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Content Encryption Key could not be decrypted."); @@ -159,6 +185,9 @@ namespace Aws // Should be "AES/GCM/NoPadding" by default Aws::String cekAlg = ContentCryptoSchemeMapper::GetNameForContentCryptoScheme(contentCryptoMaterial.GetContentCryptoScheme()); contentCryptoMaterial.AddMaterialsDescription(kmsEncryptionContextKey, cekAlg); + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. request.SetEncryptionContext(contentCryptoMaterial.GetMaterialsDescription()); request.SetKeySpec(DataKeySpec::AES_256); GenerateDataKeyOutcome outcome = m_kmsClient->GenerateDataKey(request); @@ -175,6 +204,46 @@ namespace Aws contentCryptoMaterial.SetContentEncryptionKey(result.GetPlaintext()); contentCryptoMaterial.SetEncryptedContentEncryptionKey(result.GetCiphertextBlob()); contentCryptoMaterial.SetFinalCEK(result.GetCiphertextBlob()); + + //= ../specification/s3-encryption/decryption.md#decrypting-with-commitment + //= type=implication + //# When using an algorithm suite which supports key commitment, the client MUST verify that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the same bytes as the stored key commitment retrieved from the stored object's metadata. + // All four of these are implications, because there's no reasonable way to test it. + + //= ../specification/s3-encryption/decryption.md#decrypting-with-commitment + //= type=implication + //# When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + + //= ../specification/s3-encryption/decryption.md#decrypting-with-commitment + //= type=implication + //# When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value and stored key commitment value do not match. + + //= ../specification/s3-encryption/decryption.md#decrypting-with-commitment + //= type=implication + //# When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving the [derived encryption key](./key-derivation.md#hkdf-operation). + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM_COMMIT) { + + //= ../specification/s3-encryption/encryption.md#content-encryption + //= type=implication + //# The client MUST generate an IV or Message ID using the length of the IV or Message ID defined in the algorithm suite. + + //= ../specification/s3-encryption/encryption.md#content-encryption + //= type=implication + //# The generated IV or Message ID MUST be set or returned from the encryption process such that it can be included in the content metadata. + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + //= type=implication + //# The derived key commitment value MUST be set or returned from the encryption process such that it can be included in the content metadata. + contentCryptoMaterial.SetV3IV(); + contentCryptoMaterial.SetMessageID(SymmetricCipher::GenerateIV(MESSAGE_ID_BYTES, false)); + CryptoBuffer CommitmentKey(COMMITMENT_KEY_BYTES); + derive_commitment_key(contentCryptoMaterial.GetContentEncryptionKey(), contentCryptoMaterial.GetMessageID(), CommitmentKey); + contentCryptoMaterial.SetKeyCommitment(CommitmentKey); + + CryptoBuffer EncryptionKey(ENCRYPTION_KEY_BYTES); + derive_encryption_key(contentCryptoMaterial.GetContentEncryptionKey(), contentCryptoMaterial.GetMessageID(), EncryptionKey); + contentCryptoMaterial.SetContentEncryptionKey(EncryptionKey); + } return CryptoOutcome(Aws::NoResult()); } }//namespace Materials diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/SimpleEncryptionMaterials.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/SimpleEncryptionMaterials.cpp index bdb5c42d35e..cbf6eb56c59 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/SimpleEncryptionMaterials.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/materials/SimpleEncryptionMaterials.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace Aws::Utils; using namespace Aws::Utils::Crypto; @@ -67,6 +68,10 @@ namespace Aws CryptoOutcome SimpleEncryptionMaterialsBase::EncryptCEK(ContentCryptoMaterial& contentCryptoMaterial) { // non empty context map passed in. + //= ../specification/s3-encryption/data-format/content-metadata.md#v3-only + //= type=implication + //# The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + //# If the mapkey x-amz-m is not present, the default Material Description value MUST be set to an empty map (`{}`). if (!contentCryptoMaterial.GetMaterialsDescription().empty()) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Customized encryption context map is not allowed for AES/GCM Key wrap algorithm."); @@ -92,6 +97,18 @@ namespace Aws contentCryptoMaterial.SetEncryptedContentEncryptionKey(CryptoBuffer({ &encryptResult, &encryptFinalizeResult })); contentCryptoMaterial.SetCEKGCMTag(cipher->GetTag()); + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM_COMMIT) { + contentCryptoMaterial.SetV3IV(); + contentCryptoMaterial.SetMessageID(SymmetricCipher::GenerateIV(MESSAGE_ID_BYTES, false)); + CryptoBuffer CommitmentKey(COMMITMENT_KEY_BYTES); + Aws::S3Encryption::derive_commitment_key(contentEncryptionKey, contentCryptoMaterial.GetMessageID(), CommitmentKey); + contentCryptoMaterial.SetKeyCommitment(CommitmentKey); + + CryptoBuffer EncryptionKey(ENCRYPTION_KEY_BYTES); + Aws::S3Encryption::derive_encryption_key(contentEncryptionKey, contentCryptoMaterial.GetMessageID(), EncryptionKey); + contentCryptoMaterial.SetContentEncryptionKey(EncryptionKey); + } + if (contentCryptoMaterial.GetKeyWrapAlgorithm() == KeyWrapAlgorithm::AES_GCM) { CryptoBuffer iv = contentCryptoMaterial.GetCekIV(); @@ -147,9 +164,21 @@ namespace Aws const CryptoBuffer& encryptedContentEncryptionKey = contentCryptoMaterial.GetEncryptedContentEncryptionKey(); CryptoBuffer&& decryptResult = cipher->DecryptBuffer(encryptedContentEncryptionKey); CryptoBuffer&& decryptFinalizeResult = cipher->FinalizeDecryption(); - CryptoBuffer decryptedBuffer = CryptoBuffer({ &decryptResult, &decryptFinalizeResult }); - contentCryptoMaterial.SetContentEncryptionKey(decryptedBuffer); - if (!(*cipher) || contentCryptoMaterial.GetContentEncryptionKey().GetLength() == 0u) + CryptoBuffer decryptedKey = CryptoBuffer({ &decryptResult, &decryptFinalizeResult }); + if (contentCryptoMaterial.GetContentCryptoScheme() == ContentCryptoScheme::GCM_COMMIT) { + CryptoBuffer CommitmentKey(COMMITMENT_KEY_BYTES); + Aws::S3Encryption::derive_commitment_key(decryptedKey, contentCryptoMaterial.GetMessageID(), CommitmentKey); + if (!constant_time_equal(CommitmentKey, contentCryptoMaterial.GetKeyCommitment())) { + AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Commitment Key does not match."); + return errorOutcome; + } + CryptoBuffer EncryptionKey(ENCRYPTION_KEY_BYTES); + Aws::S3Encryption::derive_encryption_key(decryptedKey, contentCryptoMaterial.GetMessageID(), EncryptionKey); + contentCryptoMaterial.SetContentEncryptionKey(EncryptionKey); + } else { + contentCryptoMaterial.SetContentEncryptionKey(decryptedKey); + } + if (cipher->Fail() || contentCryptoMaterial.GetContentEncryptionKey().GetLength() == 0u) { AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Content Encryption Key could not be decrypted."); return errorOutcome; diff --git a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/modules/CryptoModule.cpp b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/modules/CryptoModule.cpp index 9234a9fcc7f..9e87683e9ce 100644 --- a/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/modules/CryptoModule.cpp +++ b/src/aws-cpp-sdk-s3-encryption/source/s3-encryption/modules/CryptoModule.cpp @@ -37,6 +37,26 @@ namespace Aws { } + //= ../specification/s3-encryption/encryption.md#content-encryption + //= type=implication + //# The client MUST validate that the length of the plaintext bytes does not exceed the algorithm suite's cipher's maximum content length in bytes. + // The expectation is that this is handled by the underlying cryptographic provider. + // For example, if this is OpenSSL, + // See OpenSSL: https://github.com/openssl/openssl/blob/master/crypto/modes/gcm128.c#L784 + // The relevant line is: + // if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len)) + // return -1; + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-ctr-iv16-tag16-no-kdf + //= type=implication + //# Attempts to encrypt using AES-CTR MUST fail. + // There is no way to attempt to use AES-CTR + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-ctr-hkdf-sha512-commit-key + //= type=implication + //# Attempts to encrypt using key committing AES-CTR MUST fail. + // There is no way to attempt to use key committing AES-CTR + S3EncryptionPutObjectOutcome CryptoModule::PutObjectSecurely(const Aws::S3::Model::PutObjectRequest& request, const PutObjectFunction& putObjectFunction, const Aws::Map& contextMap) { PutObjectRequest copyRequest(request); @@ -57,7 +77,7 @@ namespace Aws PutObjectRequest instructionFileRequest; instructionFileRequest.WithBucket(copyRequest.GetBucket()); instructionFileRequest.WithKey(copyRequest.GetKey()); - handler.PopulateRequest(instructionFileRequest, m_contentCryptoMaterial); + handler.PopulateRequest(copyRequest, instructionFileRequest, m_contentCryptoMaterial); PutObjectOutcome instructionOutcome = putObjectFunction(instructionFileRequest); if (!instructionOutcome.IsSuccess()) { @@ -102,7 +122,7 @@ namespace Aws rangeEnd = range.second; } - InitDecryptionCipher(rangeStart, rangeEnd, tagFromBody); + InitDecryptionCipher(rangeStart, rangeEnd, tagFromBody, m_contentCryptoMaterial.GetAAD()); auto newRange = AdjustRange(copyRequest, headObjectResult); if (newRange.first > newRange.second) { @@ -160,6 +180,16 @@ namespace Aws } GetObjectResult&& result = outcome.GetResultWithOwnership(); + + //= ../specification/s3-encryption/decryption.md#ranged-gets + //= type=implication + //# If the GetObject response contains a range, but the GetObject request does not contain a range, the S3EC MUST throw an exception. + // implication because there's no way to test this + if (request.GetRange().empty() && !result.GetContentRange().empty()) { + AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "S3 Response contained a range, but the request did not."); + return S3EncryptionGetObjectOutcome(BuildS3EncryptionError(AWSError(S3Errors::VALIDATION, "FailedToDecryptContent", + "S3 Response contained a range, but the request did not.", false/*not retryable*/))); + } ((SymmetricCryptoStream&)result.GetBody()).Finalize(); userSuppliedStream->clear(); @@ -241,7 +271,7 @@ namespace Aws m_contentCryptoMaterial.SetIV(m_cipher->GetIV()); } - void CryptoModuleEO::InitDecryptionCipher(int64_t, int64_t, const Aws::Utils::CryptoBuffer &) + void CryptoModuleEO::InitDecryptionCipher(int64_t, int64_t, const Aws::Utils::CryptoBuffer &, const Aws::Utils::CryptoBuffer &) { m_cipher = CreateAES_CBCImplementation(m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV()); } @@ -288,22 +318,51 @@ namespace Aws { m_contentCryptoMaterial.SetContentEncryptionKey(SymmetricCipher::GenerateKey()); m_contentCryptoMaterial.SetCryptoTagLength(TAG_SIZE_BYTES * BITS_IN_BYTE); - m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM); - auto aad_raw = GetNameForContentCryptoScheme(ContentCryptoScheme::GCM); + if (m_cryptoConfig.GetEncryptionAlgorithm() == AlgorithmSuite::AES_GCM_WITH_COMMITMENT) { + m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM_COMMIT); + } else { + m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM); + } + auto aad_raw = GetNameForContentCryptoScheme(m_contentCryptoMaterial.GetContentCryptoScheme()); m_contentCryptoMaterial.SetGCMAAD(CryptoBuffer((const unsigned char*)aad_raw.c_str(), aad_raw.size())); } + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=implication + //# The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + //# and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + //= type=implication + //# The client MUST initialize the cipher, or call an AES-GCM encryption API, with the plaintext data key, the generated IV, and the tag length defined in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + //= type=implication + //# The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + + //= ../specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + //= type=implication + //# The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. void CryptoModuleAE::InitEncryptionCipher() { - m_cipher = Aws::MakeShared(ALLOCATION_TAG, m_contentCryptoMaterial.GetContentEncryptionKey()); - m_contentCryptoMaterial.SetIV(m_cipher->GetIV()); + m_cipher = Aws::MakeShared(ALLOCATION_TAG, m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), m_contentCryptoMaterial.GetAAD()); + if (m_contentCryptoMaterial.GetContentCryptoScheme() != ContentCryptoScheme::GCM_COMMIT) + m_contentCryptoMaterial.SetIV(m_cipher->GetIV()); } - void CryptoModuleAE::InitDecryptionCipher(int64_t rangeStart, int64_t rangeEnd, const Aws::Utils::CryptoBuffer& tag) + void CryptoModuleAE::InitDecryptionCipher(int64_t rangeStart, int64_t rangeEnd, const Aws::Utils::CryptoBuffer& tag, const Aws::Utils::CryptoBuffer& aad) { (void)GCM_IV_SIZE; // get ride of unused variable warning if (rangeStart > 0 || rangeEnd > 0) { + //= ../specification/s3-encryption/decryption.md#ranged-gets + //= type=implication + //# If the object was encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF, then ALG_AES_256_CTR_IV16_TAG16_NO_KDF MUST be used to decrypt the range of the object. + + //= ../specification/s3-encryption/decryption.md#ranged-gets + //= type=implication + //# If the object was encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, then ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY MUST be used to decrypt the range of the object. + //See http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf for decrypting a GCM message using CTR mode. assert(m_contentCryptoMaterial.GetIV().GetLength() == GCM_IV_SIZE); CryptoBuffer counter(4); @@ -316,7 +375,7 @@ namespace Aws } else { - m_cipher = CreateAES_GCMImplementation(m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), tag); + m_cipher = CreateAES_GCMImplementation(m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), tag, aad); } } @@ -356,6 +415,9 @@ namespace Aws return true; } + //= ../specification/s3-encryption/decryption.md#ranged-gets + //= type=implication + //# If the S3EC supports Ranged Gets, the S3EC MUST adjust the customer-provided range to include the beginning and end of the cipher blocks for the given range. std::pair CryptoModuleAE::AdjustRange(Aws::S3::Model::GetObjectRequest& getObjectRequest, const Aws::S3::Model::HeadObjectResult& headObjectResult) { Aws::StringStream ss; @@ -412,25 +474,30 @@ namespace Aws { m_contentCryptoMaterial.SetContentEncryptionKey(SymmetricCipher::GenerateKey()); m_contentCryptoMaterial.SetCryptoTagLength(TAG_SIZE_BYTES * BITS_IN_BYTE); - m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM); - auto aad_raw = GetNameForContentCryptoScheme(ContentCryptoScheme::GCM); + if (m_cryptoConfig.GetEncryptionAlgorithm() == AlgorithmSuite::AES_GCM_WITH_COMMITMENT) { + m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM_COMMIT); + } else { + m_contentCryptoMaterial.SetContentCryptoScheme(ContentCryptoScheme::GCM); + } + auto aad_raw = GetNameForContentCryptoScheme(m_contentCryptoMaterial.GetContentCryptoScheme()); m_contentCryptoMaterial.SetGCMAAD(CryptoBuffer((const unsigned char*)aad_raw.c_str(), aad_raw.size())); } void CryptoModuleStrictAE::InitEncryptionCipher() { - m_cipher = Aws::MakeShared(ALLOCATION_TAG, m_contentCryptoMaterial.GetContentEncryptionKey()); - m_contentCryptoMaterial.SetIV(m_cipher->GetIV()); + m_cipher = Aws::MakeShared(ALLOCATION_TAG, m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), m_contentCryptoMaterial.GetAAD()); + if (m_contentCryptoMaterial.GetContentCryptoScheme() != ContentCryptoScheme::GCM_COMMIT) + m_contentCryptoMaterial.SetIV(m_cipher->GetIV()); } - void CryptoModuleStrictAE::InitDecryptionCipher(int64_t rangeStart, int64_t rangeEnd, const Aws::Utils::CryptoBuffer & tag) + void CryptoModuleStrictAE::InitDecryptionCipher(int64_t rangeStart, int64_t rangeEnd, const Aws::Utils::CryptoBuffer & tag, const Aws::Utils::CryptoBuffer & aad) { //range gets not allowed in Strict AE. assert(rangeStart == 0); assert(rangeEnd == 0); AWS_UNREFERENCED_PARAM(rangeStart); AWS_UNREFERENCED_PARAM(rangeEnd); - m_cipher = CreateAES_GCMImplementation(m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), tag); + m_cipher = CreateAES_GCMImplementation(m_contentCryptoMaterial.GetContentEncryptionKey(), m_contentCryptoMaterial.GetIV(), tag, aad); } Aws::Utils::CryptoBuffer CryptoModuleStrictAE::GetTag(const Aws::S3::Model::GetObjectRequest & request, const std::function < Aws::S3::Model::GetObjectOutcome(const Aws::S3::Model::GetObjectRequest&) >& getObjectFunction) @@ -455,7 +522,7 @@ namespace Aws AWS_LOGSTREAM_FATAL(ALLOCATION_TAG, "Range-Get Operations are not allowed with Strict Authenticated Encryption mode."); return false; } - if (m_contentCryptoMaterial.GetContentCryptoScheme() != ContentCryptoScheme::GCM) + if (!IsGCM(m_contentCryptoMaterial.GetContentCryptoScheme())) { AWS_LOGSTREAM_FATAL(ALLOCATION_TAG, "Strict Authentication Encryption only allows decryption of GCM encrypted objects."); return false; @@ -472,8 +539,8 @@ namespace Aws - AES_GCM_AppendedTag::AES_GCM_AppendedTag(const CryptoBuffer& key) : Aws::Utils::Crypto::SymmetricCipher(), - m_cipher(CreateAES_GCMImplementation(key)) + AES_GCM_AppendedTag::AES_GCM_AppendedTag(const CryptoBuffer& key, const CryptoBuffer& iv, const CryptoBuffer& aad) : Aws::Utils::Crypto::SymmetricCipher(), + m_cipher(CreateAES_GCMImplementation(key, iv, CryptoBuffer(), aad)) { m_key = key; m_initializationVector = m_cipher->GetIV(); @@ -506,7 +573,6 @@ namespace Aws CryptoBuffer AES_GCM_AppendedTag::FinalizeDecryption() { - //don't use this in decryption mode. assert(0); m_failure = true; diff --git a/tests/aws-cpp-sdk-s3-encryption-integration-tests/LiveClientTests.cpp b/tests/aws-cpp-sdk-s3-encryption-integration-tests/LiveClientTests.cpp index 9228a47b21d..6dff87c4178 100644 --- a/tests/aws-cpp-sdk-s3-encryption-integration-tests/LiveClientTests.cpp +++ b/tests/aws-cpp-sdk-s3-encryption-integration-tests/LiveClientTests.cpp @@ -23,6 +23,7 @@ #include #include #include +#include // TODO: temporary fix for naming conflicts on Windows. #if _WIN32 @@ -44,6 +45,11 @@ using namespace Aws::Utils; using namespace Aws::Client; using namespace Aws::Utils::Crypto; using namespace Aws::Region; +using namespace Aws::Utils::Json; + +// Public key owned by CryptoTools +// Needed for EncryptionContext tests, that need KMS materials +static const char * KMS_KEY = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f"; static const char* ENCRYPTED_BUCKET_TEST_NAME = "awsnativesdks3encotest"; static const char* ALLOCATION_TAG = "LiveClientTest"; @@ -235,6 +241,476 @@ TEST_F(LiveClientTest, TestEncryptionContextEmpty) EXPECT_FALSE(getObjectResult.IsSuccess()); } +TEST_F(LiveClientTest, TestV2EncryptionContextEmptyKms) +{ + auto materials = std::make_shared(KMS_KEY); + CryptoConfigurationV2 configuration(materials); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestEncryptionContextKms"; + + S3EncryptionClientV2 client(configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + getObjectResult = client.GetObject(getObjectRequest, {}); + AWS_EXPECT_SUCCESS(getObjectResult); + getObjectResult = client.GetObject(getObjectRequest, {{"foo", "bar"}}); + EXPECT_FALSE(getObjectResult.IsSuccess()); +} + +TEST_F(LiveClientTest, TestV2EncryptionContextFullKms) +{ + auto materials = std::make_shared(KMS_KEY); + CryptoConfigurationV2 configuration(materials); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestEncryptionContextKms"; + + S3EncryptionClientV2 client(configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + // PUT with non-empty EC + auto putObjectResult = client.PutObject(putObjectRequest, {{"foo", "bar"}}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + + getObjectResult = client.GetObject(getObjectRequest, {}); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + getObjectResult = client.GetObject(getObjectRequest, {{"foo", "bar"}}); + AWS_EXPECT_SUCCESS(getObjectResult); + + getObjectResult = client.GetObject(getObjectRequest, {{"foo", "baz"}}); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + getObjectResult = client.GetObject(getObjectRequest, {{"boo", "bar"}}); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + getObjectResult = client.GetObject(getObjectRequest, {{"foo", "bar"}, {"foo2", "bar"}}); + EXPECT_FALSE(getObjectResult.IsSuccess()); +} + +//= ../specification/s3-encryption/key-commitment.md#commitment-policy +//= type=test +//# When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. +//# When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + +//= ../specification/s3-encryption/key-commitment.md#commitment-policy +//= type=test +//# When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + +//= ../specification/s3-encryption/key-commitment.md#commitment-policy +//= type=test +//# When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. + +//= ../specification/s3-encryption/decryption.md#key-commitment +//= type=test +//# The S3EC MUST validate the algorithm suite used for decryption against the key commitment policy before attempting to decrypt the content ciphertext. +//# If the commitment policy requires decryption using a committing algorithm suite, and the algorithm suite associated with the object does not support key commitment, then the S3EC MUST throw an exception. + +TEST_F(LiveClientTest, TestWriteV2ReadV3) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV2 v2_configuration(materials); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestWriteV2ReadV3"; + + S3EncryptionClientV2 v2_client(v2_configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = v2_client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + CryptoConfigurationV3 v3_configuration(materials); + S3EncryptionClientV3 v3_client_dflt(v3_configuration, s3ClientConfig); + + auto getObjectResult = v3_client_dflt.GetObject(getObjectRequest); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + S3EncryptionClientV3 v3_client_1(v3_configuration, s3ClientConfig); + getObjectResult = v3_client_1.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + S3EncryptionClientV3 v3_client_2(v3_configuration, s3ClientConfig); + getObjectResult = v3_client_2.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + S3EncryptionClientV3 v3_client_3(v3_configuration, s3ClientConfig); + getObjectResult = v3_client_3.GetObject(getObjectRequest); + EXPECT_FALSE(getObjectResult.IsSuccess()); +} + +//= ../specification/s3-encryption/decryption.md#legacy-decryption +//= type=test +//# The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites unless specifically configured to do so. +//# If the S3EC is not configured to enable legacy unauthenticated content decryption, the client MUST throw an exception when attempting to decrypt an object encrypted with a legacy unauthenticated algorithm suite. + +//= ../specification/s3-encryption/client.md#enable-legacy-wrapping-algorithms +//= type=test +//# When enabled, the S3EC MUST be able to decrypt objects encrypted with all supported wrapping algorithms (both legacy and fully supported). +//# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy wrapping algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy wrapping algorithm. + +//= ../specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes +//= type=test +//# When enabled, the S3EC MUST be able to decrypt objects encrypted with all content encryption algorithms (both legacy and fully supported). +//# When disabled, the S3EC MUST NOT decrypt objects encrypted using legacy content encryption algorithms; it MUST throw an exception when attempting to decrypt an object encrypted with a legacy content encryption algorithm. + +TEST_F(LiveClientTest, TestWriteV1ReadV3) +{ + CryptoConfiguration configuration; + configuration.SetCryptoMode(CryptoMode::ENCRYPTION_ONLY); + configuration.SetStorageMethod(StorageMethod::METADATA); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + + static const char* objectKey = "TestWriteV1ReadV3"; + + S3EncryptionClient client(materials, configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = client.PutObject(putObjectRequest); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_ASSERT_SUCCESS(getObjectResult); + + CryptoConfigurationV3 v3_configuration(materials); + S3EncryptionClientV3 v3_client_dflt(v3_configuration, s3ClientConfig); + + getObjectResult = v3_client_dflt.GetObject(getObjectRequest); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + S3EncryptionClientV3 v3_client_1(v3_configuration, s3ClientConfig); + getObjectResult = v3_client_1.GetObject(getObjectRequest); + EXPECT_FALSE(getObjectResult.IsSuccess()); + + v3_configuration.AllowLegacy(true); + S3EncryptionClientV3 v3_client_2(v3_configuration, s3ClientConfig); + getObjectResult = v3_client_2.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); +} + +TEST_F(LiveClientTest, TestWriteV3ReadV2) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV3 v3_configuration(materials); + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestWriteV2ReadV3"; + + S3EncryptionClientV3 v3_client(v3_configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = v3_client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + CryptoConfigurationV2 v2_configuration(materials); + S3EncryptionClientV2 v2_client(v2_configuration, s3ClientConfig); + + auto getObjectResult = v2_client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); +} + +//= ../specification/s3-encryption/key-commitment.md#commitment-policy +//= type=test +//# When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + +//= ../specification/s3-encryption/key-commitment.md#commitment-policy +//= type=test +//# When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. +TEST_F(LiveClientTest, TestWriteV3ReadV3) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV3 v3_configuration(materials); + v3_configuration.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestWriteV3ReadV3"; + + S3EncryptionClientV3 v3_client(v3_configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = v3_client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = v3_client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + + CryptoConfigurationV3 req_req_conf(materials); + S3EncryptionClientV3 req_req_client(req_req_conf, s3ClientConfig); + + getObjectResult = req_req_client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); +} + +static Aws::Map DeserializeMap(const Aws::String& jsonString) +{ + Aws::Map materialsDescriptionMap; + JsonValue jsonObject(jsonString); + if (jsonObject.WasParseSuccessful()) + { + Aws::Map jsonMap = jsonObject.View().GetAllObjects(); + for (auto& mapItem : jsonMap) + { + materialsDescriptionMap[mapItem.first] = mapItem.second.AsString(); + } + return materialsDescriptionMap; + } + else + { + AWS_LOGSTREAM_ERROR(ALLOCATION_TAG, "Json Parse failed with message: " << jsonObject.GetErrorMessage()); + return materialsDescriptionMap; + } +} + +TEST_F(LiveClientTest, TestWriteV2Instruction) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV2 configuration(materials); + configuration.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestWriteV2Instruction"; + static const char* objectKeyInst = "TestWriteV2Instruction.instruction"; + + S3EncryptionClientV2 client(configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_ASSERT_SUCCESS(getObjectResult); + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + //= type=test + //# In the V1/V2 message format, all of the content metadata MUST be stored in the Instruction File. + auto metadata = getObjectResult.GetResult().GetMetadata(); + EXPECT_TRUE(metadata.empty()); + + getObjectRequest.WithKey(objectKeyInst); + Aws::S3::S3Client s3Client(s3ClientConfig); + + getObjectResult = s3Client.GetObject(getObjectRequest); + AWS_ASSERT_SUCCESS(getObjectResult); + + metadata = getObjectResult.GetResult().GetMetadata(); + EXPECT_TRUE(metadata.size() == 1); + EXPECT_STREQ("default instruction file header", metadata["x-amz-crypto-instr-file"].c_str()); + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + //= type=test + //# The content metadata stored in the Instruction File MUST be serialized to a JSON string. + //# The serialized JSON string MUST be the only contents of the Instruction File. + Aws::StringStream responseStringStream; + responseStringStream << getObjectResult.GetResult().GetBody().rdbuf(); + auto jsonString = responseStringStream.str(); + auto inst_metadata = DeserializeMap(jsonString); +} + +static bool contains(Aws::Map& m, const char * str) +{ + return m.find(str) != m.end(); +} + +TEST_F(LiveClientTest, TestWriteV3Instruction) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV3 configuration(materials); + configuration.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestWriteV3Instruction"; + static const char* objectKeyInst = "TestWriteV3Instruction.instruction"; + + S3EncryptionClientV3 client(configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_ASSERT_SUCCESS(getObjectResult); + + auto metadata = getObjectResult.GetResult().GetMetadata(); + + getObjectRequest.WithKey(objectKeyInst); + Aws::S3::S3Client s3Client(s3ClientConfig); + + auto getInstrResult = s3Client.GetObject(getObjectRequest); + AWS_ASSERT_SUCCESS(getInstrResult); + + auto inst_metadata = getInstrResult.GetResult().GetMetadata(); + + EXPECT_TRUE(inst_metadata.size() == 1); + EXPECT_STREQ("default instruction file header", inst_metadata["x-amz-crypto-instr-file"].c_str()); + + Aws::StringStream responseStringStream; + responseStringStream << getInstrResult.GetResult().GetBody().rdbuf(); + auto jsonString = responseStringStream.str(); + inst_metadata = DeserializeMap(jsonString); + + //= ../specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + //= type=test + //# - The V3 message format MUST store the mapkey "x-amz-c" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-d" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-i" and its value in the Object Metadata when writing with an Instruction File. + //# - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. + //# + //# - The V3 message format MUST store the mapkey "x-amz-3" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-w" and its value in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-m" and its value (when present in the content metadata) in the Instruction File. + //# - The V3 message format MUST store the mapkey "x-amz-t" and its value (when present in the content metadata) in the Instruction File. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# In the V3 format, the mapkeys "x-amz-c", "x-amz-d", and "x-amz-i" MUST be stored exclusively in the Object Metadata. + + EXPECT_TRUE(contains(metadata, "x-amz-c" )); + EXPECT_FALSE(contains(inst_metadata, "x-amz-c" )); + EXPECT_TRUE(contains(metadata, "x-amz-d" )); + EXPECT_FALSE(contains(inst_metadata, "x-amz-d" )); + EXPECT_TRUE(contains(metadata, "x-amz-i" )); + EXPECT_FALSE(contains(inst_metadata, "x-amz-i" )); + + EXPECT_TRUE(contains(inst_metadata, "x-amz-3" )); + EXPECT_TRUE(contains(inst_metadata, "x-amz-w" )); +} + TEST_F(LiveClientTest, TestAEMode) { CryptoConfiguration configuration; @@ -410,6 +886,86 @@ TEST_F(LiveClientTest, TestAEModeRangeGet) AWS_EXPECT_SUCCESS(deleteResult); } +//= ../specification/s3-encryption/client.md#enable-delayed-authentication +//= type=test +//# When enabled, the S3EC MAY release plaintext from a stream which has not been authenticated. +//# When disabled the S3EC MUST NOT release plaintext from a stream which has not been authenticated. + +TEST_F(LiveClientTest, TestGcmModeRangeGet) +{ + auto key = SymmetricCipher::GenerateKey(); + auto materials = Aws::MakeShared(ALLOCATION_TAG, key); + CryptoConfigurationV3 configuration(materials); + + ClientConfiguration s3ClientConfig; + s3ClientConfig.region = AWS_TEST_REGION; + + static const char* objectKey = "TestAERangeGetKey"; + + S3EncryptionClientV3 unauth_client(configuration, s3ClientConfig); + configuration.SetUnAuthenticatedRangeGet(RangeGetMode::ALL); + S3EncryptionClientV3 client(configuration, s3ClientConfig); + + Model::PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()) + .WithKey(objectKey); + + auto ss = Aws::MakeShared(ALLOCATION_TAG); + *ss << TEST_STRING; + ss->flush(); + + putObjectRequest.SetBody(ss); + + auto putObjectResult = client.PutObject(putObjectRequest, {}); + AWS_ASSERT_SUCCESS(putObjectResult); + + Model::GetObjectRequest getObjectRequest; + getObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey).WithRange(RANGE_GET_STR); + + auto getObjectResult = client.GetObject(getObjectRequest); + AWS_EXPECT_SUCCESS(getObjectResult); + + Aws::StringStream responseStringStream; + responseStringStream << getObjectResult.GetResult().GetBody().rdbuf(); + + EXPECT_STREQ(RANGE_GET_TEST_STRING, responseStringStream.str().c_str()); + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-c" MUST be present for V3 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-3" MUST be present for V3 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-w" MUST be present for V3 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-d" MUST be present for V3 format objects. + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-i" MUST be present for V3 format objects. + + auto metadata = getObjectResult.GetResult().GetMetadata(); + auto encKeyStr = metadata["x-amz-3"]; + auto cipherStr = metadata["x-amz-c"]; + auto commitKeyStr = metadata["x-amz-d"]; + auto messageIdStr = metadata["x-amz-i"]; + auto algStr = metadata["x-amz-w"]; + EXPECT_FALSE(encKeyStr.empty()); + EXPECT_STREQ("115", cipherStr.c_str()); + EXPECT_FALSE(commitKeyStr.empty()); + EXPECT_FALSE(messageIdStr.empty()); + EXPECT_STREQ("02", algStr.c_str()); + + getObjectResult = unauth_client.GetObject(getObjectRequest); + ASSERT_FALSE(getObjectResult.IsSuccess()); +} + TEST_F(LiveClientTest, TestS3EncryptionError) { ClientConfiguration s3ClientConfig; @@ -482,15 +1038,29 @@ TEST_F(LiveClientTest, TestV2AEMode) EXPECT_STREQ(TEST_STRING, responseStringStream.str().c_str()); auto metadata = getObjectResult.GetResult().GetMetadata(); - auto ivStr = metadata["x-amz-iv"]; + + //= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys + //= type=test + //# - The mapkey "x-amz-key-v2" MUST be present for V2 format objects. + //# - The mapkey "x-amz-matdesc" MUST be present for V2 format objects. + //# - The mapkey "x-amz-iv" MUST be present for V2 format objects. + //# - The mapkey "x-amz-wrap-alg" MUST be present for V2 format objects. + //# - The mapkey "x-amz-cek-alg" MUST be present for V2 format objects. + //# - The mapkey "x-amz-tag-len" MUST be present for V2 format objects. + auto cekStr = metadata["x-amz-key-v2"]; - auto tagLenStr = metadata["x-amz-tag-len"]; + auto matdesc = metadata["x-amz-matdesc"]; + auto ivStr = metadata["x-amz-iv"]; + auto alg = metadata["x-amz-wrap-alg"]; auto aad = metadata["x-amz-cek-alg"]; - EXPECT_FALSE(ivStr.empty()); + auto tagLenStr = metadata["x-amz-tag-len"]; EXPECT_FALSE(cekStr.empty()); - EXPECT_FALSE(tagLenStr.empty()); - EXPECT_STREQ("128", tagLenStr.c_str()); + EXPECT_STREQ("{}", matdesc.c_str()); + EXPECT_FALSE(ivStr.empty()); + EXPECT_STREQ("AES/GCM", alg.c_str()); + EXPECT_STREQ("AES/GCM/NoPadding", aad.c_str()); EXPECT_FALSE(aad.empty()); + EXPECT_STREQ("128", tagLenStr.c_str()); ContentCryptoMaterial cryptoMaterial(ContentCryptoScheme::GCM); cryptoMaterial.SetFinalCEK(HashingUtils::Base64Decode(cekStr)); diff --git a/tests/aws-cpp-sdk-s3-encryption-tests/DataHandlersTest.cpp b/tests/aws-cpp-sdk-s3-encryption-tests/DataHandlersTest.cpp index 8ff2d4de274..e156eabb71c 100644 --- a/tests/aws-cpp-sdk-s3-encryption-tests/DataHandlersTest.cpp +++ b/tests/aws-cpp-sdk-s3-encryption-tests/DataHandlersTest.cpp @@ -14,11 +14,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include static const char* const INSTRUCTION_HEADER_VALUE = "default instruction file header"; @@ -33,7 +35,7 @@ class MockS3Client : public Aws::S3::S3Client MockS3Client(Aws::Client::ClientConfiguration clientConfiguration = Aws::Client::ClientConfiguration()) : S3Client(Aws::Auth::AWSCredentials("", ""), Aws::MakeShared(Aws::S3::S3Client::GetAllocationTag()), - clientConfiguration), m_putObjectCalled(0), m_getObjectCalled(0), m_body(nullptr) + clientConfiguration), m_putObjectCalled(0), m_getObjectCalled(0), m_headObjectCalled(0), m_body(nullptr) { } @@ -62,8 +64,17 @@ class MockS3Client : public Aws::S3::S3Client return Aws::S3::Model::GetObjectOutcome(std::move(getObjectResult)); } + Aws::S3::Model::HeadObjectOutcome HeadObject(const Aws::S3::Model::HeadObjectRequest& ) const override + { + m_headObjectCalled++; + Aws::S3::Model::HeadObjectResult headObjectResult; + headObjectResult.SetMetadata(m_metadata); + return Aws::S3::Model::HeadObjectOutcome(std::move(headObjectResult)); + } + mutable size_t m_putObjectCalled; mutable size_t m_getObjectCalled; + mutable size_t m_headObjectCalled; mutable Aws::Map m_metadata; mutable std::shared_ptr m_body; }; @@ -108,7 +119,7 @@ namespace contentCryptoMaterial.SetMaterialsDescription(testMap); } - static void PopulateGetObjectResultMetadata(GetObjectResult& result) + static void PopulateHeadObjectResultMetadata(HeadObjectResult& result) { auto metadata = result.GetMetadata(); auto cekIV = SymmetricCipher::GenerateIV(IV_SIZE); @@ -153,8 +164,9 @@ namespace ASSERT_NE(metadata.find(MATERIALS_DESCRIPTION_HEADER), metadata.end()); ASSERT_STREQ(metadata[MATERIALS_DESCRIPTION_HEADER].c_str(), handler.SerializeMap(contentCryptoMaterial.GetMaterialsDescription()).c_str()); - GetObjectResult result; + HeadObjectResult result; result.SetMetadata(metadata); + ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(result); ASSERT_EQ(contentCryptoMaterial.GetEncryptedContentEncryptionKey(), readContentCryptoMaterial.GetEncryptedContentEncryptionKey()); ASSERT_EQ(contentCryptoMaterial.GetIV(), readContentCryptoMaterial.GetIV()); @@ -170,9 +182,9 @@ namespace //This tests the read metadata functionality of the handler without a mock client. TEST_F(HandlerTest, ReadMetadataTest) { - GetObjectResult result; + HeadObjectResult result; MetadataHandler handler; - PopulateGetObjectResultMetadata(result); + PopulateHeadObjectResultMetadata(result); ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(result); auto metadata = result.GetMetadata(); @@ -246,15 +258,15 @@ namespace PutObjectOutcome putObjectOutcome = myClient->PutObject(putObjectRequest); - GetObjectRequest getObjectRequest; - getObjectRequest.SetBucket(fullBucketName); - getObjectRequest.SetKey(TEST_OBJ_KEY); + HeadObjectRequest headObjectRequest; + headObjectRequest.SetBucket(fullBucketName); + headObjectRequest.SetKey(TEST_OBJ_KEY); - GetObjectOutcome getObjectOutcome = myClient->GetObject(getObjectRequest); + auto headObjectOutcome = myClient->HeadObject(headObjectRequest); - GetObjectResult& getObjectResult = getObjectOutcome.GetResult(); + auto& headObjectResult = headObjectOutcome.GetResult(); - ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(getObjectResult); + ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(headObjectResult); ASSERT_EQ(contentCryptoMaterial.GetEncryptedContentEncryptionKey(), readContentCryptoMaterial.GetEncryptedContentEncryptionKey()); ASSERT_EQ(contentCryptoMaterial.GetIV(), readContentCryptoMaterial.GetIV()); @@ -266,7 +278,7 @@ namespace ASSERT_EQ(contentCryptoMaterial.GetCEKGCMTag(), readContentCryptoMaterial.GetCEKGCMTag()); ASSERT_EQ(contentCryptoMaterial.GetGCMAAD(), readContentCryptoMaterial.GetGCMAAD()); - ASSERT_EQ(myClient->m_getObjectCalled, 1u); + ASSERT_EQ(myClient->m_headObjectCalled, 1u); ASSERT_EQ(myClient->m_putObjectCalled, 1u); } @@ -274,10 +286,11 @@ namespace TEST_F(HandlerTest, WriteInstructionFileTest) { PutObjectRequest request; + PutObjectRequest objRequest; ContentCryptoMaterial contentCryptoMaterial; PopulateContentCryptoMaterial(contentCryptoMaterial); InstructionFileHandler handler; - handler.PopulateRequest(request, contentCryptoMaterial); + handler.PopulateRequest(objRequest, request, contentCryptoMaterial); auto bodyStream = request.GetBody(); Aws::String jsonString; @@ -303,6 +316,99 @@ namespace ASSERT_STREQ(cryptoContentMap[MATERIALS_DESCRIPTION_HEADER].c_str(), handler.SerializeMap(contentCryptoMaterial.GetMaterialsDescription()).c_str()); } + struct keygen_test + { + std::vector data_key; + std::vector message_id; + std::vector encryption_key; + std::vector commitment_key; + void run_test() + { + CryptoBuffer EncryptionKey(32); + CryptoBuffer CommitmentKey(28); + CryptoBuffer dataKey = CryptoBuffer(data_key.data(), data_key.size()); + CryptoBuffer messageId = CryptoBuffer(message_id.data(), message_id.size()); + Aws::S3Encryption::derive_encryption_key(dataKey, messageId, EncryptionKey); + Aws::S3Encryption::derive_commitment_key(dataKey, messageId, CommitmentKey); + + CryptoBuffer encKey = CryptoBuffer(encryption_key.data(), encryption_key.size()); + CryptoBuffer comKey = CryptoBuffer(commitment_key.data(), commitment_key.size()); + + ASSERT_EQ(CommitmentKey, comKey); + ASSERT_EQ(EncryptionKey, encKey); + } + }; + std::vector hex_decode(const std::string &x) + { + std::vector bytes; + if (x.length() % 2 != 0) + { + return bytes; + } + + for (size_t i = 0; i < x.length(); i += 2) + { + std::string byteString = x.substr(i, 2); + char byte = (char)strtol(byteString.c_str(), NULL, 16); + bytes.push_back(byte); + } + return bytes; + } + keygen_test make_keygen_test(const char *data_key, const char *message_id, const char *encryption_key, const char *commitment_key) + { + keygen_test test; + test.data_key = hex_decode(data_key); + test.message_id = hex_decode(message_id); + test.encryption_key = hex_decode(encryption_key); + test.commitment_key = hex_decode(commitment_key); + return test; + } + + // ** If these test vector pass, then the HKDF key derivation must be properly implemented + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=test + //# - The hash function MUST be specified by the algorithm suite commitment settings. + //# - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + //# - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + //# - The salt MUST be the Message ID with the length defined in the algorithm suite. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=test + //# - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + //# - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=test + //# - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + //# - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. + + //= ../specification/s3-encryption/key-derivation.md#hkdf-operation + //= type=test + //# When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + //# the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + //# The IV's total length MUST match the IV length defined by the algorithm suite. + //# The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + //# and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + //# This means that the IV for ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, which is 12 bytes long, would be `[0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 ]`. + //# The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + + //This tests that the V3 key generation works as expected + TEST_F(HandlerTest, HkdfTest) + { + auto test1 = make_keygen_test( + "80d90dc4cc7e77d8a6332efa44eba56230a7fe7b89af37d1e501ab2e07c0a163", + "b8ea76bed24c7b85382a148cb9dcd1cfdfb765f55ded4dfa6e0c4c79", + "6dd14f546cc006e639126e83f5d4d1b118576bb5df97f38c6fb3a1db87bbc338", + "f89818bc0a346d3a3426b68e9509b6b2ae5fe1f904aa329fb73625db"); + auto test2 = make_keygen_test( + "501afb8227d22e75e68010414b8abdaf3064c081e8e922dafef4992036394d60", + "61a00b4981a5aacfd136c55cb726e32d2a547dc7600a7d4675c69127", + "e14786a714748d1d2c3a4a6816dec56ddf1881bbeabb4f39420ffb9f63700b2f", + "5c1e73e47f6fe3a70d6d094283aceaa76d2975feb829212d88f0afc1"); + test1.run_test(); + test2.run_test(); + + } //This tests the instruction file read/write functionality by using a mock S3 client. TEST_F(HandlerTest, InstructionFileS3OperationsTest) { @@ -310,6 +416,7 @@ namespace Aws::String fullBucketName = TEST_BUCKET_NAME; PutObjectRequest instructionPutObjectRequest; + PutObjectRequest objPutObjectRequest; instructionPutObjectRequest.SetBucket(fullBucketName); instructionPutObjectRequest.SetKey(TEST_OBJ_KEY); @@ -318,7 +425,7 @@ namespace //content crypto material into body of a putObjectRequest InstructionFileHandler handler; - handler.PopulateRequest(instructionPutObjectRequest, contentCryptoMaterial); + handler.PopulateRequest(objPutObjectRequest, instructionPutObjectRequest, contentCryptoMaterial); PutObjectOutcome putObjectOutcome = myClient->PutObject(instructionPutObjectRequest); @@ -329,8 +436,8 @@ namespace GetObjectOutcome getObjectOutcome = myClient->GetObject(getObjectRequest); GetObjectResult& getObjectResult = getObjectOutcome.GetResult(); - - ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(getObjectResult); + HeadObjectResult hResult; + ContentCryptoMaterial readContentCryptoMaterial = handler.ReadContentCryptoMaterial(getObjectResult, hResult); auto metadata = getObjectResult.GetMetadata(); ASSERT_TRUE(metadata.find(INSTRUCTION_FILE_HEADER) != metadata.end());