diff --git a/DEPENDENCIES b/DEPENDENCIES index ff2a459b4..2b295a0a2 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -1,5 +1,5 @@ vendorpull https://github.com/sourcemeta/vendorpull 1dcbac42809cf87cb5b045106b863e17ad84ba02 -core https://github.com/sourcemeta/core fe9ef27a95281775fe3a98b9fb8eb4c51837af0a +core https://github.com/sourcemeta/core cc387d6ab68dc1a467ff83173bde3f7938f582db jsonschema-test-suite https://github.com/json-schema-org/JSON-Schema-Test-Suite 1acd90e53554fa24d2529b49fd7d50bab18f8b7e jsonschema-2020-12 https://github.com/json-schema-org/json-schema-spec 769daad75a9553562333a8937a187741cb708c72 jsonschema-2019-09 https://github.com/json-schema-org/json-schema-spec 41014ea723120ce70b314d72f863c6929d9f3cfd diff --git a/vendor/core/DEPENDENCIES b/vendor/core/DEPENDENCIES index b0bedf0ac..29ef05774 100644 --- a/vendor/core/DEPENDENCIES +++ b/vendor/core/DEPENDENCIES @@ -11,3 +11,4 @@ libdeflate https://github.com/ebiggers/libdeflate v1.25 unicodetools https://github.com/unicode-org/unicodetools final-17.0-20250910 jose-cookbook https://github.com/ietf-jose/cookbook 13692b68bfc18b99557a5b1ed311fd5077bfff04 w3c-json-ld https://github.com/w3c/json-ld-api 8654ac22b6cf4f441d2fee915ae634d36b5a8067 +aws-sig-v4-test-suite https://github.com/saibotsivad/aws-sig-v4-test-suite 38179d1173c9547a964e7f92e9308e73795534ff diff --git a/vendor/core/config.cmake.in b/vendor/core/config.cmake.in index 50a91e0bb..90b13bf34 100644 --- a/vendor/core/config.cmake.in +++ b/vendor/core/config.cmake.in @@ -76,8 +76,10 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_crypto.cmake") elseif(component STREQUAL "regex") find_dependency(PCRE2 CONFIG) + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_regex.cmake") elseif(component STREQUAL "ip") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_ip.cmake") elseif(component STREQUAL "idna") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") @@ -108,6 +110,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) elseif(component STREQUAL "uritemplate") find_dependency(PCRE2 CONFIG) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_regex.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_uritemplate.cmake") elseif(component STREQUAL "json") @@ -115,6 +118,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_numeric.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") elseif(component STREQUAL "jsonl") find_dependency(LibDeflate CONFIG) @@ -137,6 +141,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_jsonpointer.cmake") elseif(component STREQUAL "jsonld") find_dependency(PCRE2 CONFIG) @@ -163,6 +168,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_jsonpointer.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_yaml.cmake") elseif(component STREQUAL "jsonrpc") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_preprocessor.cmake") @@ -170,6 +176,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_jsonrpc.cmake") elseif(component STREQUAL "mcp") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_preprocessor.cmake") @@ -177,6 +184,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_json.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_jsonrpc.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_mcp.cmake") elseif(component STREQUAL "http") diff --git a/vendor/core/src/core/crypto/CMakeLists.txt b/vendor/core/src/core/crypto/CMakeLists.txt index 679bfd55f..16bf78a7d 100644 --- a/vendor/core/src/core/crypto/CMakeLists.txt +++ b/vendor/core/src/core/crypto/CMakeLists.txt @@ -1,8 +1,10 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME crypto - PRIVATE_HEADERS sha256.h sha384.h sha512.h sha1.h fnv128.h uuid.h crc32.h - base64.h verify.h - SOURCES crypto_sha256.cc crypto_sha384.cc crypto_sha512.cc crypto_sha1.cc + PRIVATE_HEADERS sha256.h hmac_sha256.h sha384.h sha512.h sha1.h fnv128.h + uuid.h crc32.h base64.h verify.h + SOURCES crypto_sha256.cc crypto_hmac_sha256.cc crypto_sha384.cc + crypto_sha512.cc crypto_sha1.cc crypto_uuid.cc crypto_fnv128.cc crypto_crc32.cc crypto_base64.cc + crypto_random.cc crypto_sha2_64.h crypto_random.h crypto_helpers.h) target_link_libraries(sourcemeta_core_crypto @@ -10,9 +12,10 @@ target_link_libraries(sourcemeta_core_crypto if(SOURCEMETA_CORE_CRYPTO_USE_SYSTEM_OPENSSL) target_sources(sourcemeta_core_crypto PRIVATE - crypto_sha1_openssl.cc crypto_sha256_openssl.cc crypto_sha384_openssl.cc + crypto_sha1_openssl.cc crypto_sha256_openssl.cc + crypto_hmac_sha256_openssl.cc crypto_sha384_openssl.cc crypto_sha512_openssl.cc crypto_random_openssl.cc - crypto_verify_openssl.cc) + crypto_verify_openssl.cc crypto_secure_equals_openssl.cc) target_link_libraries(sourcemeta_core_crypto PRIVATE OpenSSL::Crypto) elseif(APPLE) enable_language(OBJCXX) @@ -96,9 +99,10 @@ elseif(APPLE) PROPERTIES OBJECT_DEPENDS "${CRYPTOKIT_HEADER}") target_sources(sourcemeta_core_crypto PRIVATE - crypto_sha1_apple.cc crypto_sha256_apple.cc crypto_sha384_apple.cc + crypto_sha1_apple.cc crypto_sha256_apple.cc crypto_hmac_sha256_apple.cc + crypto_sha384_apple.cc crypto_sha512_apple.cc crypto_random_apple.cc - crypto_verify_apple.cc + crypto_verify_apple.cc crypto_secure_equals_apple.cc crypto_eddsa_cryptokit.mm "${CRYPTOKIT_OBJECT}" crypto_eddsa.h crypto_eddsa_apple.h crypto_bignum.h crypto_shake256.h) @@ -121,17 +125,20 @@ elseif(APPLE) "SHELL:-Xlinker -rpath -Xlinker /usr/lib/swift") elseif(WIN32) target_sources(sourcemeta_core_crypto PRIVATE - crypto_sha1_windows.cc crypto_sha256_windows.cc crypto_sha384_windows.cc + crypto_sha1_windows.cc crypto_sha256_windows.cc + crypto_hmac_sha256_windows.cc crypto_sha384_windows.cc crypto_sha512_windows.cc crypto_random_windows.cc - crypto_verify_windows.cc crypto_eddsa.h + crypto_verify_windows.cc crypto_secure_equals_generic.cc crypto_eddsa.h crypto_bignum.h crypto_shake256.h) target_link_libraries(sourcemeta_core_crypto PRIVATE bcrypt) else() message(WARNING "Building the reference cryptography backend, instead of the " "OpenSSL recommended production one") target_sources(sourcemeta_core_crypto PRIVATE - crypto_sha1_other.cc crypto_sha256_other.cc crypto_sha384_other.cc + crypto_sha1_other.cc crypto_sha256_other.cc crypto_hmac_sha256_other.cc + crypto_sha384_other.cc crypto_sha512_other.cc crypto_random_other.cc crypto_verify_other.cc + crypto_secure_equals_generic.cc crypto_bignum.h crypto_ecc.h crypto_eddsa.h crypto_shake256.h) endif() diff --git a/vendor/core/src/core/crypto/crypto_bignum.h b/vendor/core/src/core/crypto/crypto_bignum.h index 5244bc253..31afc9def 100644 --- a/vendor/core/src/core/crypto/crypto_bignum.h +++ b/vendor/core/src/core/crypto/crypto_bignum.h @@ -7,10 +7,12 @@ // required, since verification consumes only public inputs #include +#include #include // std::array #include // std::size_t #include // std::uint8_t, std::uint64_t +#include // std::optional #include // std::string #include // std::string_view @@ -62,33 +64,13 @@ inline auto bignum_from_u64(const std::uint64_t value) noexcept -> Bignum { } inline auto bignum_from_hex(const std::string_view hex) -> Bignum { - const auto nibble{[](const char character) noexcept -> std::uint8_t { - if (character >= '0' && character <= '9') { - return static_cast(character - '0'); - } else if (character >= 'a' && character <= 'f') { - return static_cast(character - 'a' + 10); - } else { - return static_cast(character - 'A' + 10); - } - }}; - - std::string bytes; - bytes.reserve((hex.size() + 1) / 2); - - // An odd length means the leading nibble forms a byte on its own, as if a - // zero had been prepended - std::size_t index{0}; - if (hex.size() % 2 != 0) { - bytes.push_back(static_cast(nibble(hex[0]))); - index = 1; - } - - for (; index + 1 < hex.size(); index += 2) { - bytes.push_back( - static_cast((nibble(hex[index]) << 4u) | nibble(hex[index + 1]))); + // An odd length is decoded as if a zero nibble had been prepended + const auto bytes{hex_to_bytes(hex, true)}; + if (!bytes.has_value()) { + return Bignum{}; } - return bignum_from_bytes(bytes); + return bignum_from_bytes(bytes.value()); } inline auto bignum_is_zero(const Bignum &value) noexcept -> bool { diff --git a/vendor/core/src/core/crypto/crypto_hmac_sha256.cc b/vendor/core/src/core/crypto/crypto_hmac_sha256.cc new file mode 100644 index 000000000..1ddf2f290 --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_hmac_sha256.cc @@ -0,0 +1,23 @@ +#include +#include + +#include // std::ostream, std::streamsize +#include // std::string +#include // std::string_view + +namespace sourcemeta::core { + +auto hmac_sha256(const std::string_view key, const std::string_view message) + -> std::string { + const auto digest{hmac_sha256_digest(key, message)}; + return bytes_to_hex( + {reinterpret_cast(digest.data()), digest.size()}); +} + +auto hmac_sha256(const std::string_view key, const std::string_view message, + std::ostream &output) -> void { + const auto result{hmac_sha256(key, message)}; + output.write(result.data(), static_cast(result.size())); +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_hmac_sha256_apple.cc b/vendor/core/src/core/crypto/crypto_hmac_sha256_apple.cc new file mode 100644 index 000000000..4c8e3a3b2 --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_hmac_sha256_apple.cc @@ -0,0 +1,19 @@ +#include + +#include // CCHmac, kCCHmacAlgSHA256 + +#include // std::array +#include // std::uint8_t + +namespace sourcemeta::core { + +auto hmac_sha256_digest(const std::string_view key, + const std::string_view message) + -> std::array { + std::array digest{}; + CCHmac(kCCHmacAlgSHA256, key.data(), key.size(), message.data(), + message.size(), digest.data()); + return digest; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_hmac_sha256_openssl.cc b/vendor/core/src/core/crypto/crypto_hmac_sha256_openssl.cc new file mode 100644 index 000000000..6e1deb16a --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_hmac_sha256_openssl.cc @@ -0,0 +1,41 @@ +#include +#include + +#include // EVP_sha256 +#include // HMAC + +#include // std::array +#include // std::size_t +#include // std::uint8_t +#include // std::runtime_error + +namespace sourcemeta::core { + +auto hmac_sha256_digest(const std::string_view key, + const std::string_view message) + -> std::array { + // A key longer than the block size is hashed first (RFC 2104 Section 2), + // which also keeps the key length within the OpenSSL length parameter + constexpr std::size_t block_size{64}; + std::array key_digest{}; + const unsigned char *key_data{ + reinterpret_cast(key.data())}; + auto key_size{key.size()}; + if (key_size > block_size) { + key_digest = sha256_digest(key); + key_data = key_digest.data(); + key_size = key_digest.size(); + } + + std::array digest{}; + unsigned int length{0}; + if (HMAC(EVP_sha256(), key_data, static_cast(key_size), + reinterpret_cast(message.data()), + message.size(), digest.data(), &length) == nullptr) { + throw std::runtime_error("Could not compute HMAC-SHA256 digest"); + } + + return digest; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_hmac_sha256_other.cc b/vendor/core/src/core/crypto/crypto_hmac_sha256_other.cc new file mode 100644 index 000000000..93432d42b --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_hmac_sha256_other.cc @@ -0,0 +1,45 @@ +#include +#include + +#include // std::array +#include // std::size_t +#include // std::uint8_t +#include // std::memcpy +#include // std::string + +namespace sourcemeta::core { + +auto hmac_sha256_digest(const std::string_view key, + const std::string_view message) + -> std::array { + // SHA-256 operates on 64-byte blocks (FIPS 180-4 Section 5.1.1), which is the + // block size the HMAC construction pads the key to (RFC 2104 Section 2) + constexpr std::size_t block_size{64}; + + // "If the key is longer than the block size, hash it and use the result" + // (RFC 2104 Section 2) + std::array padded_key{}; + if (key.size() > block_size) { + const auto digest{sha256_digest(key)}; + std::memcpy(padded_key.data(), digest.data(), digest.size()); + } else if (!key.empty()) { + std::memcpy(padded_key.data(), key.data(), key.size()); + } + + // H((K XOR opad) || H((K XOR ipad) || message)) with ipad = 0x36 and + // opad = 0x5c repeated to the block size (RFC 2104 Section 2) + std::string inner_input(block_size, '\x00'); + std::string outer_input(block_size, '\x00'); + for (std::size_t index = 0; index < block_size; ++index) { + inner_input[index] = static_cast(padded_key[index] ^ 0x36); + outer_input[index] = static_cast(padded_key[index] ^ 0x5c); + } + + inner_input.append(message); + const auto inner_digest{sha256_digest(inner_input)}; + outer_input.append(reinterpret_cast(inner_digest.data()), + inner_digest.size()); + return sha256_digest(outer_input); +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_hmac_sha256_windows.cc b/vendor/core/src/core/crypto/crypto_hmac_sha256_windows.cc new file mode 100644 index 000000000..beea02c83 --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_hmac_sha256_windows.cc @@ -0,0 +1,82 @@ +#include +#include + +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include // ULONG + +#include // BCrypt*, BCRYPT_* + +#include // std::array +#include // std::size_t +#include // std::uint8_t +#include // std::numeric_limits +#include // std::runtime_error + +namespace sourcemeta::core { + +auto hmac_sha256_digest(const std::string_view key, + const std::string_view message) + -> std::array { + BCRYPT_ALG_HANDLE algorithm{nullptr}; + if (!BCRYPT_SUCCESS( + BCryptOpenAlgorithmProvider(&algorithm, BCRYPT_SHA256_ALGORITHM, + nullptr, BCRYPT_ALG_HANDLE_HMAC_FLAG))) { + throw std::runtime_error("Could not open the CNG HMAC-SHA256 provider"); + } + + // A key longer than the block size is hashed first (RFC 2104 Section 2), + // which also keeps the key length within the CNG length parameter. The + // secret interface is not const-qualified but never writes through the + // pointer + constexpr std::size_t block_size{64}; + std::array key_digest{}; + auto *secret{ + reinterpret_cast(const_cast(key.data()))}; + auto secret_size{key.size()}; + if (secret_size > block_size) { + key_digest = sha256_digest(key); + secret = key_digest.data(); + secret_size = key_digest.size(); + } + + BCRYPT_HASH_HANDLE hash{nullptr}; + if (!BCRYPT_SUCCESS(BCryptCreateHash(algorithm, &hash, nullptr, 0, secret, + static_cast(secret_size), 0))) { + BCryptCloseAlgorithmProvider(algorithm, 0); + throw std::runtime_error("Could not create the CNG HMAC-SHA256 hash"); + } + + // The data interface is not const-qualified but never writes through + // the pointer, and it takes a 32-bit length, so larger inputs must be + // fed in chunks + auto *remaining_data{ + reinterpret_cast(const_cast(message.data()))}; + auto remaining_size{message.size()}; + constexpr std::size_t maximum_chunk{std::numeric_limits::max()}; + auto success{true}; + while (remaining_size > 0 && success) { + const auto chunk_size{remaining_size > maximum_chunk ? maximum_chunk + : remaining_size}; + success = BCRYPT_SUCCESS(BCryptHashData(hash, remaining_data, + static_cast(chunk_size), 0)); + remaining_data += chunk_size; + remaining_size -= chunk_size; + } + + std::array digest{}; + if (success) { + success = BCRYPT_SUCCESS(BCryptFinishHash( + hash, digest.data(), static_cast(digest.size()), 0)); + } + + BCryptDestroyHash(hash); + BCryptCloseAlgorithmProvider(algorithm, 0); + if (!success) { + throw std::runtime_error("Could not compute the CNG HMAC-SHA256 digest"); + } + + return digest; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_random.cc b/vendor/core/src/core/crypto/crypto_random.cc new file mode 100644 index 000000000..080d9a989 --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_random.cc @@ -0,0 +1,33 @@ +#include + +#include "crypto_random.h" + +#include // std::size_t +#include // std::uint8_t +#include // std::span +#include // std::string +#include // std::is_same_v + +namespace sourcemeta::core { + +// Writing random bytes into the string's storage reinterprets its char buffer +// as bytes, which is well-defined only because std::uint8_t aliases unsigned +// char, the type permitted to alias any object representation +static_assert(std::is_same_v); + +auto random_bytes(std::span buffer) -> void { + if (buffer.empty()) { + return; + } + + fill_random_bytes(buffer); +} + +auto random_bytes(const std::size_t length) -> std::string { + std::string result(length, '\0'); + random_bytes(std::span{ + reinterpret_cast(result.data()), length}); + return result; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_random.h b/vendor/core/src/core/crypto/crypto_random.h index a6a220c65..b370bdb20 100644 --- a/vendor/core/src/core/crypto/crypto_random.h +++ b/vendor/core/src/core/crypto/crypto_random.h @@ -1,13 +1,14 @@ #ifndef SOURCEMETA_CORE_CRYPTO_RANDOM_H_ #define SOURCEMETA_CORE_CRYPTO_RANDOM_H_ -#include // std::array +#include // std::uint8_t +#include // std::span namespace sourcemeta::core { // Fill the given buffer with random bytes from the system provider where // available. Defined once per backend -auto fill_random_bytes(std::array &bytes) -> void; +auto fill_random_bytes(std::span bytes) -> void; } // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_random_apple.cc b/vendor/core/src/core/crypto/crypto_random_apple.cc index aafa7f6ae..972c18ccd 100644 --- a/vendor/core/src/core/crypto/crypto_random_apple.cc +++ b/vendor/core/src/core/crypto/crypto_random_apple.cc @@ -3,12 +3,13 @@ #include // errSecSuccess #include // SecRandomCopyBytes, kSecRandomDefault -#include // std::array +#include // std::uint8_t +#include // std::span #include // std::runtime_error namespace sourcemeta::core { -auto fill_random_bytes(std::array &bytes) -> void { +auto fill_random_bytes(std::span bytes) -> void { if (SecRandomCopyBytes(kSecRandomDefault, bytes.size(), bytes.data()) != errSecSuccess) { throw std::runtime_error( diff --git a/vendor/core/src/core/crypto/crypto_random_openssl.cc b/vendor/core/src/core/crypto/crypto_random_openssl.cc index ac4c63257..0a7bd9ce4 100644 --- a/vendor/core/src/core/crypto/crypto_random_openssl.cc +++ b/vendor/core/src/core/crypto/crypto_random_openssl.cc @@ -2,14 +2,26 @@ #include // RAND_bytes -#include // std::array +#include // std::min +#include // std::size_t +#include // std::uint8_t +#include // std::numeric_limits +#include // std::span #include // std::runtime_error namespace sourcemeta::core { -auto fill_random_bytes(std::array &bytes) -> void { - if (RAND_bytes(bytes.data(), static_cast(bytes.size())) != 1) { - throw std::runtime_error("Could not generate random bytes with OpenSSL"); +auto fill_random_bytes(std::span bytes) -> void { + // RAND_bytes takes a signed int length, so fill in chunks to avoid narrowing + // a larger span into a truncated or negative length + while (!bytes.empty()) { + const auto chunk{static_cast( + std::min(bytes.size(), std::numeric_limits::max()))}; + if (RAND_bytes(bytes.data(), chunk) != 1) { + throw std::runtime_error("Could not generate random bytes with OpenSSL"); + } + + bytes = bytes.subspan(static_cast(chunk)); } } diff --git a/vendor/core/src/core/crypto/crypto_random_other.cc b/vendor/core/src/core/crypto/crypto_random_other.cc index 72e2a83c0..8802534b8 100644 --- a/vendor/core/src/core/crypto/crypto_random_other.cc +++ b/vendor/core/src/core/crypto/crypto_random_other.cc @@ -1,18 +1,19 @@ #include "crypto_random.h" -#include // std::array +#include // std::uint8_t #include // std::random_device, std::mt19937, std::uniform_int_distribution +#include // std::span namespace sourcemeta::core { -auto fill_random_bytes(std::array &bytes) -> void { +auto fill_random_bytes(std::span bytes) -> void { // Not a cryptographically secure generator. This fallback only exists to // keep the module buildable on platforms without a system provider thread_local std::random_device device; thread_local std::mt19937 generator{device()}; std::uniform_int_distribution distribution{0, 255}; for (auto &byte : bytes) { - byte = static_cast(distribution(generator)); + byte = static_cast(distribution(generator)); } } diff --git a/vendor/core/src/core/crypto/crypto_random_windows.cc b/vendor/core/src/core/crypto/crypto_random_windows.cc index 695da9908..59f90d9e8 100644 --- a/vendor/core/src/core/crypto/crypto_random_windows.cc +++ b/vendor/core/src/core/crypto/crypto_random_windows.cc @@ -6,16 +6,27 @@ #include // BCrypt*, BCRYPT_* -#include // std::array +#include // std::min +#include // std::size_t +#include // std::uint8_t +#include // std::numeric_limits +#include // std::span #include // std::runtime_error namespace sourcemeta::core { -auto fill_random_bytes(std::array &bytes) -> void { - if (!BCRYPT_SUCCESS(BCryptGenRandom(nullptr, bytes.data(), - static_cast(bytes.size()), - BCRYPT_USE_SYSTEM_PREFERRED_RNG))) { - throw std::runtime_error("Could not generate random bytes with CNG"); +auto fill_random_bytes(std::span bytes) -> void { + // BCryptGenRandom takes a ULONG length, so fill in chunks to avoid narrowing + // a larger span into a wrapped length that would leave a suffix unfilled + while (!bytes.empty()) { + const auto chunk{static_cast(std::min( + bytes.size(), std::numeric_limits::max()))}; + if (!BCRYPT_SUCCESS(BCryptGenRandom(nullptr, bytes.data(), chunk, + BCRYPT_USE_SYSTEM_PREFERRED_RNG))) { + throw std::runtime_error("Could not generate random bytes with CNG"); + } + + bytes = bytes.subspan(chunk); } } diff --git a/vendor/core/src/core/crypto/crypto_secure_equals_apple.cc b/vendor/core/src/core/crypto/crypto_secure_equals_apple.cc new file mode 100644 index 000000000..8a25cf1fb --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_secure_equals_apple.cc @@ -0,0 +1,24 @@ +#include + +#include // timingsafe_bcmp + +#include // std::string_view + +namespace sourcemeta::core { + +auto secure_equals(const std::string_view left, + const std::string_view right) noexcept -> bool { + if (left.size() != right.size()) { + return false; + } + + // An empty view may hold a null data pointer, which is undefined to pass to + // the C API even with a zero length + if (left.empty()) { + return true; + } + + return timingsafe_bcmp(left.data(), right.data(), left.size()) == 0; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_secure_equals_generic.cc b/vendor/core/src/core/crypto/crypto_secure_equals_generic.cc new file mode 100644 index 000000000..9977bc404 --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_secure_equals_generic.cc @@ -0,0 +1,28 @@ +#include + +#include // std::size_t +#include // std::string_view + +namespace sourcemeta::core { + +auto secure_equals(const std::string_view left, + const std::string_view right) noexcept -> bool { + if (left.size() != right.size()) { + return false; + } + + // Accumulate the difference across every byte rather than returning on the + // first mismatch, so that the running time depends only on the length. The + // accumulator is volatile so that the compiler cannot reintroduce a + // short-circuit that would leak the position of the first difference + volatile unsigned char difference{0}; + for (std::size_t index = 0; index < left.size(); ++index) { + difference = static_cast( + difference | (static_cast(left[index]) ^ + static_cast(right[index]))); + } + + return difference == 0; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_secure_equals_openssl.cc b/vendor/core/src/core/crypto/crypto_secure_equals_openssl.cc new file mode 100644 index 000000000..ec700cb3e --- /dev/null +++ b/vendor/core/src/core/crypto/crypto_secure_equals_openssl.cc @@ -0,0 +1,24 @@ +#include + +#include // CRYPTO_memcmp + +#include // std::string_view + +namespace sourcemeta::core { + +auto secure_equals(const std::string_view left, + const std::string_view right) noexcept -> bool { + if (left.size() != right.size()) { + return false; + } + + // An empty view may hold a null data pointer, which is undefined to pass to + // the C API even with a zero length + if (left.empty()) { + return true; + } + + return CRYPTO_memcmp(left.data(), right.data(), left.size()) == 0; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/crypto/crypto_uuid.cc b/vendor/core/src/core/crypto/crypto_uuid.cc index c0aac888a..adeda4c84 100644 --- a/vendor/core/src/core/crypto/crypto_uuid.cc +++ b/vendor/core/src/core/crypto/crypto_uuid.cc @@ -1,10 +1,11 @@ #include -#include // is_hex_digit +#include #include "crypto_random.h" #include // std::array #include // std::size_t +#include // std::uint8_t #include // std::string #include // std::string_view @@ -20,8 +21,8 @@ auto uuidv4() -> std::string { {false, false, false, false, true, false, true, false, true, false, true, false, false, false, false, false}}; - std::array random_bytes{}; - fill_random_bytes(random_bytes); + std::array bytes{}; + fill_random_bytes(bytes); std::string result; result.reserve(36); @@ -30,8 +31,8 @@ auto uuidv4() -> std::string { result += '-'; } - const auto high_nibble = (random_bytes[index] >> 4u) & 0x0fu; - const auto low_nibble = random_bytes[index] & 0x0fu; + const auto high_nibble = (bytes[index] >> 4u) & 0x0fu; + const auto low_nibble = bytes[index] & 0x0fu; // RFC 9562 Section 5.4: version bits (48-51) must be 0b0100 if (index == 6) { diff --git a/vendor/core/src/core/crypto/include/sourcemeta/core/crypto.h b/vendor/core/src/core/crypto/include/sourcemeta/core/crypto.h index bab7ef2e4..1ebac2f09 100644 --- a/vendor/core/src/core/crypto/include/sourcemeta/core/crypto.h +++ b/vendor/core/src/core/crypto/include/sourcemeta/core/crypto.h @@ -1,6 +1,10 @@ #ifndef SOURCEMETA_CORE_CRYPTO_H_ #define SOURCEMETA_CORE_CRYPTO_H_ +#ifndef SOURCEMETA_CORE_CRYPTO_EXPORT +#include +#endif + /// @defgroup crypto Crypto /// @brief Cryptographic hash functions, UUID generation, and Base64 codecs. /// @@ -13,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -20,4 +25,65 @@ #include #include +#include // std::size_t +#include // std::uint8_t +#include // std::span +#include // std::string +#include // std::string_view + +namespace sourcemeta::core { + +/// @ingroup crypto +/// Fill a buffer with random bytes drawn from the operating system's +/// cryptographically secure provider. The bytes are only cryptographically +/// secure when the library is built against a system provider (OpenSSL, the +/// Apple Security framework, or Windows CNG). The reference backend used when +/// no system provider is available falls back to a non-cryptographic generator. +/// For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// std::array buffer{}; +/// sourcemeta::core::random_bytes(buffer); +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT random_bytes(std::span buffer) + -> void; + +/// @ingroup crypto +/// Return the given number of random bytes as a string, drawn from the +/// operating system's cryptographically secure provider. The same backend +/// caveat as the buffer-filling overload applies. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto bytes{sourcemeta::core::random_bytes(32)}; +/// assert(bytes.size() == 32); +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT random_bytes(const std::size_t length) + -> std::string; + +/// @ingroup crypto +/// Compare two byte sequences for equality in constant time when their lengths +/// match, returning false immediately when the lengths differ. The comparison +/// does not short-circuit on the first differing byte, so it does not leak the +/// position of a mismatch through timing. The length of the inputs is not +/// treated as secret. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// assert(sourcemeta::core::secure_equals("expected", "expected")); +/// assert(!sourcemeta::core::secure_equals("expected", "actual")); +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT secure_equals( + const std::string_view left, const std::string_view right) noexcept -> bool; + +} // namespace sourcemeta::core + #endif diff --git a/vendor/core/src/core/crypto/include/sourcemeta/core/crypto_hmac_sha256.h b/vendor/core/src/core/crypto/include/sourcemeta/core/crypto_hmac_sha256.h new file mode 100644 index 000000000..21d72fb28 --- /dev/null +++ b/vendor/core/src/core/crypto/include/sourcemeta/core/crypto_hmac_sha256.h @@ -0,0 +1,64 @@ +#ifndef SOURCEMETA_CORE_CRYPTO_HMAC_SHA256_H_ +#define SOURCEMETA_CORE_CRYPTO_HMAC_SHA256_H_ + +#ifndef SOURCEMETA_CORE_CRYPTO_EXPORT +#include +#endif + +#include // std::array +#include // std::uint8_t +#include // std::ostream +#include // std::string +#include // std::string_view + +namespace sourcemeta::core { + +/// @ingroup crypto +/// Authenticate a message under a key using HMAC-SHA256, writing the hex +/// digest. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// std::ostringstream result; +/// sourcemeta::core::hmac_sha256("key", "foo bar", result); +/// std::cout << result.str() << "\n"; +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT hmac_sha256(const std::string_view key, + const std::string_view message, + std::ostream &output) -> void; + +/// @ingroup crypto +/// Authenticate a message under a key using HMAC-SHA256, returning the hex +/// digest as a string. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// std::cout << sourcemeta::core::hmac_sha256("key", "foo bar") << "\n"; +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT hmac_sha256(const std::string_view key, + const std::string_view message) + -> std::string; + +/// @ingroup crypto +/// Authenticate a message under a key using HMAC-SHA256, returning the raw +/// digest bytes. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto digest{sourcemeta::core::hmac_sha256_digest("key", "foo bar")}; +/// assert(digest.size() == 32); +/// ``` +auto SOURCEMETA_CORE_CRYPTO_EXPORT +hmac_sha256_digest(const std::string_view key, const std::string_view message) + -> std::array; + +} // namespace sourcemeta::core + +#endif diff --git a/vendor/core/src/core/dns/idn_hostname.cc b/vendor/core/src/core/dns/idn_hostname.cc index b1d102d6d..97f43464f 100644 --- a/vendor/core/src/core/dns/idn_hostname.cc +++ b/vendor/core/src/core/dns/idn_hostname.cc @@ -5,6 +5,7 @@ #include #include // std::size_t +#include // std::optional #include // std::string, std::u32string #include // std::string_view, std::u32string_view #include // std::vector @@ -19,25 +20,19 @@ static constexpr auto is_idna_label_separator(const char32_t codepoint) codepoint == U'\uFF0E' || codepoint == U'\uFF61'; } -auto is_idn_hostname(const std::string_view value) -> bool { - // No conformance pass against `vendor/unicodetools/IdnaTestV2.txt` is - // wired here. That corpus tests UTS #46, which prepends a mapping pass - // we deliberately do not implement, so strict IDNA 2008 validation - // rejects most of its successful rows. Enabling it would require - // shipping a UTS #46 mapping module first. - - if (value.empty()) { - return false; - } +namespace { - const auto codepoints{utf8_to_utf32(value)}; - if (!codepoints.has_value() || codepoints->empty()) { +// Validate an already-decoded presentation-form domain name against the +// IDNA 2008 rules (RFC 5890-5893). The caller is responsible for any prior +// transformation, such as the UTS #46 mapping. +auto validate_idn_labels(const std::u32string_view codepoints) -> bool { + if (codepoints.empty()) { return false; } // UTS #46 §3.1: a leading or trailing separator means an empty label - if (is_idna_label_separator(codepoints->front()) || - is_idna_label_separator(codepoints->back())) { + if (is_idna_label_separator(codepoints.front()) || + is_idna_label_separator(codepoints.back())) { return false; } @@ -45,12 +40,12 @@ auto is_idn_hostname(const std::string_view value) -> bool { // RFC 1035 §3.1: presentation form ≤ 253 octets in A-label form std::size_t total_octets{0}; std::size_t label_start{0}; - for (std::size_t position = 0; position <= codepoints->size(); ++position) { - const bool at_end{position == codepoints->size()}; - if (!at_end && !is_idna_label_separator((*codepoints)[position])) { + for (std::size_t position = 0; position <= codepoints.size(); ++position) { + const bool at_end{position == codepoints.size()}; + if (!at_end && !is_idna_label_separator(codepoints[position])) { continue; } - const std::u32string_view label{codepoints->data() + label_start, + const std::u32string_view label{codepoints.data() + label_start, position - label_start}; std::u32string decoded; @@ -72,6 +67,13 @@ auto is_idn_hostname(const std::string_view value) -> bool { return false; } } else if (*kind == IDNALabelKind::Ascii) { + // RFC 5891 §4.2.3.1: a label must not contain "--" in the third and + // fourth positions unless it is a valid A-label (an "xn--" ACE label, + // which is classified as ALabel rather than Ascii). This matches the + // U-label path, which already rejects such labels. + if (label.size() >= 4 && label[2] == U'-' && label[3] == U'-') { + return false; + } // RFC 1123 §2.1: ASCII labels still need the LDH grammar check std::string ascii; ascii.reserve(label.size()); @@ -131,4 +133,32 @@ auto is_idn_hostname(const std::string_view value) -> bool { return true; } +} // namespace + +auto is_idn_hostname(const std::string_view value) -> bool { + // This is strict IDNA 2008 validation: the input is validated as-is, with + // no mapping pass. Callers that need the UTS #46 lookup behaviour (case + // folding, deviation handling, ignorable removal) should use + // `is_idn_hostname_uts46`, which is what the `vendor/unicodetools` + // IdnaTestV2.txt corpus exercises. + const auto codepoints{utf8_to_utf32(value)}; + if (!codepoints.has_value()) { + return false; + } + + return validate_idn_labels(*codepoints); +} + +auto is_idn_hostname_uts46(const std::string_view value) -> bool { + const auto codepoints{utf8_to_utf32(value)}; + if (!codepoints.has_value()) { + return false; + } + + // UTS #46 §4 steps 1-2: map and normalise before validating. Disallowed + // codepoints are preserved by the mapping and rejected here by the per-label + // validity check. + return validate_idn_labels(idna_uts46_map(*codepoints)); +} + } // namespace sourcemeta::core diff --git a/vendor/core/src/core/dns/include/sourcemeta/core/dns.h b/vendor/core/src/core/dns/include/sourcemeta/core/dns.h index 810f59b27..ba3b1985f 100644 --- a/vendor/core/src/core/dns/include/sourcemeta/core/dns.h +++ b/vendor/core/src/core/dns/include/sourcemeta/core/dns.h @@ -62,6 +62,28 @@ auto is_hostname(const std::string_view value) -> bool; SOURCEMETA_CORE_DNS_EXPORT auto is_idn_hostname(const std::string_view value) -> bool; +/// @ingroup dns +/// Check whether the given string is a valid internationalized host name after +/// UTS #46 processing. The input is first mapped and normalised to NFC under +/// UTS #46 Nontransitional Processing, then the mapped result is validated with +/// the same IDNA 2008 per-label rules as a strict host name. This is the +/// lookup-side behaviour: human-typed forms such as uppercase ASCII and +/// fullwidth characters are folded before validation rather than rejected. See +/// https://www.unicode.org/reports/tr46/ for the algorithm. For example: +/// +/// ```cpp +/// #include +/// +/// #include +/// +/// assert(sourcemeta::core::is_idn_hostname_uts46("www.example.com")); +/// // Uppercase and fullwidth input is mapped before validation +/// assert(sourcemeta::core::is_idn_hostname_uts46("WWW.EXAMPLE.COM")); +/// assert(!sourcemeta::core::is_idn_hostname_uts46("-bad")); +/// ``` +SOURCEMETA_CORE_DNS_EXPORT +auto is_idn_hostname_uts46(const std::string_view value) -> bool; + } // namespace sourcemeta::core #endif diff --git a/vendor/core/src/core/email/email.cc b/vendor/core/src/core/email/email.cc index 51a578ff3..9ae4bbdcb 100644 --- a/vendor/core/src/core/email/email.cc +++ b/vendor/core/src/core/email/email.cc @@ -8,8 +8,10 @@ namespace sourcemeta::core { // RFC 5321 §4.1.2 Mailbox grammar. When AllowUtf8 is true, RFC 6531 §3.3 -// extends atext, qtextSMTP, and sub-domain with UTF8-non-ascii alternatives -template +// extends atext, qtextSMTP, and sub-domain with UTF8-non-ascii alternatives. +// When UseUts46 is also true, the domain is validated under UTS #46 processing +// rather than strict IDNA 2008. +template static auto is_mailbox(const std::string_view value) -> bool { if (value.empty()) { return false; @@ -116,7 +118,11 @@ static auto is_mailbox(const std::string_view value) -> bool { if constexpr (AllowUtf8) { // RFC 6531 §3.3: sub-domain =/ U-label - return is_idn_hostname(domain); + if constexpr (UseUts46) { + return is_idn_hostname_uts46(domain); + } else { + return is_idn_hostname(domain); + } } else { // RFC 5321 §4.1.2 Domain matches is_hostname (RFC 1123 §2.1) by // grammar, by 63-octet label cap (RFC 1035 §2.3.4), and by @@ -133,4 +139,8 @@ auto is_idn_email(const std::string_view value) -> bool { return is_mailbox(value); } +auto is_idn_email_uts46(const std::string_view value) -> bool { + return is_mailbox(value); +} + } // namespace sourcemeta::core diff --git a/vendor/core/src/core/email/include/sourcemeta/core/email.h b/vendor/core/src/core/email/include/sourcemeta/core/email.h index 947c1b939..907d98191 100644 --- a/vendor/core/src/core/email/include/sourcemeta/core/email.h +++ b/vendor/core/src/core/email/include/sourcemeta/core/email.h @@ -56,6 +56,32 @@ auto is_email(const std::string_view value) -> bool; SOURCEMETA_CORE_EMAIL_EXPORT auto is_idn_email(const std::string_view value) -> bool; +/// @ingroup email +/// Check whether the given string is a valid internationalized `Mailbox` per +/// RFC 6531 Section 3.3, validating the domain under UTS #46 processing rather +/// than strict IDNA 2008. The domain is mapped (case folding, compatibility +/// mappings such as fullwidth to ASCII, and removal of ignorable characters) +/// and NFC-normalised before validation, so forms that strict validation +/// rejects, such as fullwidth characters and non-normalised (non-NFC) labels, +/// are accepted. The local part carries no normalisation requirement +/// (RFC 6531) and is validated as-is. See https://www.unicode.org/reports/tr46/ +/// for the algorithm. For example: +/// +/// ```cpp +/// #include +/// +/// #include +/// +/// assert(sourcemeta::core::is_idn_email_uts46("joe.bloggs@example.com")); +/// // The fullwidth domain U+FF41 U+FF42 U+FF43 maps to "abc" and is accepted, +/// // whereas strict IDNA 2008 validation rejects it +/// assert(sourcemeta::core::is_idn_email_uts46( +/// "user@\xef\xbd\x81\xef\xbd\x82\xef\xbd\x83")); +/// assert(!sourcemeta::core::is_idn_email_uts46("2962")); +/// ``` +SOURCEMETA_CORE_EMAIL_EXPORT +auto is_idn_email_uts46(const std::string_view value) -> bool; + } // namespace sourcemeta::core #endif diff --git a/vendor/core/src/core/http/CMakeLists.txt b/vendor/core/src/core/http/CMakeLists.txt index 588c6c9e9..ad184a816 100644 --- a/vendor/core/src/core/http/CMakeLists.txt +++ b/vendor/core/src/core/http/CMakeLists.txt @@ -10,10 +10,11 @@ endif() sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME http PRIVATE_HEADERS problem.h status.h method.h message.h error.h system.h + aws_sigv4.h SOURCES helpers.h problem.cc match_accept.cc match_accept_language.cc negotiate_encoding.cc from_date.cc format_link.cc field_list.cc accept_includes_all.cc content_type_matches.cc parse_bearer.cc - cache_control_max_age.cc + cache_control_max_age.cc serialize_cookie.cc aws_sigv4.cc ${SOURCEMETA_CORE_HTTP_CLIENT_SOURCE}) if(SOURCEMETA_CORE_INSTALL) @@ -23,6 +24,9 @@ endif() target_link_libraries(sourcemeta_core_http PUBLIC sourcemeta::core::json) target_link_libraries(sourcemeta_core_http PUBLIC sourcemeta::core::text) target_link_libraries(sourcemeta_core_http PRIVATE sourcemeta::core::time) +target_link_libraries(sourcemeta_core_http PRIVATE sourcemeta::core::dns) +target_link_libraries(sourcemeta_core_http PRIVATE sourcemeta::core::crypto) +target_link_libraries(sourcemeta_core_http PRIVATE sourcemeta::core::uri) if(SOURCEMETA_CORE_HTTP_USE_SYSTEM_CURL) find_package(CURL REQUIRED) diff --git a/vendor/core/src/core/http/aws_sigv4.cc b/vendor/core/src/core/http/aws_sigv4.cc new file mode 100644 index 000000000..83b17850b --- /dev/null +++ b/vendor/core/src/core/http/aws_sigv4.cc @@ -0,0 +1,250 @@ +#include +#include +#include +#include + +#include "helpers.h" + +#include // std::sort, std::stable_sort +#include // std::array +#include // std::size_t +#include // std::uint8_t +#include // std::span +#include // std::string +#include // std::string_view +#include // std::pair, std::move +#include // std::vector + +namespace { + +auto digest_view(const std::array &bytes) + -> std::string_view { + return {reinterpret_cast(bytes.data()), bytes.size()}; +} + +auto sorted_headers( + const std::span> + headers) -> std::vector> { + std::vector> entries; + entries.reserve(headers.size()); + for (const auto &[name, value] : headers) { + entries.emplace_back(name, + sourcemeta::core::http_trim_trailing_ows( + sourcemeta::core::http_trim_leading_ows(value))); + } + + std::ranges::stable_sort( + entries, sourcemeta::core::less_ignore_case, + &std::pair::first); + return entries; +} + +auto append_lowercased(std::string &output, const std::string_view name) + -> void { + for (const auto character : name) { + output.push_back(sourcemeta::core::to_lowercase(character)); + } +} + +auto append_canonical_headers( + std::string &output, + const std::vector> &entries) + -> void { + std::size_t index{0}; + while (index < entries.size()) { + const auto name{entries[index].first}; + append_lowercased(output, name); + output.push_back(':'); + bool first{true}; + while (index < entries.size() && + sourcemeta::core::equals_ignore_case(entries[index].first, name)) { + if (!first) { + output.push_back(','); + } + + sourcemeta::core::squeeze(entries[index].second, ' ', output); + first = false; + ++index; + } + + output.push_back('\n'); + } +} + +auto append_signed_headers( + std::string &output, + const std::vector> &entries) + -> void { + std::string_view previous; + bool first{true}; + for (const auto &[name, value] : entries) { + if (!first && sourcemeta::core::equals_ignore_case(name, previous)) { + continue; + } + + if (!first) { + output.push_back(';'); + } + + append_lowercased(output, name); + previous = name; + first = false; + } +} + +auto append_canonical_uri(std::string &output, std::string_view path, + const bool normalize) -> void { + std::string normalized; + if (normalize) { + // Beyond RFC 3986 dot-segment removal, the default canonicalization also + // collapses redundant slashes, which Amazon S3 does not + normalized = sourcemeta::core::URI::normalize_path( + sourcemeta::core::squeeze(path, '/')); + path = normalized; + } + + if (path.empty()) { + output.push_back('/'); + return; + } + + std::size_t start{0}; + while (true) { + const auto slash{path.find('/', start)}; + if (slash == std::string_view::npos) { + sourcemeta::core::URI::escape(path.substr(start), output); + break; + } + + sourcemeta::core::URI::escape(path.substr(start, slash - start), output); + output.push_back('/'); + start = slash + 1; + } +} + +auto append_canonical_query(std::string &output, const std::string_view query) + -> void { + if (query.empty()) { + return; + } + + std::vector> parameters; + for (const auto &[name, value] : sourcemeta::core::URI::Query{query}) { + std::string encoded_name; + std::string encoded_value; + sourcemeta::core::URI::escape(name, encoded_name); + sourcemeta::core::URI::escape(value, encoded_value); + parameters.emplace_back(std::move(encoded_name), std::move(encoded_value)); + } + + std::ranges::sort(parameters); + bool first{true}; + for (const auto &[key, value] : parameters) { + if (!first) { + output.push_back('&'); + } + + output.append(key); + output.push_back('='); + output.append(value); + first = false; + } +} + +} // namespace + +namespace sourcemeta::core { + +auto http_aws_sigv4_canonical_request( + const std::string_view method, const std::string_view path, + const std::string_view query, + const std::span> + headers, + const std::string_view payload_hash, const bool normalize) -> std::string { + const auto entries{sorted_headers(headers)}; + std::string result; + result.append(method); + result.push_back('\n'); + append_canonical_uri(result, path, normalize); + result.push_back('\n'); + append_canonical_query(result, query); + result.push_back('\n'); + append_canonical_headers(result, entries); + result.push_back('\n'); + append_signed_headers(result, entries); + result.push_back('\n'); + result.append(payload_hash); + return result; +} + +auto http_aws_sigv4_signed_headers( + const std::span> + headers) -> std::string { + std::string result; + append_signed_headers(result, sorted_headers(headers)); + return result; +} + +auto http_aws_sigv4_credential_scope(const std::string_view date, + const std::string_view region, + const std::string_view service) + -> std::string { + std::string result; + result.append(date); + result.push_back('/'); + result.append(region); + result.push_back('/'); + result.append(service); + result.append("/aws4_request"); + return result; +} + +auto http_aws_sigv4_string_to_sign(const std::string_view amz_date, + const std::string_view scope, + const std::string_view canonical_request) + -> std::string { + std::string result{"AWS4-HMAC-SHA256\n"}; + result.append(amz_date); + result.push_back('\n'); + result.append(scope); + result.push_back('\n'); + result.append(sha256(canonical_request)); + return result; +} + +auto http_aws_sigv4_signing_key(const std::string_view secret, + const std::string_view date, + const std::string_view region, + const std::string_view service) + -> std::array { + std::string initial{"AWS4"}; + initial.append(secret); + const auto key_date{hmac_sha256_digest(initial, date)}; + const auto key_region{hmac_sha256_digest(digest_view(key_date), region)}; + const auto key_service{hmac_sha256_digest(digest_view(key_region), service)}; + return hmac_sha256_digest(digest_view(key_service), "aws4_request"); +} + +auto http_aws_sigv4_signature(const std::array &signing_key, + const std::string_view string_to_sign) + -> std::string { + return hmac_sha256(digest_view(signing_key), string_to_sign); +} + +auto http_aws_sigv4_authorization(const std::string_view access_key_id, + const std::string_view scope, + const std::string_view signed_headers, + const std::string_view signature) + -> std::string { + std::string result{"AWS4-HMAC-SHA256 Credential="}; + result.append(access_key_id); + result.push_back('/'); + result.append(scope); + result.append(", SignedHeaders="); + result.append(signed_headers); + result.append(", Signature="); + result.append(signature); + return result; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/http/cache_control_max_age.cc b/vendor/core/src/core/http/cache_control_max_age.cc index 7d3e5c077..d00aebf82 100644 --- a/vendor/core/src/core/http/cache_control_max_age.cc +++ b/vendor/core/src/core/http/cache_control_max_age.cc @@ -24,7 +24,7 @@ auto http_cache_control_max_age(const std::string_view cache_control) noexcept const auto separator{directive.find('=')}; if (separator == std::string_view::npos || - !http_iequals_ascii(http_subview(directive, 0, separator), + !equals_ignore_case(http_subview(directive, 0, separator), "max-age")) { return; } diff --git a/vendor/core/src/core/http/content_type_matches.cc b/vendor/core/src/core/http/content_type_matches.cc index aee92a99f..283e4b0fc 100644 --- a/vendor/core/src/core/http/content_type_matches.cc +++ b/vendor/core/src/core/http/content_type_matches.cc @@ -11,7 +11,7 @@ auto http_content_type_matches(const std::string_view content_type_header, -> bool { const auto bare{ http_trim_leading_ows(http_split_entry(content_type_header).first)}; - return http_iequals_ascii(bare, media_type); + return equals_ignore_case(bare, media_type); } } // namespace sourcemeta::core diff --git a/vendor/core/src/core/http/helpers.h b/vendor/core/src/core/http/helpers.h index 826db7ecc..2a607910b 100644 --- a/vendor/core/src/core/http/helpers.h +++ b/vendor/core/src/core/http/helpers.h @@ -1,6 +1,8 @@ #ifndef SOURCEMETA_CORE_HTTP_HELPERS_H_ #define SOURCEMETA_CORE_HTTP_HELPERS_H_ +#include + #include // std::size_t #include // std::uint8_t, std::uint16_t #include // std::string_view @@ -14,25 +16,6 @@ inline auto http_is_ows(const char character) noexcept -> bool { return character == ' ' || character == '\t'; } -inline auto http_ascii_lower(const char character) noexcept -> char { - return (character >= 'A' && character <= 'Z') - ? static_cast(character + ('a' - 'A')) - : character; -} - -inline auto http_iequals_ascii(const std::string_view left, - const std::string_view right) noexcept -> bool { - if (left.size() != right.size()) { - return false; - } - for (std::size_t index{0}; index < left.size(); ++index) { - if (http_ascii_lower(left[index]) != http_ascii_lower(right[index])) { - return false; - } - } - return true; -} - inline auto http_subview(const std::string_view value, const std::size_t offset, const std::size_t length) noexcept -> std::string_view { @@ -42,7 +25,7 @@ inline auto http_subview(const std::string_view value, const std::size_t offset, inline auto http_media_specificity(const std::string_view range, const std::string_view candidate) noexcept -> std::uint8_t { - if (http_iequals_ascii(range, candidate)) { + if (equals_ignore_case(range, candidate)) { return 3; } if (range == "*/*") { @@ -60,10 +43,9 @@ inline auto http_media_specificity(const std::string_view range, if (range_slash != candidate_slash) { return 0; } - for (std::size_t index{0}; index < range_slash; ++index) { - if (http_ascii_lower(range[index]) != http_ascii_lower(candidate[index])) { - return 0; - } + if (!equals_ignore_case(http_subview(range, 0, range_slash), + http_subview(candidate, 0, range_slash))) { + return 0; } return 2; } diff --git a/vendor/core/src/core/http/include/sourcemeta/core/http.h b/vendor/core/src/core/http/include/sourcemeta/core/http.h index caa6ba739..99a3c9d81 100644 --- a/vendor/core/src/core/http/include/sourcemeta/core/http.h +++ b/vendor/core/src/core/http/include/sourcemeta/core/http.h @@ -6,6 +6,7 @@ #endif // NOLINTBEGIN(misc-include-cleaner) +#include #include #include #include @@ -253,6 +254,82 @@ auto http_format_links(std::span links, std::string &out) SOURCEMETA_CORE_HTTP_EXPORT auto http_format_links(std::span links) -> std::string; +/// @ingroup http +/// The `SameSite` attribute of a cookie per RFC 6265bis §5.2. +enum class HTTPCookieSameSite : std::uint8_t { Strict, Lax, None }; + +/// @ingroup http +/// A cookie to serialise into an RFC 6265 §4.1 `Set-Cookie` response header +/// value. The caller owns the backing storage for every field. A valid cookie +/// has a name that is an RFC 9110 §5.6.2 token and a value made of RFC 6265 +/// §4.1.1 cookie-octets. RFC 6265bis §5.7 requires a cookie with a same-site +/// mode of none to also be secure. +struct HTTPCookie { + std::string_view name{}; + std::string_view value{}; + std::optional path{}; + std::optional domain{}; + std::optional max_age{}; + bool http_only{false}; + bool secure{false}; + std::optional same_site{}; +}; + +/// @ingroup http +/// Test whether a cookie can be serialised into a valid RFC 6265 §4.1 +/// `Set-Cookie` header value: the name is a non-empty RFC 9110 §5.6.2 token, +/// the value is made of RFC 6265 §4.1.1 cookie-octets, any path is made of RFC +/// 6265bis av-octets, any domain is a valid RFC 1123 host name allowing an +/// ignorable leading dot, a present `max_age` is not negative, per RFC 6265bis +/// §5.7 a `HTTPCookieSameSite::None` cookie is also `secure`, and the RFC +/// 6265bis §4.1.3 `__Secure-` and `__Host-` name prefixes carry their required +/// attributes. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// assert(sourcemeta::core::http_cookie_valid({.name = "a", .value = "b"})); +/// assert(!sourcemeta::core::http_cookie_valid({.name = "a", .value = "b;c"})); +/// ``` +SOURCEMETA_CORE_HTTP_EXPORT +auto http_cookie_valid(const HTTPCookie &cookie) -> bool; + +/// @ingroup http +/// Append an RFC 6265 §4.1 `Set-Cookie` header value to `out`, returning `true` +/// on success. When the cookie is not `http_cookie_valid`, `out` is left +/// unchanged and this returns `false`. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// std::string buffer; +/// const auto ok{sourcemeta::core::http_serialize_cookie( +/// {.name = "session", .value = "abc", .http_only = true}, buffer)}; +/// assert(ok); +/// assert(buffer == "session=abc; HttpOnly"); +/// ``` +SOURCEMETA_CORE_HTTP_EXPORT +auto http_serialize_cookie(const HTTPCookie &cookie, std::string &out) -> bool; + +/// @ingroup http +/// Serialise an RFC 6265 §4.1 `Set-Cookie` header value, returning no value +/// when the cookie is not `http_cookie_valid`. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto value{sourcemeta::core::http_serialize_cookie( +/// {.name = "session", .value = "abc", .secure = true})}; +/// assert(value == "session=abc; Secure"); +/// ``` +SOURCEMETA_CORE_HTTP_EXPORT +auto http_serialize_cookie(const HTTPCookie &cookie) + -> std::optional; + /// @ingroup http /// Test whether a comma-separated header value per RFC 9110 §5.6.1 lists any /// of the given tokens. For example: diff --git a/vendor/core/src/core/http/include/sourcemeta/core/http_aws_sigv4.h b/vendor/core/src/core/http/include/sourcemeta/core/http_aws_sigv4.h new file mode 100644 index 000000000..1ec78202b --- /dev/null +++ b/vendor/core/src/core/http/include/sourcemeta/core/http_aws_sigv4.h @@ -0,0 +1,156 @@ +#ifndef SOURCEMETA_CORE_HTTP_AWS_SIGV4_H_ +#define SOURCEMETA_CORE_HTTP_AWS_SIGV4_H_ + +#ifndef SOURCEMETA_CORE_HTTP_EXPORT +#include +#endif + +#include // std::array +#include // std::uint8_t +#include // std::span +#include // std::string +#include // std::string_view +#include // std::pair + +namespace sourcemeta::core { + +/// @ingroup http +/// Compute the AWS Signature Version 4 canonical request from a request method, +/// path, query, headers, and payload hash. The path and query are percent- +/// encoded during canonicalization, so they must be provided in decoded form. +/// When `normalize` is set the path is normalised by collapsing sequential +/// slashes and removing dot segments, which every service except Amazon S3 +/// requires. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// #include +/// #include +/// +/// const std::vector> headers{ +/// {"Host", "example.amazonaws.com"}, {"X-Amz-Date", "20150830T123600Z"}}; +/// const auto canonical{sourcemeta::core::http_aws_sigv4_canonical_request( +/// "GET", "/", "", headers, +/// "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")}; +/// assert(canonical.starts_with("GET\n/\n")); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_canonical_request( + const std::string_view method, const std::string_view path, + const std::string_view query, + const std::span> + headers, + const std::string_view payload_hash, const bool normalize = true) + -> std::string; + +/// @ingroup http +/// Compute the AWS Signature Version 4 signed headers list, the lowercased +/// header names sorted and joined with a semicolon. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// #include +/// #include +/// +/// const std::vector> headers{ +/// {"Host", "example.amazonaws.com"}, {"X-Amz-Date", "20150830T123600Z"}}; +/// assert(sourcemeta::core::http_aws_sigv4_signed_headers(headers) == +/// "host;x-amz-date"); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_signed_headers( + const std::span> + headers) -> std::string; + +/// @ingroup http +/// Compute the AWS Signature Version 4 credential scope from a date, region, +/// and service. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// assert(sourcemeta::core::http_aws_sigv4_credential_scope( +/// "20150830", "us-east-1", "service") == +/// "20150830/us-east-1/service/aws4_request"); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_credential_scope( + const std::string_view date, const std::string_view region, + const std::string_view service) -> std::string; + +/// @ingroup http +/// Compute the AWS Signature Version 4 string to sign from a timestamp, a +/// credential scope, and a canonical request. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto result{sourcemeta::core::http_aws_sigv4_string_to_sign( +/// "20150830T123600Z", "20150830/us-east-1/service/aws4_request", +/// "canonical request")}; +/// assert(result.starts_with("AWS4-HMAC-SHA256\n20150830T123600Z\n")); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_string_to_sign( + const std::string_view amz_date, const std::string_view scope, + const std::string_view canonical_request) -> std::string; + +/// @ingroup http +/// Derive the AWS Signature Version 4 signing key from a secret access key, +/// date, region, and service. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto key{sourcemeta::core::http_aws_sigv4_signing_key( +/// "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", +/// "service")}; +/// assert(key.size() == 32); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_signing_key( + const std::string_view secret, const std::string_view date, + const std::string_view region, const std::string_view service) + -> std::array; + +/// @ingroup http +/// Compute the AWS Signature Version 4 hex signature from a signing key and a +/// string to sign. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto key{sourcemeta::core::http_aws_sigv4_signing_key( +/// "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", +/// "service")}; +/// assert(sourcemeta::core::http_aws_sigv4_signature(key, "string to sign") +/// .size() == 64); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT +http_aws_sigv4_signature(const std::array &signing_key, + const std::string_view string_to_sign) -> std::string; + +/// @ingroup http +/// Assemble the AWS Signature Version 4 `Authorization` header value from an +/// access key, credential scope, signed headers, and signature. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// const auto header{sourcemeta::core::http_aws_sigv4_authorization( +/// "AKIDEXAMPLE", "20150830/us-east-1/service/aws4_request", +/// "host;x-amz-date", "signature")}; +/// assert(header.starts_with("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/")); +/// ``` +auto SOURCEMETA_CORE_HTTP_EXPORT http_aws_sigv4_authorization( + const std::string_view access_key_id, const std::string_view scope, + const std::string_view signed_headers, const std::string_view signature) + -> std::string; + +} // namespace sourcemeta::core + +#endif diff --git a/vendor/core/src/core/http/include/sourcemeta/core/http_message.h b/vendor/core/src/core/http/include/sourcemeta/core/http_message.h index c6a62322b..c5e87526a 100644 --- a/vendor/core/src/core/http/include/sourcemeta/core/http_message.h +++ b/vendor/core/src/core/http/include/sourcemeta/core/http_message.h @@ -145,6 +145,90 @@ inline auto http_parse_headers(const std::string_view input, Callback callback) } } +/// @ingroup http +/// Parse the value of an RFC 6265 §4.2 `Cookie` request header, given without +/// the field name, into its cookie-pairs, invoking the callback once per pair +/// with the name and value. A request cookie header carries only names and +/// values, never attributes. Surrounding whitespace is trimmed, values are +/// otherwise reported verbatim, and pairs that lack a `=` or have an empty name +/// are skipped. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// +/// std::string_view last_value; +/// sourcemeta::core::http_parse_cookies( +/// "session=abc; theme=dark", +/// [&last_value](const std::string_view, const std::string_view value) { +/// last_value = value; +/// }); +/// assert(last_value == "dark"); +/// ``` +template + requires std::invocable +inline auto http_parse_cookies(const std::string_view input, Callback callback) + -> void { + std::string_view rest{input}; + while (!rest.empty()) { + std::string_view pair; + const auto separator{rest.find(';')}; + if (separator == std::string_view::npos) { + pair = rest; + rest = {}; + } else { + pair = rest.substr(0, separator); + rest = rest.substr(separator + 1); + } + + const auto parts{split_once(trim(pair), '=')}; + if (!parts.has_value()) { + continue; + } + + const auto name{trim(parts->first)}; + if (name.empty()) { + continue; + } + + callback(name, trim(parts->second)); + } +} + +/// @ingroup http +/// Parse the value of an RFC 6265 §4.2 `Cookie` request header, given without +/// the field name, into any container of name and value pairs. The names and +/// values are forwarded as views that borrow from `input`, so a container of +/// `std::string_view` pairs collects them without copying, while a container of +/// `std::string` pairs owns them. For example: +/// +/// ```cpp +/// #include +/// #include +/// #include +/// #include +/// #include +/// +/// std::vector> cookies; +/// sourcemeta::core::http_parse_cookies("session=abc; theme=dark", cookies); +/// assert(cookies.size() == 2); +/// assert(cookies.at(0).first == "session"); +/// assert(cookies.at(0).second == "abc"); +/// ``` +template + requires requires(Container container, std::string_view entry) { + container.emplace_back(entry, entry); + } +inline auto http_parse_cookies(const std::string_view input, Container &cookies) + -> void { + http_parse_cookies(input, + [&cookies](const std::string_view name, + const std::string_view value) -> void { + cookies.emplace_back(name, value); + }); +} + /// @ingroup http /// Parse the field lines of a raw message header block, skipping the start /// line, into any container of name and value pairs, normalising names to diff --git a/vendor/core/src/core/http/match_accept_language.cc b/vendor/core/src/core/http/match_accept_language.cc index caece96ce..eac790be1 100644 --- a/vendor/core/src/core/http/match_accept_language.cc +++ b/vendor/core/src/core/http/match_accept_language.cc @@ -15,11 +15,11 @@ auto language_specificity(const std::string_view range, if (range == "*") { return 1; } - if (sourcemeta::core::http_iequals_ascii(range, candidate)) { + if (sourcemeta::core::equals_ignore_case(range, candidate)) { return candidate.size() + 1; } if (range.size() > candidate.size() && range[candidate.size()] == '-' && - sourcemeta::core::http_iequals_ascii( + sourcemeta::core::equals_ignore_case( sourcemeta::core::http_subview(range, 0, candidate.size()), candidate)) { return candidate.size(); diff --git a/vendor/core/src/core/http/negotiate_encoding.cc b/vendor/core/src/core/http/negotiate_encoding.cc index 09f89fcc0..a0df72528 100644 --- a/vendor/core/src/core/http/negotiate_encoding.cc +++ b/vendor/core/src/core/http/negotiate_encoding.cc @@ -26,13 +26,13 @@ auto http_negotiate_encoding( http_for_each_accept_entry( accept_encoding_header, [&](const std::string_view token, const float quality) -> void { - if (http_iequals_ascii(token, "gzip") || - http_iequals_ascii(token, "x-gzip")) { + if (equals_ignore_case(token, "gzip") || + equals_ignore_case(token, "x-gzip")) { gzip_listed = true; if (quality > gzip_quality) { gzip_quality = quality; } - } else if (http_iequals_ascii(token, "identity")) { + } else if (equals_ignore_case(token, "identity")) { identity_listed = true; if (quality > identity_quality) { identity_quality = quality; diff --git a/vendor/core/src/core/http/parse_bearer.cc b/vendor/core/src/core/http/parse_bearer.cc index aaa3ca48c..cb6c8bca7 100644 --- a/vendor/core/src/core/http/parse_bearer.cc +++ b/vendor/core/src/core/http/parse_bearer.cc @@ -50,7 +50,7 @@ auto http_parse_bearer(const std::string_view authorization) noexcept return {}; } - if (!http_iequals_ascii(http_subview(authorization, 0, scheme.size()), + if (!equals_ignore_case(http_subview(authorization, 0, scheme.size()), scheme)) { return {}; } diff --git a/vendor/core/src/core/http/serialize_cookie.cc b/vendor/core/src/core/http/serialize_cookie.cc new file mode 100644 index 000000000..008b653ac --- /dev/null +++ b/vendor/core/src/core/http/serialize_cookie.cc @@ -0,0 +1,278 @@ +#include +#include +#include + +#include // std::array +#include // std::to_chars +#include // std::chrono::seconds +#include // std::size_t +#include // std::numeric_limits +#include // std::optional, std::nullopt +#include // std::string +#include // std::string_view +#include // std::unreachable + +namespace { + +auto is_token_character(const char character) noexcept -> bool { + // RFC 9110 §5.6.2: tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / + // "-" / "." / "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA + switch (character) { + case '!': + case '#': + case '$': + case '%': + case '&': + case '\'': + case '*': + case '+': + case '-': + case '.': + case '^': + case '_': + case '`': + case '|': + case '~': + return true; + default: + return sourcemeta::core::is_alphanum(character); + } +} + +auto is_cookie_octet(const char character) noexcept -> bool { + // RFC 6265 §4.1.1: cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / + // %x5D-7E, that is US-ASCII excluding CTLs, whitespace, DQUOTE, comma, + // semicolon, and backslash + const auto value{static_cast(character)}; + return value == 0x21 || (value >= 0x23 && value <= 0x2B) || + (value >= 0x2D && value <= 0x3A) || (value >= 0x3C && value <= 0x5B) || + (value >= 0x5D && value <= 0x7E); +} + +auto is_attribute_octet(const char character) noexcept -> bool { + // RFC 6265bis §4.1.1: path-value and domain-value are made of av-octet = + // %x20-3A / %x3C-7E, that is printable US-ASCII excluding the ";" delimiter, + // control characters, DEL, and any non-ASCII byte + const auto value{static_cast(character)}; + return value >= 0x20 && value <= 0x7E && character != ';'; +} + +auto is_token(const std::string_view value) noexcept -> bool { + if (value.empty()) { + return false; + } + + for (const auto character : value) { + if (!is_token_character(character)) { + return false; + } + } + + return true; +} + +auto is_cookie_value(const std::string_view value) noexcept -> bool { + // RFC 6265 §4.1.1: cookie-value is either bare cookie-octets or the same + // wrapped in a matched pair of double quotes + for (const auto character : sourcemeta::core::unquote(value, '"')) { + if (!is_cookie_octet(character)) { + return false; + } + } + + return true; +} + +auto is_attribute_value(const std::string_view value) noexcept -> bool { + for (const auto character : value) { + if (!is_attribute_octet(character)) { + return false; + } + } + + return true; +} + +auto is_cookie_domain(const std::string_view value) -> bool { + // RFC 6265 §4.1.2.3 lets a domain carry an ignorable leading dot, which is + // dropped before the remainder is checked as an RFC 1123 host name + const auto host{value.starts_with('.') ? value.substr(1) : value}; + return sourcemeta::core::is_hostname(host); +} + +constexpr std::string_view COOKIE_PAIR_SEPARATOR{"="}; +constexpr std::string_view COOKIE_ATTRIBUTE_SEPARATOR{"; "}; +constexpr std::string_view COOKIE_DOMAIN{"Domain="}; +constexpr std::string_view COOKIE_PATH{"Path="}; +constexpr std::string_view COOKIE_MAX_AGE{"Max-Age="}; +constexpr std::string_view COOKIE_SECURE{"Secure"}; +constexpr std::string_view COOKIE_HTTP_ONLY{"HttpOnly"}; +constexpr std::string_view COOKIE_SAME_SITE{"SameSite="}; +constexpr std::string_view COOKIE_SAME_SITE_STRICT{"Strict"}; +constexpr std::string_view COOKIE_SAME_SITE_LAX{"Lax"}; +constexpr std::string_view COOKIE_SAME_SITE_NONE{"None"}; + +auto same_site_token(const sourcemeta::core::HTTPCookieSameSite value) noexcept + -> std::string_view { + switch (value) { + case sourcemeta::core::HTTPCookieSameSite::Strict: + return COOKIE_SAME_SITE_STRICT; + case sourcemeta::core::HTTPCookieSameSite::Lax: + return COOKIE_SAME_SITE_LAX; + case sourcemeta::core::HTTPCookieSameSite::None: + return COOKIE_SAME_SITE_NONE; + } + + std::unreachable(); +} + +auto required_size(const sourcemeta::core::HTTPCookie &cookie, + const std::string_view max_age) noexcept -> std::size_t { + std::size_t size{cookie.name.size() + COOKIE_PAIR_SEPARATOR.size() + + cookie.value.size()}; + if (cookie.domain.has_value()) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_DOMAIN.size() + + cookie.domain->size(); + } + if (cookie.path.has_value()) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_PATH.size() + + cookie.path->size(); + } + if (cookie.max_age.has_value()) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_MAX_AGE.size() + + max_age.size(); + } + if (cookie.secure) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_SECURE.size(); + } + if (cookie.http_only) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_HTTP_ONLY.size(); + } + if (cookie.same_site.has_value()) { + size += COOKIE_ATTRIBUTE_SEPARATOR.size() + COOKIE_SAME_SITE.size() + + same_site_token(*cookie.same_site).size(); + } + return size; +} + +} // namespace + +namespace sourcemeta::core { + +auto http_cookie_valid(const HTTPCookie &cookie) -> bool { + if (!is_token(cookie.name) || !is_cookie_value(cookie.value)) { + return false; + } + + if (cookie.path.has_value() && !is_attribute_value(*cookie.path)) { + return false; + } + + if (cookie.domain.has_value() && !is_cookie_domain(*cookie.domain)) { + return false; + } + + // RFC 6265bis §4.1.1 defines the max age attribute value as one or more + // digits, so a negative expiry cannot be serialised. Zero or a positive value + // is valid, and zero is the canonical way to expire a cookie + if (cookie.max_age.has_value() && cookie.max_age->count() < 0) { + return false; + } + + // RFC 6265bis §5.7 ignores a cookie whose same-site mode is none unless it is + // also marked secure + if (cookie.same_site == HTTPCookieSameSite::None && !cookie.secure) { + return false; + } + + // RFC 6265bis §4.1.3.1 requires a cookie whose name carries the secure prefix + // to be marked secure. The prefix is matched case-insensitively because RFC + // 6265bis §5.4 has user agents do so, and a mismatched cookie is dropped + if (starts_with_ignore_case(cookie.name, "__Secure-") && !cookie.secure) { + return false; + } + + // RFC 6265bis §4.1.3.2 requires a cookie whose name carries the host prefix + // to be marked secure, scoped to the root path, and free of a domain, again + // matched case-insensitively to agree with the user agent + if (starts_with_ignore_case(cookie.name, "__Host-") && + (!cookie.secure || cookie.domain.has_value() || + !cookie.path.has_value() || *cookie.path != "/")) { + return false; + } + + return true; +} + +auto http_serialize_cookie(const HTTPCookie &cookie, std::string &out) -> bool { + if (!http_cookie_valid(cookie)) { + return false; + } + + // Format the max age into a stack buffer, avoiding an intermediate heap + // allocation. A non-negative representation never exceeds these digits + std::array::digits10 + 2> + max_age_buffer; + std::string_view max_age; + if (cookie.max_age.has_value()) { + const auto result{std::to_chars( + max_age_buffer.data(), max_age_buffer.data() + max_age_buffer.size(), + cookie.max_age->count())}; + max_age = std::string_view{ + max_age_buffer.data(), + static_cast(result.ptr - max_age_buffer.data())}; + } + + out.reserve(out.size() + required_size(cookie, max_age)); + out.append(cookie.name); + out.append(COOKIE_PAIR_SEPARATOR); + out.append(cookie.value); + + if (cookie.domain.has_value()) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_DOMAIN); + out.append(*cookie.domain); + } + + if (cookie.path.has_value()) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_PATH); + out.append(*cookie.path); + } + + if (cookie.max_age.has_value()) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_MAX_AGE); + out.append(max_age); + } + + if (cookie.secure) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_SECURE); + } + + if (cookie.http_only) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_HTTP_ONLY); + } + + if (cookie.same_site.has_value()) { + out.append(COOKIE_ATTRIBUTE_SEPARATOR); + out.append(COOKIE_SAME_SITE); + out.append(same_site_token(*cookie.same_site)); + } + + return true; +} + +auto http_serialize_cookie(const HTTPCookie &cookie) + -> std::optional { + std::string out; + if (!http_serialize_cookie(cookie, out)) { + return std::nullopt; + } + + return out; +} + +} // namespace sourcemeta::core diff --git a/vendor/core/src/core/idna/CMakeLists.txt b/vendor/core/src/core/idna/CMakeLists.txt index 21006e31e..14c7cfd03 100644 --- a/vendor/core/src/core/idna/CMakeLists.txt +++ b/vendor/core/src/core/idna/CMakeLists.txt @@ -18,10 +18,12 @@ add_custom_command( COMMAND "${SOURCEMETA_CORE_IDNA_CODEGEN_TARGET}" "${SOURCEMETA_CORE_IDNA_DATA_HEADER}" "${SOURCEMETA_CORE_IDNA_UCD_DIR}/Idna2008.txt" + "${SOURCEMETA_CORE_IDNA_UCD_DIR}/IdnaMappingTable.txt" DEPENDS "${SOURCEMETA_CORE_IDNA_CODEGEN_TARGET}" "${SOURCEMETA_CORE_IDNA_UCD_DIR}/Idna2008.txt" - COMMENT "Generating IDNA property tables" + "${SOURCEMETA_CORE_IDNA_UCD_DIR}/IdnaMappingTable.txt" + COMMENT "Generating IDNA property and mapping tables" VERBATIM) sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME idna diff --git a/vendor/core/src/core/idna/codegen.cc b/vendor/core/src/core/idna/codegen.cc index 88bfba34b..2d44d79e4 100644 --- a/vendor/core/src/core/idna/codegen.cc +++ b/vendor/core/src/core/idna/codegen.cc @@ -5,7 +5,8 @@ #include #include -#include // std::size_t, std::ptrdiff_t +#include // std::sort +#include // std::size_t, std::ptrdiff_t #include // std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t #include // EXIT_FAILURE, EXIT_SUCCESS #include // std::exception @@ -108,17 +109,7 @@ auto parse_idna_file(const std::filesystem::path &input_path) return result; } -auto build_pages(const std::vector &entries) -> TwoStageTable { - std::vector values( - TOTAL_CODEPOINTS, - static_cast(sourcemeta::core::IDNAProperty::PValid)); - for (const auto &entry : entries) { - for (std::uint32_t codepoint{entry.first}; codepoint <= entry.last; - codepoint += 1) { - values[codepoint] = static_cast(entry.value); - } - } - +auto compress_pages(const std::vector &values) -> TwoStageTable { std::unordered_map page_to_id; TwoStageTable table; table.stage1.reserve(NUM_PAGES); @@ -145,6 +136,19 @@ auto build_pages(const std::vector &entries) -> TwoStageTable { return table; } +auto build_pages(const std::vector &entries) -> TwoStageTable { + std::vector values( + TOTAL_CODEPOINTS, + static_cast(sourcemeta::core::IDNAProperty::PValid)); + for (const auto &entry : entries) { + for (std::uint32_t codepoint{entry.first}; codepoint <= entry.last; + codepoint += 1) { + values[codepoint] = static_cast(entry.value); + } + } + return compress_pages(values); +} + template auto emit_row(std::ostream &stream, const std::span items) -> void { constexpr std::size_t row_width{16}; @@ -173,6 +177,161 @@ auto emit_property(std::ostream &stream, const std::string_view prefix, stream << "};\n\n"; } +struct MappingEntry { + std::uint32_t first; + std::uint32_t last; + sourcemeta::core::IDNAMappingStatus status; + std::u32string mapping; +}; + +struct MappingIndexEntry { + std::uint32_t codepoint; + std::uint32_t offset; + std::uint32_t length; +}; + +struct MappingTable { + TwoStageTable status; + std::u32string pool; + std::vector index; +}; + +auto mapping_status_from_token(const std::string_view token) + -> sourcemeta::core::IDNAMappingStatus { +#define SOURCEMETA_CORE_IDNA_MAPPING_CASE(name, alias) \ + if (token == alias) { \ + return sourcemeta::core::IDNAMappingStatus::name; \ + } + SOURCEMETA_CORE_IDNA_MAPPING_STATUS_LIST(SOURCEMETA_CORE_IDNA_MAPPING_CASE) +#undef SOURCEMETA_CORE_IDNA_MAPPING_CASE + throw std::runtime_error{ + std::string{"Unknown IDNA mapping status: "}.append(token)}; +} + +// Parse a space-separated list of hexadecimal codepoints (the mapping column) +auto parse_mapping_sequence(const std::string_view field) -> std::u32string { + std::u32string result; + sourcemeta::core::split(field, ' ', [&](const std::string_view token) { + const auto trimmed{sourcemeta::core::trim(token)}; + if (!trimmed.empty()) { + result.push_back(static_cast(parse_hex_codepoint(trimmed))); + } + }); + return result; +} + +auto parse_mapping_line(const std::string_view payload) -> MappingEntry { + const auto trimmed{ + sourcemeta::core::trim(sourcemeta::core::take_until(payload, '#'))}; + const auto first_split{sourcemeta::core::split_once(trimmed, ';')}; + if (!first_split.has_value()) { + throw std::runtime_error{ + std::string{"Missing ';' in line: "}.append(payload)}; + } + const auto range_part{sourcemeta::core::trim(first_split->first)}; + const auto range_split{ + sourcemeta::core::split_once(range_part, std::string_view{".."})}; + const auto first{parse_hex_codepoint( + range_split.has_value() ? range_split->first : range_part)}; + const auto last{range_split.has_value() + ? parse_hex_codepoint(range_split->second) + : first}; + + const auto second_split{ + sourcemeta::core::split_once(first_split->second, ';')}; + const auto status_part{sourcemeta::core::trim( + second_split.has_value() ? second_split->first : first_split->second)}; + const auto status{mapping_status_from_token(status_part)}; + + std::u32string mapping; + if (status == sourcemeta::core::IDNAMappingStatus::Mapped && + second_split.has_value()) { + const auto third_split{ + sourcemeta::core::split_once(second_split->second, ';')}; + const auto mapping_part{sourcemeta::core::trim( + third_split.has_value() ? third_split->first : second_split->second)}; + mapping = parse_mapping_sequence(mapping_part); + } + + return {first, last, status, std::move(mapping)}; +} + +auto parse_mapping_file(const std::filesystem::path &input_path) + -> std::vector { + auto stream{sourcemeta::core::read_file(input_path)}; + std::vector entries; + sourcemeta::core::for_each_line(stream, [&](const std::string_view raw_line) { + const auto line{sourcemeta::core::trim(raw_line)}; + if (line.empty() || line.front() == '#') { + return; + } + entries.push_back(parse_mapping_line(line)); + }); + return entries; +} + +auto build_mapping(const std::vector &entries) -> MappingTable { + // Codepoints absent from the table default to disallowed per UTS #46 + std::vector status_values( + TOTAL_CODEPOINTS, static_cast( + sourcemeta::core::IDNAMappingStatus::Disallowed)); + MappingTable table; + // Deduplicate identical replacement sequences into a shared pool + std::unordered_map sequence_to_offset; + for (const auto &entry : entries) { + std::uint32_t offset{0}; + if (entry.status == sourcemeta::core::IDNAMappingStatus::Mapped) { + const auto existing{sequence_to_offset.find(entry.mapping)}; + if (existing != sequence_to_offset.end()) { + offset = existing->second; + } else { + offset = static_cast(table.pool.size()); + sequence_to_offset.emplace(entry.mapping, offset); + table.pool.append(entry.mapping); + } + } + for (std::uint32_t codepoint{entry.first}; codepoint <= entry.last; + codepoint += 1) { + status_values[codepoint] = static_cast(entry.status); + if (entry.status == sourcemeta::core::IDNAMappingStatus::Mapped) { + table.index.push_back( + {codepoint, offset, + static_cast(entry.mapping.size())}); + } + } + } + + std::sort(table.index.begin(), table.index.end(), + [](const MappingIndexEntry &left, const MappingIndexEntry &right) { + return left.codepoint < right.codepoint; + }); + table.status = compress_pages(status_values); + return table; +} + +auto emit_mapping(std::ostream &stream, const MappingTable &table) -> void { + emit_property(stream, "IDNA_MAPPING_STATUS", table.status); + + stream << "constexpr char32_t IDNA_MAPPING_POOL[" << table.pool.size() + << "] = {\n"; + emit_row( + stream, std::span{table.pool.data(), table.pool.size()}); + stream << "};\n\n"; + + stream << "struct IDNAMappingEntry {\n"; + stream << " std::uint32_t codepoint;\n"; + stream << " std::uint32_t offset;\n"; + stream << " std::uint32_t length;\n"; + stream << "};\n\n"; + stream << "constexpr IDNAMappingEntry IDNA_MAPPING_INDEX[" + << table.index.size() << "] = {\n"; + for (const auto &entry : table.index) { + stream << " {" << entry.codepoint << ", " << entry.offset << ", " + << entry.length << "},\n"; + } + stream << "};\n\n"; +} + } // namespace auto main(const int argc, const char *const argv[]) -> int { @@ -180,21 +339,23 @@ auto main(const int argc, const char *const argv[]) -> int { sourcemeta::core::Options app; app.parse(argc, argv); const auto &positional{app.positional()}; - if (positional.size() != 2) { + if (positional.size() != 3) { std::cerr << "Usage: " << (argc > 0 ? argv[0] : "codegen") - << " \n"; + << " \n"; return EXIT_FAILURE; } const std::filesystem::path output_path{positional.at(0)}; - const std::filesystem::path input_path{positional.at(1)}; + const std::filesystem::path property_path{positional.at(1)}; + const std::filesystem::path mapping_path{positional.at(2)}; - const auto entries{parse_idna_file(input_path)}; - const auto table{build_pages(entries)}; + const auto table{build_pages(parse_idna_file(property_path))}; + const auto mapping{build_mapping(parse_mapping_file(mapping_path))}; sourcemeta::core::write_file(output_path, [&](std::ostream &stream) { stream << "#include \n\n"; stream << "namespace {\n\n"; emit_property(stream, "IDNA_PROPERTY", table); + emit_mapping(stream, mapping); stream << "} // namespace\n"; }); } catch (const std::exception &error) { diff --git a/vendor/core/src/core/idna/idna.cc b/vendor/core/src/core/idna/idna.cc index 87018b7ed..41926b54a 100644 --- a/vendor/core/src/core/idna/idna.cc +++ b/vendor/core/src/core/idna/idna.cc @@ -3,6 +3,7 @@ #include #include +#include // std::ranges::lower_bound #include // std::size_t #include // std::optional, std::nullopt #include // std::string, std::u32string @@ -428,4 +429,52 @@ auto idna_is_valid_a_label(const std::string_view label) -> bool { return validate_a_label_body(encoded, decoded); } +namespace { + +auto mapping_status(const char32_t codepoint) noexcept -> IDNAMappingStatus { + if (codepoint > 0x10FFFF) { + return IDNAMappingStatus::Disallowed; + } + const std::size_t page{IDNA_MAPPING_STATUS_STAGE1[codepoint >> 10U]}; + return static_cast( + IDNA_MAPPING_STATUS_STAGE2[(page << 10U) | (codepoint & 0x3FFU)]); +} + +// The replacement sequence for a Mapped codepoint. The caller must only +// invoke this when `mapping_status(codepoint)` is Mapped, in which case the +// codepoint is guaranteed to be present in the index. +auto mapping_replacement(const char32_t codepoint) noexcept + -> std::u32string_view { + const auto *const found{std::ranges::lower_bound( + IDNA_MAPPING_INDEX, codepoint, {}, &IDNAMappingEntry::codepoint)}; + return std::u32string_view{IDNA_MAPPING_POOL + found->offset, found->length}; +} + +} // namespace + +auto idna_uts46_map(const std::u32string_view input) -> std::u32string { + std::u32string mapped; + mapped.reserve(input.size()); + for (const auto codepoint : input) { + switch (mapping_status(codepoint)) { + // UTS #46 §4 step 1. Valid and (under Nontransitional Processing) + // deviation code points are kept unchanged. Disallowed code points are + // also left in place, so the later validity check rejects them rather + // than the mapping silently dropping the whole input. + case IDNAMappingStatus::Valid: + case IDNAMappingStatus::Deviation: + case IDNAMappingStatus::Disallowed: + mapped.push_back(codepoint); + break; + case IDNAMappingStatus::Mapped: + mapped.append(mapping_replacement(codepoint)); + break; + case IDNAMappingStatus::Ignored: + break; + } + } + + return nfc(mapped); +} + } // namespace sourcemeta::core diff --git a/vendor/core/src/core/idna/include/sourcemeta/core/idna.h b/vendor/core/src/core/idna/include/sourcemeta/core/idna.h index 9c7874682..6f86204f8 100644 --- a/vendor/core/src/core/idna/include/sourcemeta/core/idna.h +++ b/vendor/core/src/core/idna/include/sourcemeta/core/idna.h @@ -183,6 +183,33 @@ auto idna_is_valid_u_label(const std::u32string_view label) -> bool; SOURCEMETA_CORE_IDNA_EXPORT auto idna_is_valid_a_label(const std::string_view label) -> bool; +/// @ingroup idna +/// Apply the UTS #46 mapping to `input` and return the result normalised to +/// NFC. This is the mapping and normalisation half of UTS #46 processing +/// (steps 1 and 2), performed with Nontransitional Processing and +/// UseSTD3ASCIIRules disabled, so the four deviation characters (U+00DF, +/// U+03C2, U+200C, U+200D) map to themselves. Following the UTS #46 Map step, +/// disallowed codepoints are left unchanged rather than removed, so that the +/// later per-label validity check is what rejects them (the same is true of +/// ASCII characters outside the LDH set). Label separation and the per-label +/// validity checks are not performed here. See +/// https://www.unicode.org/reports/tr46/ for the algorithm. For example: +/// +/// ```cpp +/// #include +/// #include +/// +/// // Uppercase is case-folded, fullwidth digits map to ASCII +/// assert(sourcemeta::core::idna_uts46_map(U"EXAMPLE") == U"example"); +/// assert(sourcemeta::core::idna_uts46_map(U"\uFF11\uFF12\uFF13") == U"123"); +/// // Soft hyphen is ignored +/// assert(sourcemeta::core::idna_uts46_map(U"a\u00ADb") == U"ab"); +/// // A disallowed codepoint is preserved for validation to reject +/// assert(sourcemeta::core::idna_uts46_map(U"\u0080") == U"\u0080"); +/// ``` +SOURCEMETA_CORE_IDNA_EXPORT +auto idna_uts46_map(const std::u32string_view input) -> std::u32string; + } // namespace sourcemeta::core #endif diff --git a/vendor/core/src/core/idna/include/sourcemeta/core/idna_ucd.h b/vendor/core/src/core/idna/include/sourcemeta/core/idna_ucd.h index 737537cb0..589f4d670 100644 --- a/vendor/core/src/core/idna/include/sourcemeta/core/idna_ucd.h +++ b/vendor/core/src/core/idna/include/sourcemeta/core/idna_ucd.h @@ -23,6 +23,26 @@ enum class IDNAProperty : std::uint8_t { #undef SOURCEMETA_CORE_IDNA_ENUM_ENTRY }; +/// @ingroup idna +/// Each entry maps an `IDNAMappingStatus` enum name to its UTS #46 +/// IdnaMappingTable.txt token. +#define SOURCEMETA_CORE_IDNA_MAPPING_STATUS_LIST(X) \ + X(Valid, "valid") \ + X(Ignored, "ignored") \ + X(Mapped, "mapped") \ + X(Deviation, "deviation") \ + X(Disallowed, "disallowed") + +/// @ingroup idna +/// The UTS #46 status of a Unicode codepoint in the mapping table. See +/// https://www.unicode.org/reports/tr46/ for the status definitions. +enum class IDNAMappingStatus : std::uint8_t { +#define SOURCEMETA_CORE_IDNA_MAPPING_ENUM_ENTRY(name, alias) name, + SOURCEMETA_CORE_IDNA_MAPPING_STATUS_LIST( + SOURCEMETA_CORE_IDNA_MAPPING_ENUM_ENTRY) +#undef SOURCEMETA_CORE_IDNA_MAPPING_ENUM_ENTRY +}; + } // namespace sourcemeta::core #endif diff --git a/vendor/core/src/core/ip/CMakeLists.txt b/vendor/core/src/core/ip/CMakeLists.txt index 3c2fdd7e6..99c9faf2b 100644 --- a/vendor/core/src/core/ip/CMakeLists.txt +++ b/vendor/core/src/core/ip/CMakeLists.txt @@ -4,3 +4,5 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME ip if(SOURCEMETA_CORE_INSTALL) sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME ip) endif() + +target_link_libraries(sourcemeta_core_ip PRIVATE sourcemeta::core::text) diff --git a/vendor/core/src/core/ip/ipv6.cc b/vendor/core/src/core/ip/ipv6.cc index 099be9bdb..4d7883b38 100644 --- a/vendor/core/src/core/ip/ipv6.cc +++ b/vendor/core/src/core/ip/ipv6.cc @@ -1,26 +1,8 @@ #include - -#include // std::array -#include // std::uint8_t +#include namespace sourcemeta::core { -static constexpr auto make_hex_table() -> std::array { - std::array table{}; - for (auto index{0u}; index < 256; index++) { - table[index] = (index >= '0' && index <= '9') || - (index >= 'a' && index <= 'f') || - (index >= 'A' && index <= 'F'); - } - return table; -} - -static constexpr auto HEX_TABLE{make_hex_table()}; - -static constexpr auto is_hex_digit(const char character) -> bool { - return HEX_TABLE[static_cast(character)]; -} - auto is_ipv6(const std::string_view address) -> bool { if (address.empty()) { return false; diff --git a/vendor/core/src/core/json/CMakeLists.txt b/vendor/core/src/core/json/CMakeLists.txt index a9c059067..898e9728a 100644 --- a/vendor/core/src/core/json/CMakeLists.txt +++ b/vendor/core/src/core/json/CMakeLists.txt @@ -7,6 +7,7 @@ if(SOURCEMETA_CORE_INSTALL) endif() target_link_libraries(sourcemeta_core_json PRIVATE sourcemeta::core::io) +target_link_libraries(sourcemeta_core_json PRIVATE sourcemeta::core::text) target_link_libraries(sourcemeta_core_json PRIVATE sourcemeta::core::unicode) target_link_libraries(sourcemeta_core_json PUBLIC sourcemeta::core::numeric) target_link_libraries(sourcemeta_core_json PUBLIC sourcemeta::core::preprocessor) diff --git a/vendor/core/src/core/json/construct.h b/vendor/core/src/core/json/construct.h index 8c385096a..f837136f1 100644 --- a/vendor/core/src/core/json/construct.h +++ b/vendor/core/src/core/json/construct.h @@ -5,6 +5,7 @@ #include #include +#include #include #include "parser.h" @@ -82,18 +83,9 @@ inline auto unescape_string(const char *data, const std::uint32_t length, auto parse_hex4 = [](const char *&position) -> unsigned long { unsigned long value{0}; for (std::size_t index = 0; index < 4; index++) { - const char hex_char{*position++}; - unsigned long digit; - if (hex_char >= '0' && hex_char <= '9') { - digit = static_cast(hex_char - '0'); - } else if (hex_char >= 'a' && hex_char <= 'f') { - digit = static_cast(hex_char - 'a') + 10; - } else if (hex_char >= 'A' && hex_char <= 'F') { - digit = static_cast(hex_char - 'A') + 10; - } else { - digit = 0; - } - value = (value << 4) | digit; + const auto digit{hex_digit_value(*position++)}; + value = (value << 4) | + static_cast(digit < 0 ? 0 : digit); } return value; }; diff --git a/vendor/core/src/core/json/parser.h b/vendor/core/src/core/json/parser.h index 188c3f2b5..bf8301ee5 100644 --- a/vendor/core/src/core/json/parser.h +++ b/vendor/core/src/core/json/parser.h @@ -3,6 +3,7 @@ #include #include +#include #include "grammar.h" @@ -181,18 +182,11 @@ inline auto scan_string_unicode_code_point(const std::uint64_t line, if (cursor >= end) [[unlikely]] { throw JSONParseError(line, column); } - const char hex_char{*cursor++}; - unsigned long digit; - if (hex_char >= '0' && hex_char <= '9') { - digit = static_cast(hex_char - '0'); - } else if (hex_char >= 'a' && hex_char <= 'f') { - digit = static_cast(hex_char - 'a') + 10; - } else if (hex_char >= 'A' && hex_char <= 'F') { - digit = static_cast(hex_char - 'A') + 10; - } else [[unlikely]] { + const auto digit{hex_digit_value(*cursor++)}; + if (digit < 0) [[unlikely]] { throw JSONParseError(line, column); } - result = (result << 4) | digit; + result = (result << 4) | static_cast(digit); } assert(result <= 0xFFFF); diff --git a/vendor/core/src/core/jsonld/include/sourcemeta/core/jsonld_materialize.h b/vendor/core/src/core/jsonld/include/sourcemeta/core/jsonld_materialize.h index 9a4fbf5ad..6bcbe7e61 100644 --- a/vendor/core/src/core/jsonld/include/sourcemeta/core/jsonld_materialize.h +++ b/vendor/core/src/core/jsonld/include/sourcemeta/core/jsonld_materialize.h @@ -57,10 +57,17 @@ struct JSONLDReference { }; /// @ingroup jsonld -/// A position that is an array, asserted as an RDF list when ordered and as a -/// set otherwise +/// The container form of a collection position. List and Set range over an +/// array; Language and Index range over an object. +enum class JSONLDContainer : std::uint8_t { List, Set, Language, Index }; + +/// @ingroup jsonld +/// A position that is a collection. A List is asserted as an RDF list and a Set +/// as a set, both ranging over an array. A Language container ranges over an +/// object of language tag to string, and an Index container over an object +/// whose keys are index labels that carry no RDF. struct JSONLDCollection { - bool ordered{false}; + JSONLDContainer container{JSONLDContainer::Set}; }; /// @ingroup jsonld diff --git a/vendor/core/src/core/jsonld/jsonld_materialize.cc b/vendor/core/src/core/jsonld/jsonld_materialize.cc index 8c3643c52..c501a5bbc 100644 --- a/vendor/core/src/core/jsonld/jsonld_materialize.cc +++ b/vendor/core/src/core/jsonld/jsonld_materialize.cc @@ -4,7 +4,8 @@ #include "jsonld_keywords.h" -#include // std::ranges::sort +#include // std::ranges::sort, std::ranges::none_of +#include // assert #include // std::size_t #include // std::reference_wrapper, std::cref #include // std::optional, std::nullopt @@ -189,6 +190,79 @@ auto build_collection(const JSON &value, PointerT &pointer, return result; } +// The keys of an object in sorted order, so the object walk is canonical +// regardless of the instance key order. +auto sorted_keys(const JSON &value) + -> std::vector> { + std::vector> keys; + keys.reserve(value.object_size()); + for (const auto &entry : value.as_object()) { + keys.push_back(std::cref(entry.first)); + } + std::ranges::sort(keys, [](const auto &left, const auto &right) -> bool { + return left.get() < right.get(); + }); + return keys; +} + +// The reserved @none key carries no language. +auto language_literal(const JSON &value, const JSON::String &language, + const bool none) -> JSON { + auto result{JSON::make_object()}; + result.assign_assume_new(JSON::String{KEYWORD_VALUE}, JSON{value}, + KEYWORD_VALUE_HASH); + if (!none) { + result.assign_assume_new(JSON::String{KEYWORD_LANGUAGE}, JSON{language}, + KEYWORD_LANGUAGE_HASH); + } + return result; +} + +auto build_language_collection(const JSON &value) -> JSON { + auto elements{JSON::make_array()}; + for (const auto key : sorted_keys(value)) { + const auto &member{value.at(key.get())}; + const bool none{key.get() == KEYWORD_NONE}; + if (member.is_array()) { + for (const auto &element : member.as_array()) { + assert(element.is_string()); + elements.push_back(language_literal(element, key.get(), none)); + } + } else { + assert(member.is_string()); + elements.push_back(language_literal(member, key.get(), none)); + } + } + return elements; +} + +// The index keys carry no RDF and are dropped. +template +auto build_index_collection(const JSON &value, PointerT &pointer, + const JSONLDBasicAnnotationMap &map, + std::vector &standalone) -> JSON { + auto elements{JSON::make_array()}; + for (const auto key : sorted_keys(value)) { + push_property(pointer, key.get()); + auto element{ + materialize_value(value.at(key.get()), pointer, map, standalone)}; + pointer.pop_back(); + if (!element.has_value()) { + continue; + } + + // A nested set flattens into the enclosing collection. + if (element->is_array()) { + for (auto &nested : element->as_array()) { + elements.push_back(std::move(nested)); + } + } else { + elements.push_back(std::move(element.value())); + } + } + return elements; +} + template auto materialize_node(const JSONLDNode &descriptor, const JSON &value, PointerT &pointer, @@ -272,10 +346,27 @@ auto materialize_value(const JSON &value, PointerT &pointer, } const auto &collection{std::get(descriptor.value)}; - if (!value.is_array()) { - return std::nullopt; + switch (collection.container) { + case JSONLDContainer::List: + case JSONLDContainer::Set: + if (!value.is_array()) { + return std::nullopt; + } + return build_collection(value, pointer, map, standalone, + collection.container == JSONLDContainer::List); + case JSONLDContainer::Language: + assert(value.is_object()); + // A language-tagged literal cannot be the object of a reverse property. + assert(std::ranges::none_of( + descriptor.edges, + [](const JSONLDEdge &edge) -> bool { return edge.reverse; })); + return build_language_collection(value); + case JSONLDContainer::Index: + assert(value.is_object()); + return build_index_collection(value, pointer, map, standalone); } - return build_collection(value, pointer, map, standalone, collection.ordered); + + std::unreachable(); } template diff --git a/vendor/core/src/core/jsonpointer/CMakeLists.txt b/vendor/core/src/core/jsonpointer/CMakeLists.txt index 0f89f3d78..8d169617c 100644 --- a/vendor/core/src/core/jsonpointer/CMakeLists.txt +++ b/vendor/core/src/core/jsonpointer/CMakeLists.txt @@ -13,3 +13,5 @@ target_link_libraries(sourcemeta_core_jsonpointer PUBLIC sourcemeta::core::json) target_link_libraries(sourcemeta_core_jsonpointer PUBLIC sourcemeta::core::regex) +target_link_libraries(sourcemeta_core_jsonpointer PRIVATE + sourcemeta::core::text) diff --git a/vendor/core/src/core/jsonpointer/jsonpointer.cc b/vendor/core/src/core/jsonpointer/jsonpointer.cc index 21eb6a1aa..3ca1a22e8 100644 --- a/vendor/core/src/core/jsonpointer/jsonpointer.cc +++ b/vendor/core/src/core/jsonpointer/jsonpointer.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "parser.h" @@ -25,18 +26,6 @@ namespace { -auto uri_hex_value(const char character) -> int { - if (character >= '0' && character <= '9') { - return character - '0'; - } else if (character >= 'A' && character <= 'F') { - return character - 'A' + 10; - } else if (character >= 'a' && character <= 'f') { - return character - 'a' + 10; - } else { - return -1; - } -} - template