ring/aead/aes_gcm/
aeshwclmulmovbe.rs

1// Copyright 2015-2025 Brian Smith.
2//
3// Permission to use, copy, modify, and/or distribute this software for any
4// purpose with or without fee is hereby granted, provided that the above
5// copyright notice and this permission notice appear in all copies.
6//
7// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15#![cfg(target_arch = "x86_64")]
16
17use super::{
18    super::overlapping::IndexError,
19    aes::{self, Counter, EncryptCtr32, Overlapping, OverlappingPartialBlock},
20    gcm, Aad, Tag,
21};
22use crate::{
23    c,
24    error::{self, InputTooLongError},
25    polyfill::slice,
26};
27use core::ops::RangeFrom;
28
29#[inline(never)]
30pub(super) fn seal(
31    aes_key: &aes::hw::Key,
32    gcm_key: &gcm::clmulavxmovbe::Key,
33    mut ctr: Counter,
34    tag_iv: aes::Iv,
35    aad: Aad<&[u8]>,
36    in_out: &mut [u8],
37) -> Result<Tag, error::Unspecified> {
38    prefixed_extern! {
39        // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
40        // assembly says it needs just nine values in that array.
41        fn aesni_gcm_encrypt(
42            input: *const u8,
43            output: *mut u8,
44            len: c::size_t,
45            key: &aes::AES_KEY,
46            ivec: &mut Counter,
47            Htable: &gcm::HTable,
48            Xi: &mut gcm::Xi) -> c::size_t;
49    }
50
51    let mut auth = gcm::Context::new(gcm_key, aad, in_out.len())?;
52    let (htable, xi) = auth.inner();
53
54    let processed = unsafe {
55        aesni_gcm_encrypt(
56            in_out.as_ptr(),
57            in_out.as_mut_ptr(),
58            in_out.len(),
59            aes_key.inner_less_safe(),
60            &mut ctr,
61            htable,
62            xi,
63        )
64    };
65
66    let ramaining = match in_out.get_mut(processed..) {
67        Some(remaining) => remaining,
68        None => {
69            // This can't happen. If it did, then the assembly already
70            // caused a buffer overflow.
71            unreachable!()
72        }
73    };
74    let (mut whole, remainder) = slice::as_chunks_mut(ramaining);
75    aes_key.ctr32_encrypt_within(whole.as_flattened_mut().into(), &mut ctr);
76    auth.update_blocks(whole.as_ref());
77    let remainder = OverlappingPartialBlock::new(remainder.into())
78        .unwrap_or_else(|InputTooLongError { .. }| unreachable!());
79
80    super::seal_finish(aes_key, auth, remainder, ctr, tag_iv)
81}
82
83#[inline(never)]
84pub(super) fn open(
85    aes_key: &aes::hw::Key,
86    gcm_key: &gcm::clmulavxmovbe::Key,
87    mut ctr: Counter,
88    tag_iv: aes::Iv,
89    aad: Aad<&[u8]>,
90    in_out_slice: &mut [u8],
91    src: RangeFrom<usize>,
92) -> Result<Tag, error::Unspecified> {
93    prefixed_extern! {
94        // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
95        // assembly says it needs just nine values in that array.
96        fn aesni_gcm_decrypt(
97            input: *const u8,
98            output: *mut u8,
99            len: c::size_t,
100            key: &aes::AES_KEY,
101            ivec: &mut Counter,
102            Htable: &gcm::HTable,
103            Xi: &mut gcm::Xi) -> c::size_t;
104    }
105
106    let in_out = Overlapping::new(in_out_slice, src.clone()).map_err(error::erase::<IndexError>)?;
107    let mut auth = gcm::Context::new(gcm_key, aad, in_out.len())?;
108    let processed = in_out.with_input_output_len(|input, output, len| {
109        let (htable, xi) = auth.inner();
110        unsafe {
111            aesni_gcm_decrypt(
112                input,
113                output,
114                len,
115                aes_key.inner_less_safe(),
116                &mut ctr,
117                htable,
118                xi,
119            )
120        }
121    });
122    let in_out_slice = in_out_slice.get_mut(processed..).unwrap_or_else(|| {
123        // This can't happen. If it did, then the assembly already
124        // caused a buffer overflow.
125        unreachable!()
126    });
127    // Authenticate any remaining whole blocks.
128    let in_out =
129        Overlapping::new(in_out_slice, src.clone()).unwrap_or_else(|IndexError { .. }| {
130            // This can't happen. If it did, then the assembly already
131            // overwrote part of the remaining input.
132            unreachable!()
133        });
134    let (whole, _) = slice::as_chunks(in_out.input());
135    auth.update_blocks(whole);
136
137    let whole_len = whole.as_flattened().len();
138
139    // Decrypt any remaining whole blocks.
140    let whole = Overlapping::new(&mut in_out_slice[..(src.start + whole_len)], src.clone())
141        .map_err(error::erase::<IndexError>)?;
142    aes_key.ctr32_encrypt_within(whole, &mut ctr);
143
144    let in_out_slice = match in_out_slice.get_mut(whole_len..) {
145        Some(partial) => partial,
146        None => unreachable!(),
147    };
148    let in_out =
149        Overlapping::new(in_out_slice, src).unwrap_or_else(|IndexError { .. }| unreachable!());
150    let in_out = OverlappingPartialBlock::new(in_out)
151        .unwrap_or_else(|InputTooLongError { .. }| unreachable!());
152
153    super::open_finish(aes_key, auth, in_out, ctr, tag_iv)
154}
ring/aead/aes_gcm/aeshwclmulmovbe.rs

ring/aead/aes_gcm/
aeshwclmulmovbe.rs
