winnow/stream/
token.rs

Help
1use crate::error::Needed;
2use crate::lib::std::iter::Enumerate;
3use crate::lib::std::slice::Iter;
4use crate::stream::Checkpoint;
5use crate::stream::Compare;
6use crate::stream::CompareResult;
7use crate::stream::Location;
8use crate::stream::Offset;
9#[cfg(feature = "unstable-recover")]
10#[cfg(feature = "std")]
11use crate::stream::Recover;
12use crate::stream::SliceLen;
13use crate::stream::Stream;
14use crate::stream::StreamIsPartial;
15use crate::stream::UpdateSlice;
16
17/// Specialized input for parsing lexed tokens
18///
19/// Helpful impls
20/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with
21///   [`literal`][crate::token::literal]
22/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens
23/// - [`ContainsToken`][crate::stream::ContainsToken] for `T` to for parsing with token sets
24/// - [`Location`] for `T` to extract spans from tokens
25///
26/// See also [Lexing and Parsing][crate::_topic::lexing].
27#[derive(Copy, Clone, PartialEq, Eq)]
28pub struct TokenSlice<'t, T> {
29    initial: &'t [T],
30    input: &'t [T],
31}
32
33impl<'t, T> TokenSlice<'t, T>
34where
35    T: crate::lib::std::fmt::Debug + Clone,
36{
37    /// Make a stream to parse tokens
38    #[inline]
39    pub fn new(input: &'t [T]) -> Self {
40        Self {
41            initial: input,
42            input,
43        }
44    }
45
46    /// Reset the stream to the start
47    ///
48    /// This is useful for formats that encode a graph with addresses relative to the start of the
49    /// input.
50    #[doc(alias = "fseek")]
51    #[inline]
52    pub fn reset_to_start(&mut self) {
53        let start = self.initial.checkpoint();
54        self.input.reset(&start);
55    }
56
57    /// Iterate over consumed tokens starting with the last emitted
58    ///
59    /// This is intended to help build up appropriate context when reporting errors.
60    #[inline]
61    pub fn previous_tokens(&self) -> impl Iterator<Item = &'t T> {
62        let offset = self.input.offset_from(&self.initial);
63        self.initial[0..offset].iter().rev()
64    }
65}
66
67/// Track locations by implementing [`Location`] on the Token.
68impl<T> TokenSlice<'_, T>
69where
70    T: Location,
71{
72    #[inline(always)]
73    fn previous_token_end(&self) -> Option<usize> {
74        let index = self.input.offset_from(&self.initial);
75        index
76            .checked_sub(1)
77            .map(|i| self.initial[i].previous_token_end())
78    }
79
80    #[inline(always)]
81    fn current_token_start(&self) -> Option<usize> {
82        self.input.first().map(|t| t.current_token_start())
83    }
84}
85
86impl<T> Default for TokenSlice<'_, T>
87where
88    T: crate::lib::std::fmt::Debug + Clone,
89{
90    fn default() -> Self {
91        Self::new(&[])
92    }
93}
94
95impl<T> crate::lib::std::ops::Deref for TokenSlice<'_, T> {
96    type Target = [T];
97
98    fn deref(&self) -> &Self::Target {
99        self.input
100    }
101}
102
103impl<T: core::fmt::Debug> core::fmt::Debug for TokenSlice<'_, T> {
104    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
105        self.input.fmt(f)
106    }
107}
108
109impl<T> SliceLen for TokenSlice<'_, T> {
110    #[inline(always)]
111    fn slice_len(&self) -> usize {
112        self.input.slice_len()
113    }
114}
115
116impl<'t, T> Stream for TokenSlice<'t, T>
117where
118    T: crate::lib::std::fmt::Debug + Clone,
119{
120    type Token = &'t T;
121    type Slice = &'t [T];
122
123    type IterOffsets = Enumerate<Iter<'t, T>>;
124
125    type Checkpoint = Checkpoint<&'t [T], Self>;
126
127    #[inline(always)]
128    fn iter_offsets(&self) -> Self::IterOffsets {
129        self.input.iter().enumerate()
130    }
131    #[inline(always)]
132    fn eof_offset(&self) -> usize {
133        self.input.eof_offset()
134    }
135
136    #[inline(always)]
137    fn next_token(&mut self) -> Option<Self::Token> {
138        let (token, next) = self.input.split_first()?;
139        self.input = next;
140        Some(token)
141    }
142
143    #[inline(always)]
144    fn peek_token(&self) -> Option<Self::Token> {
145        self.input.first()
146    }
147
148    #[inline(always)]
149    fn offset_for<P>(&self, predicate: P) -> Option<usize>
150    where
151        P: Fn(Self::Token) -> bool,
152    {
153        self.input.iter().position(predicate)
154    }
155    #[inline(always)]
156    fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
157        self.input.offset_at(tokens)
158    }
159    #[inline(always)]
160    fn next_slice(&mut self, offset: usize) -> Self::Slice {
161        self.input.next_slice(offset)
162    }
163    #[inline(always)]
164    unsafe fn next_slice_unchecked(&mut self, offset: usize) -> Self::Slice {
165        // SAFETY: Passing up invariants
166        unsafe { self.input.next_slice_unchecked(offset) }
167    }
168    #[inline(always)]
169    fn peek_slice(&self, offset: usize) -> Self::Slice {
170        self.input.peek_slice(offset)
171    }
172    #[inline(always)]
173    unsafe fn peek_slice_unchecked(&self, offset: usize) -> Self::Slice {
174        // SAFETY: Passing up invariants
175        unsafe { self.input.peek_slice_unchecked(offset) }
176    }
177
178    #[inline(always)]
179    fn checkpoint(&self) -> Self::Checkpoint {
180        Checkpoint::<_, Self>::new(self.input)
181    }
182    #[inline(always)]
183    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
184        self.input = checkpoint.inner;
185    }
186
187    #[inline(always)]
188    fn raw(&self) -> &dyn crate::lib::std::fmt::Debug {
189        #![allow(deprecated)]
190        self.input.raw()
191    }
192
193    fn trace(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
194        self.input.trace(f)
195    }
196}
197
198impl<T> Location for TokenSlice<'_, T>
199where
200    T: Location,
201{
202    #[inline(always)]
203    fn previous_token_end(&self) -> usize {
204        self.previous_token_end()
205            .or_else(|| self.current_token_start())
206            .unwrap_or(0)
207    }
208    #[inline(always)]
209    fn current_token_start(&self) -> usize {
210        self.current_token_start()
211            .or_else(|| self.previous_token_end())
212            .unwrap_or(0)
213    }
214}
215
216#[cfg(feature = "unstable-recover")]
217#[cfg(feature = "std")]
218impl<T, E> Recover<E> for TokenSlice<'_, T>
219where
220    T: crate::lib::std::fmt::Debug + Clone,
221{
222    #[inline(always)]
223    fn record_err(
224        &mut self,
225        _token_start: &Self::Checkpoint,
226        _err_start: &Self::Checkpoint,
227        err: E,
228    ) -> Result<(), E> {
229        Err(err)
230    }
231
232    /// Report whether the [`Stream`] can save off errors for recovery
233    #[inline(always)]
234    fn is_recovery_supported() -> bool {
235        false
236    }
237}
238
239impl<'t, T> StreamIsPartial for TokenSlice<'t, T>
240where
241    T: crate::lib::std::fmt::Debug + Clone,
242{
243    type PartialState = <&'t [T] as StreamIsPartial>::PartialState;
244
245    #[inline]
246    fn complete(&mut self) -> Self::PartialState {
247        #![allow(clippy::semicolon_if_nothing_returned)]
248        self.input.complete()
249    }
250
251    #[inline]
252    fn restore_partial(&mut self, state: Self::PartialState) {
253        self.input.restore_partial(state);
254    }
255
256    #[inline(always)]
257    fn is_partial_supported() -> bool {
258        <&[T] as StreamIsPartial>::is_partial_supported()
259    }
260
261    #[inline(always)]
262    fn is_partial(&self) -> bool {
263        self.input.is_partial()
264    }
265}
266
267impl<T> Offset for TokenSlice<'_, T>
268where
269    T: crate::lib::std::fmt::Debug + Clone,
270{
271    #[inline(always)]
272    fn offset_from(&self, other: &Self) -> usize {
273        self.offset_from(&other.checkpoint())
274    }
275}
276
277impl<T> Offset<<TokenSlice<'_, T> as Stream>::Checkpoint> for TokenSlice<'_, T>
278where
279    T: crate::lib::std::fmt::Debug + Clone,
280{
281    #[inline(always)]
282    fn offset_from(&self, other: &<TokenSlice<'_, T> as Stream>::Checkpoint) -> usize {
283        self.checkpoint().offset_from(other)
284    }
285}
286
287impl<T, O> Compare<O> for TokenSlice<'_, T>
288where
289    T: PartialEq<O> + Eq,
290{
291    #[inline]
292    fn compare(&self, t: O) -> CompareResult {
293        if let Some(token) = self.first() {
294            if *token == t {
295                CompareResult::Ok(1)
296            } else {
297                CompareResult::Error
298            }
299        } else {
300            CompareResult::Incomplete
301        }
302    }
303}
304
305impl<T> UpdateSlice for TokenSlice<'_, T>
306where
307    T: crate::lib::std::fmt::Debug + Clone,
308{
309    #[inline(always)]
310    fn update_slice(mut self, inner: Self::Slice) -> Self {
311        self.input = <&[T] as UpdateSlice>::update_slice(self.input, inner);
312        self
313    }
314}
winnow/stream/token.rs

winnow/stream/
token.rs
