winnow/token/
mod.rs

1//! Parsers extracting tokens from the stream
2
3#[cfg(test)]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::ErrMode;
9use crate::error::ErrorKind;
10use crate::error::Needed;
11use crate::error::ParserError;
12use crate::lib::std::result::Result::Ok;
13use crate::stream::Range;
14use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, SliceLen, Stream};
15use crate::stream::{StreamIsPartial, ToUsize};
16use crate::PResult;
17use crate::Parser;
18
19/// Matches one token
20///
21/// *Complete version*: Will return an error if there's not enough input data.
22///
23/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
24///
25/// # Effective Signature
26///
27/// Assuming you are parsing a `&str` [Stream]:
28/// ```rust
29/// # use winnow::prelude::*;;
30/// pub fn any(input: &mut &str) -> PResult<char>
31/// # {
32/// #     winnow::token::any.parse_next(input)
33/// # }
34/// ```
35///
36/// # Example
37///
38/// ```rust
39/// # use winnow::{token::any, error::ErrMode, error::{InputError, ErrorKind}};
40/// # use winnow::prelude::*;
41/// fn parser(input: &str) -> IResult<&str, char> {
42///     any.parse_peek(input)
43/// }
44///
45/// assert_eq!(parser("abc"), Ok(("bc",'a')));
46/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
47/// ```
48///
49/// ```rust
50/// # use winnow::{token::any, error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
51/// # use winnow::prelude::*;
52/// # use winnow::Partial;
53/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
54/// assert_eq!(any::<_, InputError<_>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
55/// ```
56#[inline(always)]
57#[doc(alias = "token")]
58pub fn any<Input, Error>(input: &mut Input) -> PResult<<Input as Stream>::Token, Error>
59where
60    Input: StreamIsPartial + Stream,
61    Error: ParserError<Input>,
62{
63    trace("any", move |input: &mut Input| {
64        if <Input as StreamIsPartial>::is_partial_supported() {
65            any_::<_, _, true>(input)
66        } else {
67            any_::<_, _, false>(input)
68        }
69    })
70    .parse_next(input)
71}
72
73fn any_<I, E: ParserError<I>, const PARTIAL: bool>(
74    input: &mut I,
75) -> PResult<<I as Stream>::Token, E>
76where
77    I: StreamIsPartial,
78    I: Stream,
79{
80    input.next_token().ok_or_else(|| {
81        if PARTIAL && input.is_partial() {
82            ErrMode::Incomplete(Needed::new(1))
83        } else {
84            ErrMode::from_error_kind(input, ErrorKind::Token)
85        }
86    })
87}
88
89/// Recognizes a literal
90///
91/// The input data will be compared to the literal combinator's argument and will return the part of
92/// the input that matches the argument
93///
94/// It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Tag)))` if the input doesn't match the literal
95///
96/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
97/// only)
98///
99/// # Effective Signature
100///
101/// Assuming you are parsing a `&str` [Stream]:
102/// ```rust
103/// # use winnow::prelude::*;;
104/// # use winnow::error::ContextError;
105/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
106/// # {
107/// #     winnow::token::literal(literal)
108/// # }
109/// ```
110///
111/// # Example
112/// ```rust
113/// # use winnow::prelude::*;
114/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
115/// #
116/// fn parser(s: &str) -> IResult<&str, &str> {
117///   "Hello".parse_peek(s)
118/// }
119///
120/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
121/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
122/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
123/// ```
124///
125/// ```rust
126/// # use winnow::prelude::*;
127/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
128/// # use winnow::Partial;
129///
130/// fn parser(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
131///   "Hello".parse_peek(s)
132/// }
133///
134/// assert_eq!(parser(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
135/// assert_eq!(parser(Partial::new("Something")), Err(ErrMode::Backtrack(InputError::new(Partial::new("Something"), ErrorKind::Tag))));
136/// assert_eq!(parser(Partial::new("S")), Err(ErrMode::Backtrack(InputError::new(Partial::new("S"), ErrorKind::Tag))));
137/// assert_eq!(parser(Partial::new("H")), Err(ErrMode::Incomplete(Needed::new(4))));
138/// ```
139///
140/// ```rust
141/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
142/// # use winnow::prelude::*;
143/// use winnow::token::literal;
144/// use winnow::ascii::Caseless;
145///
146/// fn parser(s: &str) -> IResult<&str, &str> {
147///   literal(Caseless("hello")).parse_peek(s)
148/// }
149///
150/// assert_eq!(parser("Hello, World!"), Ok((", World!", "Hello")));
151/// assert_eq!(parser("hello, World!"), Ok((", World!", "hello")));
152/// assert_eq!(parser("HeLlO, World!"), Ok((", World!", "HeLlO")));
153/// assert_eq!(parser("Something"), Err(ErrMode::Backtrack(InputError::new("Something", ErrorKind::Tag))));
154/// assert_eq!(parser(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Tag))));
155/// ```
156#[inline(always)]
157#[doc(alias = "tag")]
158#[doc(alias = "bytes")]
159#[doc(alias = "just")]
160pub fn literal<Literal, Input, Error>(
161    literal: Literal,
162) -> impl Parser<Input, <Input as Stream>::Slice, Error>
163where
164    Input: StreamIsPartial + Stream + Compare<Literal>,
165    Literal: SliceLen + Clone + crate::lib::std::fmt::Debug,
166    Error: ParserError<Input>,
167{
168    trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
169        let t = literal.clone();
170        if <Input as StreamIsPartial>::is_partial_supported() {
171            literal_::<_, _, _, true>(i, t)
172        } else {
173            literal_::<_, _, _, false>(i, t)
174        }
175    })
176}
177
178fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
179    i: &mut I,
180    t: T,
181) -> PResult<<I as Stream>::Slice, Error>
182where
183    I: StreamIsPartial,
184    I: Stream + Compare<T>,
185    T: SliceLen + crate::lib::std::fmt::Debug,
186{
187    let literal_len = t.slice_len();
188    match i.compare(t) {
189        CompareResult::Ok(len) => Ok(i.next_slice(len)),
190        CompareResult::Incomplete if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(
191            Needed::new(literal_len - i.eof_offset()),
192        )),
193        CompareResult::Incomplete | CompareResult::Error => {
194            let e: ErrorKind = ErrorKind::Tag;
195            Err(ErrMode::from_error_kind(i, e))
196        }
197    }
198}
199
200/// Recognize a token that matches a [set of tokens][ContainsToken]
201///
202/// **Note:** [`Parser`] is implemented as a convenience (complete
203/// only) for
204/// - `u8`
205/// - `char`
206///
207/// *Complete version*: Will return an error if there's not enough input data.
208///
209/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
210///
211/// # Effective Signature
212///
213/// Assuming you are parsing a `&str` [Stream]:
214/// ```rust
215/// # use winnow::prelude::*;;
216/// # use winnow::stream::ContainsToken;
217/// # use winnow::error::ContextError;
218/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
219/// # {
220/// #     winnow::token::one_of(set)
221/// # }
222/// ```
223///
224/// # Example
225///
226/// ```rust
227/// # use winnow::prelude::*;
228/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
229/// # use winnow::token::one_of;
230/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
231/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek("bc"), Err(ErrMode::Backtrack(InputError::new("bc", ErrorKind::Verify))));
232/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
233///
234/// fn parser_fn(i: &str) -> IResult<&str, char> {
235///     one_of(|c| c == 'a' || c == 'b').parse_peek(i)
236/// }
237/// assert_eq!(parser_fn("abc"), Ok(("bc", 'a')));
238/// assert_eq!(parser_fn("cd"), Err(ErrMode::Backtrack(InputError::new("cd", ErrorKind::Verify))));
239/// assert_eq!(parser_fn(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
240/// ```
241///
242/// ```
243/// # use winnow::prelude::*;
244/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
245/// # use winnow::Partial;
246/// # use winnow::token::one_of;
247/// assert_eq!(one_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
248/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("bc")), Err(ErrMode::Backtrack(InputError::new(Partial::new("bc"), ErrorKind::Verify))));
249/// assert_eq!(one_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
250///
251/// fn parser_fn(i: Partial<&str>) -> IResult<Partial<&str>, char> {
252///     one_of(|c| c == 'a' || c == 'b').parse_peek(i)
253/// }
254/// assert_eq!(parser_fn(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
255/// assert_eq!(parser_fn(Partial::new("cd")), Err(ErrMode::Backtrack(InputError::new(Partial::new("cd"), ErrorKind::Verify))));
256/// assert_eq!(parser_fn(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
257/// ```
258#[inline(always)]
259#[doc(alias = "char")]
260#[doc(alias = "token")]
261#[doc(alias = "satisfy")]
262pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
263where
264    Input: StreamIsPartial + Stream,
265    <Input as Stream>::Token: Clone,
266    Set: ContainsToken<<Input as Stream>::Token>,
267    Error: ParserError<Input>,
268{
269    trace(
270        "one_of",
271        any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
272    )
273}
274
275/// Recognize a token that does not match a [set of tokens][ContainsToken]
276///
277/// *Complete version*: Will return an error if there's not enough input data.
278///
279/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
280///
281/// # Effective Signature
282///
283/// Assuming you are parsing a `&str` [Stream]:
284/// ```rust
285/// # use winnow::prelude::*;;
286/// # use winnow::stream::ContainsToken;
287/// # use winnow::error::ContextError;
288/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
289/// # {
290/// #     winnow::token::none_of(set)
291/// # }
292/// ```
293///
294/// # Example
295///
296/// ```rust
297/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError};
298/// # use winnow::prelude::*;
299/// # use winnow::token::none_of;
300/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
301/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek("a"), Err(ErrMode::Backtrack(InputError::new("a", ErrorKind::Verify))));
302/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Token))));
303/// ```
304///
305/// ```
306/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
307/// # use winnow::prelude::*;
308/// # use winnow::Partial;
309/// # use winnow::token::none_of;
310/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
311/// assert_eq!(none_of::<_, _, InputError<_>>(['a', 'b']).parse_peek(Partial::new("a")), Err(ErrMode::Backtrack(InputError::new(Partial::new("a"), ErrorKind::Verify))));
312/// assert_eq!(none_of::<_, _, InputError<_>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
313/// ```
314#[inline(always)]
315pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
316where
317    Input: StreamIsPartial + Stream,
318    <Input as Stream>::Token: Clone,
319    Set: ContainsToken<<Input as Stream>::Token>,
320    Error: ParserError<Input>,
321{
322    trace(
323        "none_of",
324        any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
325    )
326}
327
328/// Recognize the longest (m <= len <= n) input slice that matches a [set of tokens][ContainsToken]
329///
330/// It will return an `ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice))` if the set of tokens wasn't met or is out
331/// of range (m <= len <= n).
332///
333/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
334///
335/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
336///
337/// # Effective Signature
338///
339/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
340/// ```rust
341/// # use std::ops::RangeFrom;
342/// # use winnow::prelude::*;
343/// # use winnow::stream::ContainsToken;
344/// # use winnow::error::ContextError;
345/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
346/// # {
347/// #     winnow::token::take_while(occurrences, set)
348/// # }
349/// ```
350///
351/// # Example
352///
353/// Zero or more tokens:
354/// ```rust
355/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
356/// # use winnow::prelude::*;
357/// use winnow::token::take_while;
358/// use winnow::stream::AsChar;
359///
360/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
361///   take_while(0.., AsChar::is_alpha).parse_peek(s)
362/// }
363///
364/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
365/// assert_eq!(alpha(b"12345"), Ok((&b"12345"[..], &b""[..])));
366/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
367/// assert_eq!(alpha(b""), Ok((&b""[..], &b""[..])));
368/// ```
369///
370/// ```rust
371/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
372/// # use winnow::prelude::*;
373/// # use winnow::Partial;
374/// use winnow::token::take_while;
375/// use winnow::stream::AsChar;
376///
377/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
378///   take_while(0.., AsChar::is_alpha).parse_peek(s)
379/// }
380///
381/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
382/// assert_eq!(alpha(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
383/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
384/// assert_eq!(alpha(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
385/// ```
386///
387/// One or more tokens:
388/// ```rust
389/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
390/// # use winnow::prelude::*;
391/// use winnow::token::take_while;
392/// use winnow::stream::AsChar;
393///
394/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
395///   take_while(1.., AsChar::is_alpha).parse_peek(s)
396/// }
397///
398/// assert_eq!(alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
399/// assert_eq!(alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
400/// assert_eq!(alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
401///
402/// fn hex(s: &str) -> IResult<&str, &str> {
403///   take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
404/// }
405///
406/// assert_eq!(hex("123 and voila"), Ok((" and voila", "123")));
407/// assert_eq!(hex("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
408/// assert_eq!(hex("BADBABEsomething"), Ok(("something", "BADBABE")));
409/// assert_eq!(hex("D15EA5E"), Ok(("", "D15EA5E")));
410/// assert_eq!(hex(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
411/// ```
412///
413/// ```rust
414/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
415/// # use winnow::prelude::*;
416/// # use winnow::Partial;
417/// use winnow::token::take_while;
418/// use winnow::stream::AsChar;
419///
420/// fn alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
421///   take_while(1.., AsChar::is_alpha).parse_peek(s)
422/// }
423///
424/// assert_eq!(alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
425/// assert_eq!(alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
426/// assert_eq!(alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
427///
428/// fn hex(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
429///   take_while(1.., ('0'..='9', 'A'..='F')).parse_peek(s)
430/// }
431///
432/// assert_eq!(hex(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
433/// assert_eq!(hex(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
434/// assert_eq!(hex(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
435/// assert_eq!(hex(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
436/// assert_eq!(hex(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
437/// ```
438///
439/// Arbitrary amount of tokens:
440/// ```rust
441/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
442/// # use winnow::prelude::*;
443/// use winnow::token::take_while;
444/// use winnow::stream::AsChar;
445///
446/// fn short_alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {
447///   take_while(3..=6, AsChar::is_alpha).parse_peek(s)
448/// }
449///
450/// assert_eq!(short_alpha(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
451/// assert_eq!(short_alpha(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
452/// assert_eq!(short_alpha(b"latin"), Ok((&b""[..], &b"latin"[..])));
453/// assert_eq!(short_alpha(b"ed"), Err(ErrMode::Backtrack(InputError::new(&b"ed"[..], ErrorKind::Slice))));
454/// assert_eq!(short_alpha(b"12345"), Err(ErrMode::Backtrack(InputError::new(&b"12345"[..], ErrorKind::Slice))));
455/// ```
456///
457/// ```rust
458/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
459/// # use winnow::prelude::*;
460/// # use winnow::Partial;
461/// use winnow::token::take_while;
462/// use winnow::stream::AsChar;
463///
464/// fn short_alpha(s: Partial<&[u8]>) -> IResult<Partial<&[u8]>, &[u8]> {
465///   take_while(3..=6, AsChar::is_alpha).parse_peek(s)
466/// }
467///
468/// assert_eq!(short_alpha(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
469/// assert_eq!(short_alpha(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
470/// assert_eq!(short_alpha(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
471/// assert_eq!(short_alpha(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
472/// assert_eq!(short_alpha(Partial::new(b"12345")), Err(ErrMode::Backtrack(InputError::new(Partial::new(&b"12345"[..]), ErrorKind::Slice))));
473/// ```
474#[inline(always)]
475#[doc(alias = "is_a")]
476#[doc(alias = "take_while0")]
477#[doc(alias = "take_while1")]
478pub fn take_while<Set, Input, Error>(
479    occurrences: impl Into<Range>,
480    set: Set,
481) -> impl Parser<Input, <Input as Stream>::Slice, Error>
482where
483    Input: StreamIsPartial + Stream,
484    Set: ContainsToken<<Input as Stream>::Token>,
485    Error: ParserError<Input>,
486{
487    let Range {
488        start_inclusive,
489        end_inclusive,
490    } = occurrences.into();
491    trace("take_while", move |i: &mut Input| {
492        match (start_inclusive, end_inclusive) {
493            (0, None) => {
494                if <Input as StreamIsPartial>::is_partial_supported() {
495                    take_while0_::<_, _, _, true>(i, &set)
496                } else {
497                    take_while0_::<_, _, _, false>(i, &set)
498                }
499            }
500            (1, None) => {
501                if <Input as StreamIsPartial>::is_partial_supported() {
502                    take_while1_::<_, _, _, true>(i, &set)
503                } else {
504                    take_while1_::<_, _, _, false>(i, &set)
505                }
506            }
507            (start, end) => {
508                let end = end.unwrap_or(usize::MAX);
509                if <Input as StreamIsPartial>::is_partial_supported() {
510                    take_while_m_n_::<_, _, _, true>(i, start, end, &set)
511                } else {
512                    take_while_m_n_::<_, _, _, false>(i, start, end, &set)
513                }
514            }
515        }
516    })
517}
518
519fn take_while0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
520    input: &mut I,
521    list: &T,
522) -> PResult<<I as Stream>::Slice, Error>
523where
524    I: StreamIsPartial,
525    I: Stream,
526    T: ContainsToken<<I as Stream>::Token>,
527{
528    if PARTIAL && input.is_partial() {
529        take_till0_partial(input, |c| !list.contains_token(c))
530    } else {
531        take_till0_complete(input, |c| !list.contains_token(c))
532    }
533}
534
535fn take_while1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
536    input: &mut I,
537    list: &T,
538) -> PResult<<I as Stream>::Slice, Error>
539where
540    I: StreamIsPartial,
541    I: Stream,
542    T: ContainsToken<<I as Stream>::Token>,
543{
544    if PARTIAL && input.is_partial() {
545        take_till1_partial(input, |c| !list.contains_token(c))
546    } else {
547        take_till1_complete(input, |c| !list.contains_token(c))
548    }
549}
550
551fn take_while_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
552    input: &mut I,
553    m: usize,
554    n: usize,
555    list: &T,
556) -> PResult<<I as Stream>::Slice, Error>
557where
558    I: StreamIsPartial,
559    I: Stream,
560    T: ContainsToken<<I as Stream>::Token>,
561{
562    take_till_m_n::<_, _, _, PARTIAL>(input, m, n, |c| !list.contains_token(c))
563}
564
565fn take_till0_partial<P, I: Stream, E: ParserError<I>>(
566    input: &mut I,
567    predicate: P,
568) -> PResult<<I as Stream>::Slice, E>
569where
570    P: Fn(I::Token) -> bool,
571{
572    let offset = input
573        .offset_for(predicate)
574        .ok_or_else(|| ErrMode::Incomplete(Needed::new(1)))?;
575    Ok(input.next_slice(offset))
576}
577
578fn take_till1_partial<P, I: Stream, E: ParserError<I>>(
579    input: &mut I,
580    predicate: P,
581) -> PResult<<I as Stream>::Slice, E>
582where
583    P: Fn(I::Token) -> bool,
584{
585    let e: ErrorKind = ErrorKind::Slice;
586    let offset = input
587        .offset_for(predicate)
588        .ok_or_else(|| ErrMode::Incomplete(Needed::new(1)))?;
589    if offset == 0 {
590        Err(ErrMode::from_error_kind(input, e))
591    } else {
592        Ok(input.next_slice(offset))
593    }
594}
595
596fn take_till0_complete<P, I: Stream, E: ParserError<I>>(
597    input: &mut I,
598    predicate: P,
599) -> PResult<<I as Stream>::Slice, E>
600where
601    P: Fn(I::Token) -> bool,
602{
603    let offset = input
604        .offset_for(predicate)
605        .unwrap_or_else(|| input.eof_offset());
606    Ok(input.next_slice(offset))
607}
608
609fn take_till1_complete<P, I: Stream, E: ParserError<I>>(
610    input: &mut I,
611    predicate: P,
612) -> PResult<<I as Stream>::Slice, E>
613where
614    P: Fn(I::Token) -> bool,
615{
616    let e: ErrorKind = ErrorKind::Slice;
617    let offset = input
618        .offset_for(predicate)
619        .unwrap_or_else(|| input.eof_offset());
620    if offset == 0 {
621        Err(ErrMode::from_error_kind(input, e))
622    } else {
623        Ok(input.next_slice(offset))
624    }
625}
626
627fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
628    input: &mut I,
629    m: usize,
630    n: usize,
631    predicate: P,
632) -> PResult<<I as Stream>::Slice, Error>
633where
634    I: StreamIsPartial,
635    I: Stream,
636    P: Fn(I::Token) -> bool,
637{
638    if n < m {
639        return Err(ErrMode::assert(
640            input,
641            "`occurrences` should be ascending, rather than descending",
642        ));
643    }
644
645    let mut final_count = 0;
646    for (processed, (offset, token)) in input.iter_offsets().enumerate() {
647        if predicate(token) {
648            if processed < m {
649                return Err(ErrMode::from_error_kind(input, ErrorKind::Slice));
650            } else {
651                return Ok(input.next_slice(offset));
652            }
653        } else {
654            if processed == n {
655                return Ok(input.next_slice(offset));
656            }
657            final_count = processed + 1;
658        }
659    }
660    if PARTIAL && input.is_partial() {
661        if final_count == n {
662            Ok(input.finish())
663        } else {
664            let needed = if m > input.eof_offset() {
665                m - input.eof_offset()
666            } else {
667                1
668            };
669            Err(ErrMode::Incomplete(Needed::new(needed)))
670        }
671    } else {
672        if m <= final_count {
673            Ok(input.finish())
674        } else {
675            Err(ErrMode::from_error_kind(input, ErrorKind::Slice))
676        }
677    }
678}
679
680/// Recognize the longest input slice (if any) till a member of a [set of tokens][ContainsToken] is found.
681///
682/// It doesn't consume the terminating token from the set.
683///
684/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
685/// end of input or if there was not match.
686///
687/// See also
688/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
689/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
690///
691/// # Effective Signature
692///
693/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
694/// ```rust
695/// # use std::ops::RangeFrom;
696/// # use winnow::prelude::*;
697/// # use winnow::stream::ContainsToken;
698/// # use winnow::error::ContextError;
699/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
700/// # {
701/// #     winnow::token::take_till(occurrences, set)
702/// # }
703/// ```
704///
705/// # Example
706///
707/// ```rust
708/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
709/// # use winnow::prelude::*;
710/// use winnow::token::take_till;
711///
712/// fn till_colon(s: &str) -> IResult<&str, &str> {
713///   take_till(0.., |c| c == ':').parse_peek(s)
714/// }
715///
716/// assert_eq!(till_colon("latin:123"), Ok((":123", "latin")));
717/// assert_eq!(till_colon(":empty matched"), Ok((":empty matched", ""))); //allowed
718/// assert_eq!(till_colon("12345"), Ok(("", "12345")));
719/// assert_eq!(till_colon(""), Ok(("", "")));
720/// ```
721///
722/// ```rust
723/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
724/// # use winnow::prelude::*;
725/// # use winnow::Partial;
726/// use winnow::token::take_till;
727///
728/// fn till_colon(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
729///   take_till(0.., |c| c == ':').parse_peek(s)
730/// }
731///
732/// assert_eq!(till_colon(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
733/// assert_eq!(till_colon(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
734/// assert_eq!(till_colon(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
735/// assert_eq!(till_colon(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
736/// ```
737#[inline(always)]
738#[doc(alias = "is_not")]
739pub fn take_till<Set, Input, Error>(
740    occurrences: impl Into<Range>,
741    set: Set,
742) -> impl Parser<Input, <Input as Stream>::Slice, Error>
743where
744    Input: StreamIsPartial + Stream,
745    Set: ContainsToken<<Input as Stream>::Token>,
746    Error: ParserError<Input>,
747{
748    let Range {
749        start_inclusive,
750        end_inclusive,
751    } = occurrences.into();
752    trace("take_till", move |i: &mut Input| {
753        match (start_inclusive, end_inclusive) {
754            (0, None) => {
755                if <Input as StreamIsPartial>::is_partial_supported() {
756                    take_till0_partial(i, |c| set.contains_token(c))
757                } else {
758                    take_till0_complete(i, |c| set.contains_token(c))
759                }
760            }
761            (1, None) => {
762                if <Input as StreamIsPartial>::is_partial_supported() {
763                    take_till1_partial(i, |c| set.contains_token(c))
764                } else {
765                    take_till1_complete(i, |c| set.contains_token(c))
766                }
767            }
768            (start, end) => {
769                let end = end.unwrap_or(usize::MAX);
770                if <Input as StreamIsPartial>::is_partial_supported() {
771                    take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
772                } else {
773                    take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
774                }
775            }
776        }
777    })
778}
779
780/// Recognize an input slice containing the first N input elements (I[..N]).
781///
782/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))` if the input is shorter than the argument.
783///
784/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
785/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
786/// additional bytes the parser would need to succeed.
787/// It is well defined for `&[u8]` as the number of elements is the byte size,
788/// but for types like `&str`, we cannot know how many bytes correspond for
789/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
790///
791/// # Effective Signature
792///
793/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
794/// ```rust
795/// # use std::ops::RangeFrom;
796/// # use winnow::prelude::*;
797/// # use winnow::stream::ContainsToken;
798/// # use winnow::error::ContextError;
799/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
800/// # {
801/// #     winnow::token::take(token_count)
802/// # }
803/// ```
804///
805/// # Example
806///
807/// ```rust
808/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
809/// # use winnow::prelude::*;
810/// use winnow::token::take;
811///
812/// fn take6(s: &str) -> IResult<&str, &str> {
813///   take(6usize).parse_peek(s)
814/// }
815///
816/// assert_eq!(take6("1234567"), Ok(("7", "123456")));
817/// assert_eq!(take6("things"), Ok(("", "things")));
818/// assert_eq!(take6("short"), Err(ErrMode::Backtrack(InputError::new("short", ErrorKind::Slice))));
819/// assert_eq!(take6(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
820/// ```
821///
822/// The units that are taken will depend on the input type. For example, for a
823/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
824/// take that many `u8`'s:
825///
826/// ```rust
827/// # use winnow::prelude::*;
828/// use winnow::error::InputError;
829/// use winnow::token::take;
830///
831/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙"), Ok(("", "💙")));
832/// assert_eq!(take::<_, _, InputError<_>>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
833/// ```
834///
835/// ```rust
836/// # use winnow::prelude::*;
837/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
838/// # use winnow::Partial;
839/// use winnow::token::take;
840///
841/// fn take6(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
842///   take(6usize).parse_peek(s)
843/// }
844///
845/// assert_eq!(take6(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
846/// assert_eq!(take6(Partial::new("things")), Ok((Partial::new(""), "things")));
847/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
848/// assert_eq!(take6(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
849/// ```
850#[inline(always)]
851pub fn take<UsizeLike, Input, Error>(
852    token_count: UsizeLike,
853) -> impl Parser<Input, <Input as Stream>::Slice, Error>
854where
855    Input: StreamIsPartial + Stream,
856    UsizeLike: ToUsize,
857    Error: ParserError<Input>,
858{
859    let c = token_count.to_usize();
860    trace("take", move |i: &mut Input| {
861        if <Input as StreamIsPartial>::is_partial_supported() {
862            take_::<_, _, true>(i, c)
863        } else {
864            take_::<_, _, false>(i, c)
865        }
866    })
867}
868
869fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
870    i: &mut I,
871    c: usize,
872) -> PResult<<I as Stream>::Slice, Error>
873where
874    I: StreamIsPartial,
875    I: Stream,
876{
877    match i.offset_at(c) {
878        Ok(offset) => Ok(i.next_slice(offset)),
879        Err(e) if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(e)),
880        Err(_needed) => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
881    }
882}
883
884/// Recognize the input slice up to the first occurrence of a [literal].
885///
886/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
887///
888/// It doesn't consume the literal.
889///
890/// *Complete version*: It will return `Err(ErrMode::Backtrack(InputError::new(_, ErrorKind::Slice)))`
891/// if the literal wasn't met.
892///
893/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
894/// contain the literal or if the input is smaller than the literal.
895///
896/// See also
897/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
898/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
899///
900/// # Effective Signature
901///
902/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
903/// ```rust
904/// # use std::ops::RangeFrom;
905/// # use winnow::prelude::*;;
906/// # use winnow::error::ContextError;
907/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
908/// # {
909/// #     winnow::token::take_until(occurrences, literal)
910/// # }
911/// ```
912///
913/// # Example
914///
915/// ```rust
916/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
917/// # use winnow::prelude::*;
918/// use winnow::token::take_until;
919///
920/// fn until_eof(s: &str) -> IResult<&str, &str> {
921///   take_until(0.., "eof").parse_peek(s)
922/// }
923///
924/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
925/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
926/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
927/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
928/// ```
929///
930/// ```rust
931/// # use winnow::{error::ErrMode, error::ErrorKind, error::InputError, error::Needed};
932/// # use winnow::prelude::*;
933/// # use winnow::Partial;
934/// use winnow::token::take_until;
935///
936/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
937///   take_until(0.., "eof").parse_peek(s)
938/// }
939///
940/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
941/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
942/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
943/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
944/// ```
945///
946/// ```rust
947/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
948/// # use winnow::prelude::*;
949/// use winnow::token::take_until;
950///
951/// fn until_eof(s: &str) -> IResult<&str, &str> {
952///   take_until(1.., "eof").parse_peek(s)
953/// }
954///
955/// assert_eq!(until_eof("hello, worldeof"), Ok(("eof", "hello, world")));
956/// assert_eq!(until_eof("hello, world"), Err(ErrMode::Backtrack(InputError::new("hello, world", ErrorKind::Slice))));
957/// assert_eq!(until_eof(""), Err(ErrMode::Backtrack(InputError::new("", ErrorKind::Slice))));
958/// assert_eq!(until_eof("1eof2eof"), Ok(("eof2eof", "1")));
959/// assert_eq!(until_eof("eof"), Err(ErrMode::Backtrack(InputError::new("eof", ErrorKind::Slice))));
960/// ```
961///
962/// ```rust
963/// # use winnow::{error::ErrMode, error::{InputError, ErrorKind}, error::Needed};
964/// # use winnow::prelude::*;
965/// # use winnow::Partial;
966/// use winnow::token::take_until;
967///
968/// fn until_eof(s: Partial<&str>) -> IResult<Partial<&str>, &str> {
969///   take_until(1.., "eof").parse_peek(s)
970/// }
971///
972/// assert_eq!(until_eof(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
973/// assert_eq!(until_eof(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
974/// assert_eq!(until_eof(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
975/// assert_eq!(until_eof(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
976/// assert_eq!(until_eof(Partial::new("eof")), Err(ErrMode::Backtrack(InputError::new(Partial::new("eof"), ErrorKind::Slice))));
977/// ```
978#[inline(always)]
979pub fn take_until<Literal, Input, Error>(
980    occurrences: impl Into<Range>,
981    literal: Literal,
982) -> impl Parser<Input, <Input as Stream>::Slice, Error>
983where
984    Input: StreamIsPartial + Stream + FindSlice<Literal>,
985    Literal: Clone,
986    Error: ParserError<Input>,
987{
988    let Range {
989        start_inclusive,
990        end_inclusive,
991    } = occurrences.into();
992    trace("take_until", move |i: &mut Input| {
993        match (start_inclusive, end_inclusive) {
994            (0, None) => {
995                if <Input as StreamIsPartial>::is_partial_supported() {
996                    take_until0_::<_, _, _, true>(i, literal.clone())
997                } else {
998                    take_until0_::<_, _, _, false>(i, literal.clone())
999                }
1000            }
1001            (1, None) => {
1002                if <Input as StreamIsPartial>::is_partial_supported() {
1003                    take_until1_::<_, _, _, true>(i, literal.clone())
1004                } else {
1005                    take_until1_::<_, _, _, false>(i, literal.clone())
1006                }
1007            }
1008            (start, end) => {
1009                let end = end.unwrap_or(usize::MAX);
1010                if <Input as StreamIsPartial>::is_partial_supported() {
1011                    take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
1012                } else {
1013                    take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
1014                }
1015            }
1016        }
1017    })
1018}
1019
1020fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
1021    i: &mut I,
1022    t: T,
1023) -> PResult<<I as Stream>::Slice, Error>
1024where
1025    I: StreamIsPartial,
1026    I: Stream + FindSlice<T>,
1027{
1028    match i.find_slice(t) {
1029        Some(range) => Ok(i.next_slice(range.start)),
1030        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
1031        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
1032    }
1033}
1034
1035fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
1036    i: &mut I,
1037    t: T,
1038) -> PResult<<I as Stream>::Slice, Error>
1039where
1040    I: StreamIsPartial,
1041    I: Stream + FindSlice<T>,
1042{
1043    match i.find_slice(t) {
1044        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
1045        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
1046        Some(range) => {
1047            if range.start == 0 {
1048                Err(ErrMode::from_error_kind(i, ErrorKind::Slice))
1049            } else {
1050                Ok(i.next_slice(range.start))
1051            }
1052        }
1053    }
1054}
1055
1056fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
1057    i: &mut I,
1058    start: usize,
1059    end: usize,
1060    t: T,
1061) -> PResult<<I as Stream>::Slice, Error>
1062where
1063    I: StreamIsPartial,
1064    I: Stream + FindSlice<T>,
1065{
1066    if end < start {
1067        return Err(ErrMode::assert(
1068            i,
1069            "`occurrences` should be ascending, rather than descending",
1070        ));
1071    }
1072
1073    match i.find_slice(t) {
1074        Some(range) => {
1075            let start_offset = i.offset_at(start);
1076            let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1077            if start_offset.map(|s| range.start < s).unwrap_or(true) {
1078                if PARTIAL && i.is_partial() {
1079                    return Err(ErrMode::Incomplete(Needed::Unknown));
1080                } else {
1081                    return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1082                }
1083            }
1084            if end_offset < range.start {
1085                return Err(ErrMode::from_error_kind(i, ErrorKind::Slice));
1086            }
1087            Ok(i.next_slice(range.start))
1088        }
1089        None if PARTIAL && i.is_partial() => Err(ErrMode::Incomplete(Needed::Unknown)),
1090        None => Err(ErrMode::from_error_kind(i, ErrorKind::Slice)),
1091    }
1092}
winnow/token/mod.rs

winnow/token/
mod.rs
