stix_pattern/
lexer.rs

1//! Lexer: converts a pattern string into a flat token stream.
2
3use crate::error::{ParseError, Result, Span};
4
5/// A lexed token: its kind plus the source span it was read from.
6#[derive(Debug, Clone, PartialEq)]
7pub struct Token {
8    /// What kind of token this is (and any literal payload).
9    pub kind: TokenKind,
10    /// The byte range in the source the token was read from.
11    pub span: Span,
12}
13
14/// The kind of a lexed token.
15#[derive(Debug, Clone, PartialEq)]
16pub enum TokenKind {
17    // Punctuation
18    /// `[`
19    LBracket,
20    /// `]`
21    RBracket,
22    /// `(`
23    LParen,
24    /// `)`
25    RParen,
26    /// `:`
27    Colon,
28    /// `.`
29    Dot,
30    /// `,`
31    Comma,
32    /// `*`
33    Star,
34    // Comparison operators
35    /// `=`
36    Equal,
37    /// `!=` or `<>`
38    NotEqual,
39    /// `<`
40    LessThan,
41    /// `<=`
42    LessThanOrEqual,
43    /// `>`
44    GreaterThan,
45    /// `>=`
46    GreaterThanOrEqual,
47    // Keywords
48    /// The `AND` keyword.
49    And,
50    /// The `OR` keyword.
51    Or,
52    /// The `NOT` keyword.
53    Not,
54    /// The `FOLLOWEDBY` keyword.
55    FollowedBy,
56    /// The `LIKE` keyword.
57    Like,
58    /// The `MATCHES` keyword.
59    Matches,
60    /// The `IN` keyword.
61    In,
62    /// The `ISSUBSET` keyword.
63    IsSubset,
64    /// The `ISSUPERSET` keyword.
65    IsSuperset,
66    /// The `EXISTS` keyword.
67    Exists,
68    /// The `WITHIN` keyword.
69    Within,
70    /// The `REPEATS` keyword.
71    Repeats,
72    /// The `SECONDS` keyword.
73    Seconds,
74    /// The `TIMES` keyword.
75    Times,
76    /// The `START` keyword.
77    Start,
78    /// The `STOP` keyword.
79    Stop,
80    // Literals & identifiers
81    /// A bare identifier (object type or property name).
82    Identifier(String),
83    /// A single-quoted string literal (unescaped contents).
84    String(String),
85    /// An integer literal.
86    Integer(i64),
87    /// A floating-point literal.
88    Float(f64),
89    /// A `true`/`false` literal.
90    Boolean(bool),
91    /// A `t'...'` timestamp literal (inner string).
92    Timestamp(String),
93    /// A `b'...'` base64 literal (inner string).
94    Binary(String),
95    /// An `h'...'` hex literal (inner string).
96    Hex(String),
97}
98
99/// Tokenize a STIX pattern string into a vector of tokens.
100pub fn tokenize(src: &str) -> Result<Vec<Token>> {
101    Lexer::new(src).run()
102}
103
104struct Lexer<'a> {
105    src: &'a [u8],
106    pos: usize,
107    tokens: Vec<Token>,
108}
109
110impl<'a> Lexer<'a> {
111    fn new(src: &'a str) -> Self {
112        Lexer {
113            src: src.as_bytes(),
114            pos: 0,
115            tokens: Vec::new(),
116        }
117    }
118
119    fn run(mut self) -> Result<Vec<Token>> {
120        while let Some(&c) = self.src.get(self.pos) {
121            match c {
122                b' ' | b'\t' | b'\r' | b'\n' => self.pos += 1,
123                b'[' => self.punct(TokenKind::LBracket),
124                b']' => self.punct(TokenKind::RBracket),
125                b'(' => self.punct(TokenKind::LParen),
126                b')' => self.punct(TokenKind::RParen),
127                b':' => self.punct(TokenKind::Colon),
128                b'.' if !self.peek_is_digit(1) => self.punct(TokenKind::Dot),
129                b',' => self.punct(TokenKind::Comma),
130                b'*' => self.punct(TokenKind::Star),
131                b'=' => self.punct(TokenKind::Equal),
132                b'!' => self.lex_bang()?,
133                b'<' => self.lex_lt(),
134                b'>' => self.lex_gt(),
135                b'\'' => self.lex_string()?,
136                b'-' | b'0'..=b'9' => self.lex_number()?,
137                _ if is_ident_start(c) => self.lex_word_or_typed_literal()?,
138                _ => {
139                    return Err(ParseError::new(
140                        format!("unexpected character '{}'", c as char),
141                        Span::new(self.pos, self.pos + 1),
142                    ))
143                }
144            }
145        }
146        Ok(self.tokens)
147    }
148
149    fn peek_is_digit(&self, ahead: usize) -> bool {
150        matches!(self.src.get(self.pos + ahead), Some(b'0'..=b'9'))
151    }
152
153    fn push(&mut self, kind: TokenKind, start: usize) {
154        self.tokens.push(Token {
155            kind,
156            span: Span::new(start, self.pos),
157        });
158    }
159
160    fn punct(&mut self, kind: TokenKind) {
161        let start = self.pos;
162        self.pos += 1;
163        self.push(kind, start);
164    }
165
166    fn lex_bang(&mut self) -> Result<()> {
167        let start = self.pos;
168        if self.src.get(self.pos + 1) == Some(&b'=') {
169            self.pos += 2;
170            self.push(TokenKind::NotEqual, start);
171            Ok(())
172        } else {
173            Err(ParseError::new(
174                "expected '=' after '!'",
175                Span::new(start, start + 1),
176            ))
177        }
178    }
179
180    fn lex_lt(&mut self) {
181        let start = self.pos;
182        if self.src.get(self.pos + 1) == Some(&b'=') {
183            self.pos += 2;
184            self.push(TokenKind::LessThanOrEqual, start);
185        } else {
186            self.pos += 1;
187            self.push(TokenKind::LessThan, start);
188        }
189    }
190
191    fn lex_gt(&mut self) {
192        let start = self.pos;
193        if self.src.get(self.pos + 1) == Some(&b'=') {
194            self.pos += 2;
195            self.push(TokenKind::GreaterThanOrEqual, start);
196        } else {
197            self.pos += 1;
198            self.push(TokenKind::GreaterThan, start);
199        }
200    }
201
202    /// Reads a single-quoted string body starting at the opening quote.
203    /// Returns the decoded contents and advances past the closing quote.
204    fn read_quoted(&mut self) -> Result<String> {
205        let start = self.pos;
206        debug_assert_eq!(self.src.get(self.pos), Some(&b'\''));
207        self.pos += 1; // opening quote
208        let mut out = String::new();
209        loop {
210            match self.src.get(self.pos) {
211                None => {
212                    return Err(ParseError::new(
213                        "unterminated string literal",
214                        Span::new(start, self.pos),
215                    ))
216                }
217                Some(b'\\') => match self.src.get(self.pos + 1) {
218                    Some(b'\'') => {
219                        out.push('\'');
220                        self.pos += 2;
221                    }
222                    Some(b'\\') => {
223                        out.push('\\');
224                        self.pos += 2;
225                    }
226                    _ => {
227                        out.push('\\');
228                        self.pos += 1;
229                    }
230                },
231                Some(b'\'') => {
232                    self.pos += 1; // closing quote
233                    return Ok(out);
234                }
235                Some(&b) => {
236                    out.push(b as char);
237                    self.pos += 1;
238                }
239            }
240        }
241    }
242
243    fn lex_string(&mut self) -> Result<()> {
244        let start = self.pos;
245        let s = self.read_quoted()?;
246        self.push(TokenKind::String(s), start);
247        Ok(())
248    }
249
250    fn lex_number(&mut self) -> Result<()> {
251        let start = self.pos;
252        if self.src.get(self.pos) == Some(&b'-') {
253            self.pos += 1;
254        }
255        let mut is_float = false;
256        while let Some(&c) = self.src.get(self.pos) {
257            match c {
258                b'0'..=b'9' => self.pos += 1,
259                b'.' => {
260                    is_float = true;
261                    self.pos += 1;
262                }
263                _ => break,
264            }
265        }
266        let text = std::str::from_utf8(&self.src[start..self.pos]).unwrap();
267        if is_float {
268            let v: f64 = text.parse().map_err(|_| {
269                ParseError::new("invalid float literal", Span::new(start, self.pos))
270            })?;
271            self.push(TokenKind::Float(v), start);
272        } else {
273            let v: i64 = text.parse().map_err(|_| {
274                ParseError::new("invalid integer literal", Span::new(start, self.pos))
275            })?;
276            self.push(TokenKind::Integer(v), start);
277        }
278        Ok(())
279    }
280
281    /// Handles bare words (keywords, identifiers, booleans) AND the typed-literal
282    /// prefixes `t'...'`, `b'...'`, `h'...'`.
283    fn lex_word_or_typed_literal(&mut self) -> Result<()> {
284        let start = self.pos;
285        // Typed-literal prefix: a single letter t/b/h immediately followed by a quote.
286        if matches!(self.src.get(self.pos), Some(b't' | b'b' | b'h'))
287            && self.src.get(self.pos + 1) == Some(&b'\'')
288        {
289            let prefix = self.src[self.pos];
290            self.pos += 1; // consume prefix letter
291            let body = self.read_quoted()?;
292            let kind = match prefix {
293                b't' => TokenKind::Timestamp(body),
294                b'b' => TokenKind::Binary(body),
295                _ => TokenKind::Hex(body),
296            };
297            self.push(kind, start);
298            return Ok(());
299        }
300
301        while let Some(&c) = self.src.get(self.pos) {
302            if is_ident_continue(c) {
303                self.pos += 1;
304            } else {
305                break;
306            }
307        }
308        let word = std::str::from_utf8(&self.src[start..self.pos]).unwrap();
309        let kind = keyword(word).unwrap_or_else(|| TokenKind::Identifier(word.to_string()));
310        self.push(kind, start);
311        Ok(())
312    }
313}
314
315fn is_ident_start(c: u8) -> bool {
316    c.is_ascii_alphabetic() || c == b'_'
317}
318
319/// Identifier continuation allows hyphens so object types like `ipv4-addr` lex as one token.
320fn is_ident_continue(c: u8) -> bool {
321    c.is_ascii_alphanumeric() || c == b'_' || c == b'-'
322}
323
324/// Map a bare word to a keyword token (case-insensitive), or `None` if it is an identifier.
325fn keyword(word: &str) -> Option<TokenKind> {
326    match word.to_ascii_uppercase().as_str() {
327        "AND" => Some(TokenKind::And),
328        "OR" => Some(TokenKind::Or),
329        "NOT" => Some(TokenKind::Not),
330        "FOLLOWEDBY" => Some(TokenKind::FollowedBy),
331        "LIKE" => Some(TokenKind::Like),
332        "MATCHES" => Some(TokenKind::Matches),
333        "IN" => Some(TokenKind::In),
334        "ISSUBSET" => Some(TokenKind::IsSubset),
335        "ISSUPERSET" => Some(TokenKind::IsSuperset),
336        "EXISTS" => Some(TokenKind::Exists),
337        "WITHIN" => Some(TokenKind::Within),
338        "REPEATS" => Some(TokenKind::Repeats),
339        "SECONDS" => Some(TokenKind::Seconds),
340        "TIMES" => Some(TokenKind::Times),
341        "START" => Some(TokenKind::Start),
342        "STOP" => Some(TokenKind::Stop),
343        "TRUE" => Some(TokenKind::Boolean(true)),
344        "FALSE" => Some(TokenKind::Boolean(false)),
345        _ => None,
346    }
347}
348
349#[cfg(test)]
350mod tests {
351    use super::*;
352
353    fn kinds(src: &str) -> Vec<TokenKind> {
354        tokenize(src).unwrap().into_iter().map(|t| t.kind).collect()
355    }
356
357    #[test]
358    fn lexes_simple_comparison() {
359        let k = kinds("[ipv4-addr:value = '1.2.3.4']");
360        assert_eq!(
361            k,
362            vec![
363                TokenKind::LBracket,
364                TokenKind::Identifier("ipv4-addr".to_string()),
365                TokenKind::Colon,
366                TokenKind::Identifier("value".to_string()),
367                TokenKind::Equal,
368                TokenKind::String("1.2.3.4".to_string()),
369                TokenKind::RBracket,
370            ]
371        );
372    }
373
374    #[test]
375    fn lexes_keywords_case_insensitively() {
376        assert_eq!(kinds("AND and"), vec![TokenKind::And, TokenKind::And]);
377        assert_eq!(kinds("FOLLOWEDBY"), vec![TokenKind::FollowedBy]);
378    }
379
380    #[test]
381    fn lexes_operators() {
382        assert_eq!(
383            kinds("= != < <= > >="),
384            vec![
385                TokenKind::Equal,
386                TokenKind::NotEqual,
387                TokenKind::LessThan,
388                TokenKind::LessThanOrEqual,
389                TokenKind::GreaterThan,
390                TokenKind::GreaterThanOrEqual,
391            ]
392        );
393    }
394
395    #[test]
396    fn lexes_literals() {
397        assert_eq!(kinds("42"), vec![TokenKind::Integer(42)]);
398        assert_eq!(kinds("-7"), vec![TokenKind::Integer(-7)]);
399        assert_eq!(kinds("2.5"), vec![TokenKind::Float(2.5)]);
400        assert_eq!(
401            kinds("true false"),
402            vec![TokenKind::Boolean(true), TokenKind::Boolean(false)]
403        );
404        assert_eq!(
405            kinds("t'2014-01-13T07:03:17Z'"),
406            vec![TokenKind::Timestamp("2014-01-13T07:03:17Z".to_string())]
407        );
408        assert_eq!(
409            kinds("b'aGVsbG8='"),
410            vec![TokenKind::Binary("aGVsbG8=".to_string())]
411        );
412        assert_eq!(
413            kinds("h'1234abcd'"),
414            vec![TokenKind::Hex("1234abcd".to_string())]
415        );
416    }
417
418    #[test]
419    fn lexes_string_escapes() {
420        assert_eq!(kinds(r"'a\'b'"), vec![TokenKind::String("a'b".to_string())]);
421        assert_eq!(
422            kinds(r"'a\\b'"),
423            vec![TokenKind::String(r"a\b".to_string())]
424        );
425    }
426
427    #[test]
428    fn unterminated_string_errors() {
429        let err = tokenize("'oops").unwrap_err();
430        assert!(err.message.contains("unterminated"), "got: {}", err.message);
431    }
432
433    #[test]
434    fn tracks_spans() {
435        let toks = tokenize("[a:b]").unwrap();
436        assert_eq!(toks[0].span, crate::error::Span::new(0, 1)); // '['
437    }
438}
stix_pattern/lexer.rs

stix_pattern/
lexer.rs
