diff options
Diffstat (limited to 'subprojects/frontend/src/language/tokens.ts')
-rw-r--r-- | subprojects/frontend/src/language/tokens.ts | 46 |
1 files changed, 46 insertions, 0 deletions
diff --git a/subprojects/frontend/src/language/tokens.ts b/subprojects/frontend/src/language/tokens.ts new file mode 100644 index 00000000..b5d022f1 --- /dev/null +++ b/subprojects/frontend/src/language/tokens.ts | |||
@@ -0,0 +1,46 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2018-2024 by Marijn Haverbeke <marijnh@gmail.com> and others | ||
3 | * Copyright (C) 2024 The Refinery Authors <https://refinery.tools/> | ||
4 | * | ||
5 | * SPDX-License-Identifier: MIT AND EPL-2.0 | ||
6 | * | ||
7 | * Based on the CSS tokenizer at | ||
8 | * https://github.com/lezer-parser/css/blob/790568c968a660a94bf0fbd97a86c66da1c529e5/src/tokens.js | ||
9 | */ | ||
10 | |||
11 | import { ExternalTokenizer } from '@lezer/lr'; | ||
12 | |||
13 | /* eslint-disable-next-line import/no-unresolved -- | ||
14 | Synthetic import from `@lezer/generator/rollup` cannot be found by ESLint. | ||
15 | */ | ||
16 | import { QualifiedNameSeparator } from './problem.grammar.terms'; | ||
17 | |||
18 | const colon = 58; | ||
19 | const underscore = 95; | ||
20 | |||
21 | function isAlpha(ch: number) { | ||
22 | return (ch >= 65 && ch <= 90) || (ch >= 97 && ch <= 122) || ch >= 161; | ||
23 | } | ||
24 | |||
25 | function isDigit(ch: number) { | ||
26 | return ch >= 48 && ch <= 57; | ||
27 | } | ||
28 | |||
29 | function isIdentifier(ch: number) { | ||
30 | return isAlpha(ch) || isDigit(ch) || ch === underscore; | ||
31 | } | ||
32 | |||
33 | /* eslint-disable-next-line import/prefer-default-export -- | ||
34 | Lezer requires a named export. | ||
35 | */ | ||
36 | export const qualifiedNameSeparator = new ExternalTokenizer((input) => { | ||
37 | if (input.peek(0) === colon && input.peek(1) === colon) { | ||
38 | const previous = input.peek(-1); | ||
39 | if (isIdentifier(previous)) { | ||
40 | // Inject an extra 0-length token into the token stream to let Lezer | ||
41 | // consume the `::` on its own. Explicitly consuming 2 characters here | ||
42 | // leads to inconsistent highlighting of qualified names. | ||
43 | input.acceptToken(QualifiedNameSeparator); | ||
44 | } | ||
45 | } | ||
46 | }); | ||