🌋 WebIDE 的开发记录其八（Textmate详解）

// 非常简略的加载和运行逻辑部分（官方使用 demo 演示）
const { loadWASM, createOnigScanner, createOnigString } = require('vscode-oniguruma')
// oniguruma 的加载和封装
const wasmBin = fs.readFileSync('path/onig.wasm')).buffer;
const vscodeOnigurumaLib = loadWASM(wasmBin).then(() => ({
  createOnigScanner,
  createOnigString
}));
// -> source.js 是 textmate 中 js 的 scopeName，算是唯一标识 id
const vsctm = require('vscode-textmate');
// 创建 textmate 的注册机来管理整个的语法加载和解析
const registry = new vsctm.Registry({
  onigLib: vscodeOnigurumaLib,
  loadGrammar: (scopeName) => {
    if (scopeName === 'source.js') {
      // https://github.com/textmate/javascript.tmbundle/blob/master/Syntaxes/JavaScript.plist
      return readFile('./JavaScript.plist').then(data => vsctm.parseRawGrammar(data.toString()))
    }
    console.log(`Unknown scope name: ${scopeName}`);
    return null;
  }
});
// 加载 js 的语法文件并解析
await registry.loadGrammar('source.js')
// 解析细节
const text = [
    `function sayHello(name) {`,
    `\treturn "Hello, " + name;`,
    `}`
];
let ruleStack = vsctm.INITIAL;
for (let i = 0; i < text.length; i++) {
    const line = text[i];
    const lineTokens = grammar.tokenizeLine(line, ruleStack);
    console.log(`\nTokenizing line: ${line}`);
    for (let j = 0; j < lineTokens.tokens.length; j++) {
        const token = lineTokens.tokens[j];
        console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
          `(${line.substring(token.startIndex, token.endIndex)}) ` +
          `with scopes ${token.scopes.join(', ')}`
        );
    }
    ruleStack = lineTokens.ruleStack;
}
// 输出每一行的每个关键逻辑的 token 结果

const loaderConfig = {
  paths: {
    vs: "https://cdn.jsdelivr.net/npm/monaco-editor-core@0.30.1/min/vs"
  }
};
export function monacoLoader(): Promise<Monaco> {
  return new Promise<Monaco>((resolve, reject) => {
    const script = document.createElement("script");
    script.src = `${loaderConfig.paths.vs}/loader.js`;
    script.onload = () => configureLoader(resolve, reject);
    script.onerror = reject;
    document.body.appendChild(script);
  });
}
function configureLoader(resolve: (monaco: Monaco) => void, reject: Function) {
  window.require.config(loaderConfig);
  window.require(["vs/editor/editor.main"], resolve, reject);
}
export default monacoLoader;

const monaco = await monacoLoader();
(window as any).monaco = monaco;
const defaultModel =  monaco.editor.createModel(
  `# Head1\n\nHello World.`,
  "markdown",
  monaco.Uri.parse("file:///index.md")
);
const editor = monaco.editor.create(
  containerRef.current,
  {
    model: defaultModel,
    automaticLayout: true
  },
  {}
);

import { loadWASM, OnigScanner, OnigString } from "onigasm";
const onigasmPath = "https://cdn.jsdelivr.net/npm/onigasm@2.2.5/lib/onigasm.wasm";
// onigasm 加载器
onigasmPromise = fetch(onigasmPath, { cache: "force-cache" })
  .then(res => res.arrayBuffer())
  .then(buffer => loadWASM(buffer))
  .then(() => new OnigasmLib());
export class OnigasmLib implements IOnigLib {
  createOnigScanner(sources: string[]) => new OnigScanner(sources)
  createOnigString(sources: string) => new OnigString(sources)
}

{ // vscode theme json 节选
  "$schema": "vscode://schemas/color-theme",
  "name": "Dark+ (default dark)",
  "include": "./dark_vs.json",
  "tokenColors": [
    {
      "name": "Function declarations",
      "scope": [
        "entity.name.function",
        //...
      ],
      "settings": { "foreground": "#DCDCAA" }
    },
    //...
  ]
}

/**
 * A single theme setting.
 */
export interface IRawThemeSetting {
    readonly name?: string;
    readonly scope?: string | string[];
    readonly settings: {
        readonly fontStyle?: string;
        readonly foreground?: string;
        readonly background?: string;
    };
}
/**
 * A TextMate theme.
 */
export interface IRawTheme {
    readonly name?: string;
    readonly settings: IRawThemeSetting[];
}

monaco.editor.defineTheme('myCustomTheme', {
    base: 'vs', // can also be vs-dark or hc-black
    inherit: true, // can also be false to completely replace the builtin rules
    rules: [
        { token: 'comment', foreground: 'ffa500', fontStyle: 'italic underline' },
        { token: 'comment.js', foreground: '008800', fontStyle: 'bold' },
        { token: 'comment.css', foreground: '0000ff' } // will inherit fontStyle from `comment` above
    ],
    colors: {
        'editor.foreground': '#000000'
    }
});

import { IStandaloneThemeData, BuiltinTheme, ITokenThemeRule } from "../editor.d";
import { IRawTheme, IRawThemeSetting, Registry } from "./vscode-textmate/main";
export type VSCTheme = {
  colors: { [i: string]: string };
  tokenColors: IRawThemeSetting[];
  include?: string;
};
type Includes = { [i: string]: VSCTheme };
export type ThemeMix = {
  name: string;
  textmateTheme: IRawTheme;
  monacoTheme: IStandaloneThemeData;
};
export class MonacoThemeRegistry {
  protected themes = new Map<string, ThemeMix>();
  public getRawTheme = (name: string) => this.themes.get(name)?.textmateTheme;
  public getMonacoTheme = (name: string) => this.themes.get(name)?.monacoTheme;
  mergedVSCTheme = (json: VSCTheme, includes?: Includes): VSCTheme => {
    const after: VSCTheme = {
      colors: json.colors,
      tokenColors: json.tokenColors,
    };
    if (json.include && includes && includes[json.include]) {
      const parent = this.mergedVSCTheme(includes[json.include], includes);
      after.colors = { ...(parent.colors || {}), ...after.colors };
      after.tokenColors = [...(parent.tokenColors || []), ...after.tokenColors];
    }
    return after;
  };
  genTextmateTheme = (vsctheme: VSCTheme) => {
    return { settings: [...vsctheme.tokenColors] };
  };
  genMonacoTheme = (
    vsctheme: VSCTheme,
    textmateTheme: IRawTheme,
    monacoBase: BuiltinTheme = "vs-dark"
  ) => {
    const monacoTheme: IStandaloneThemeData = {
      base: monacoBase || "vs",
      inherit: true,
      rules: [], // token 和样式对应规则
      colors: {}, // 主要是编辑器最普适的颜色表，前景背景之类
      encodedTokensColors: [], // 编码后的颜色规则，就是 css 中的 mtk1 之类
    };
    // 默认的编辑器颜色，对应背景、前景、选择、高亮等等一般规则
    if (vsctheme.colors) Object.assign(monacoTheme.colors, vsctheme.colors);
    // 把 textmate 的 IRawThemeSetting 转为 monaco 的 ITokenThemeRule
    for (const setting of vsctheme.tokenColors)
      this.transform(setting, (rule) => monacoTheme.rules.push(rule));
    // 后面就是提取 encodedTokensColors 来实现真实的 monaco 着色
    // 这里必须使用 4.x 的代码，不然这里会出现类型问题
    const reg = new Registry();
    reg.setTheme(textmateTheme);
    monacoTheme.encodedTokensColors = reg.getColorMap();
    // index 0 has to be set to null for monaco
    monacoTheme.encodedTokensColors[0] = null!;
    // index 1 and 2 are the default colors
    const foreColor = monacoTheme.colors["editor.foreground"];
    if (foreColor) monacoTheme.encodedTokensColors[1] = foreColor;
    const backColor = monacoTheme.colors["editor.background"];
    if (backColor) monacoTheme.encodedTokensColors[2] = backColor;
    return monacoTheme;
  };
  public register(
    givenName: string,
    vsc: VSCTheme,
    monacoBase: BuiltinTheme = "vs-dark"
  ): ThemeMix {
    if (this.themes.has(givenName)) return this.themes.get(givenName)!;
    const textmateTheme = this.genTextmateTheme(vsc);
    const monacoTheme = this.genMonacoTheme(vsc, textmateTheme, monacoBase);
    const result: ThemeMix = {
      name: givenName,
      textmateTheme: this.genTextmateTheme(vsc),
      monacoTheme: this.genMonacoTheme(vsc, textmateTheme, monacoBase),
    };
    this.themes.set(givenName, result);
    monaco.editor.defineTheme(givenName, monacoTheme);
    return result;
  }
  protected transform(
    setting: any,
    acceptor: (rule: ITokenThemeRule) => void
  ): void {
    if (typeof setting.scope === "undefined") setting.scope = [""];
    if (typeof setting.scope === "string") setting.scope = [setting.scope];
    for (const scope of setting.scope) {
      const settings = Object.keys(setting.settings).reduce(
        (previous: { [key: string]: string }, current) => {
          let value = setting.settings[current];
          if (typeof value === "string")
            value = value.replace(/^\#/, "").slice(0, 6);
          previous[current] = value;
          return previous;
        },
        {}
      );
      acceptor({ ...settings, token: scope });
    }
  }
}
export const monacoThemeRegistry = new MonacoThemeRegistry();
const darkPlus = monacoThemeRegistry.mergedVSCTheme(
  require("./theme/vscode/dark_plus.json"),
  {
    "./dark_defaults.json": require("./theme/vscode/dark_defaults.json"),
    "./dark_vs.json": require("./theme/vscode/dark_vs.json"),
  }
);
export const DARK_DEFAULT_THEME = monacoThemeRegistry.register(
  "dark-plus",
  darkPlus,
  "vs-dark"
);

const languageId = 'markdown'
monaco.languages.register({
  id: languageId,
  extensions: [".md", ".markdown", ".mdown", ".mkdn", ".mkd", ".mdwn", ".mdtxt", ".mdtext"],
  aliases: ["Markdown", "markdown"]
});

import { Registry, IRawGrammar, parseRawGrammar } from "./vscode-textmate/main";
const mdScopeName = "text.html.markdown"
// 初始化一个注册器，整个语法支持的核心
grammarRegistry = new Registry({
  getOnigLib: () => onigasmPromise,
  theme: DARK_DEFAULT_THEME.textmateTheme,
  loadGrammar: async (scopeName: string) => {
    if(scopeName === "text.html.markdown") {
      return await fetch('./markdown.tmLanguage.json', { cache: "force-cache" }).then(res => res.json())
    }
    return undefined;
  },
});

import { INITIAL, StackElement, IGrammar } from "./vscode-textmate/main";
export class TokenizerState implements IState {
  constructor(public readonly ruleStack: StackElement) {}
  clone = (): IState => new TokenizerState(this.ruleStack)
  equals = (other: IState): boolean => (
    other instanceof TokenizerState &&
    (other === this || other.ruleStack === this.ruleStack)
  )
}
// 激活语言支持
const _activatedLanguages = new Set<string>();
const activateLanguage = async (languageId: string): Promise<void> => {
  // 获取语言配置
  /*{
    "comments": { "blockComment": ["<!--", "-->"] },
    // symbols used as brackets
    "brackets": [["{", "}"], ["[", "]"], ["(", ")"]],
    // symbols that are auto closed when typing
    "autoClosingPairs": [
      ["{", "}"], // ---> 需要手动转为 open、close 的形式
      {
        "open": "\"",
        "close": "\"",
        "notIn": ["string"]
      },
      // ...
    ],
    // symbols that that can be used to surround a selection
    "surroundingPairs": [
      ["{", "}"],
      // ...
    ],
    "folding": {
      // ...
    }
  }
  */
  const configuration = await fetch('./language-configuration.json').then(res => res.json())
  // 转换部分配置（重要）
  raw.autoClosingPairs = raw.autoClosingPairs.map((pair: any) => {
    if (Array.isArray(pair)) return { open: pair[0], close: pair[1] };
    return pair;
  });
  monaco.languages.setLanguageConfiguration(languageId, configuration);
  // grammer 部分
  /*
      {
        "language": "markdown",
        "scopeName": "text.html.markdown",
        // 这里是引用的 IRawGrammar 文件
        "path": "./syntaxes/markdown.tmLanguage.json",
        // 这里是 IGrammarConfiguration 配置
        "embeddedLanguages": {
          "meta.embedded.block.html": "html",
          "source.js": "javascript",
          "source.css": "css",
        }
      }
  */
  // 根据之前注册的语言获得语言 id(number)
  const encodedLanguageId = monaco.languages.getEncodedLanguageId(languageId);
  // 转换部分配置（重要）
  if (tmConfig) {
    const embeddedLanguages = tmConfig.embeddedLanguages || {};
    for (const key in embeddedLanguages) {
      if (Object.prototype.hasOwnProperty.call(embeddedLanguages, key)) {
        const lang = embeddedLanguages[key];
        embeddedLanguages[key] = monaco.languages.getEncodedLanguageId(lang);
        if (embeddedLanguages[key] === 0) delete embeddedLanguages[key];
      }
    }
    tmConfig.embeddedLanguages = embeddedLanguages;
  }
  const grammar = await grammarRegistry.loadGrammarWithConfiguration(
    mdScopeName,
    encodedLanguageId,
    tmConfig
  );
  
  monaco.languages.setTokensProvider(languageId,
    {
      getInitialState: () => new TokenizerState(INITIAL),
      tokenizeEncoded(line: string, state: TokenizerState): IEncodedLineTokens {
        // console.log('unencoded token for debug', line);
        // console.log(grammar.tokenizeLine(line, state.ruleStack));
        const result = grammar.tokenizeLine2(line, state.ruleStack);
        return {
          endState: new TokenizerState(result.ruleStack),
          tokens: result.tokens
        };
      }
    }
  );
  console.info("开启语言 " + languageId + "支持");
};
activateLanguage(languageId)

bat, clojure, coffeescript, cpp, csharp, css, dart, diff, docker, fsharp, go, groovy, handlebars, hlsl, html, ini, ipynb, java, javascript, json, julia, latex, less, log, lua, make, markdown-basics, markdown-math, objective-c, perl, php, powershell, pug, python, r, razor, ruby, rust, scss, shaderlab, shellscript, sql, swift, typescript-basics, vb, xml, yaml

{
  "contributes": {
    "languages": [
      {
        // 这里是引用的 monaco.languages.ILanguageExtensionPoint 文件
        "id": "markdown",
        "aliases": ["Markdown", "markdown"],
        "extensions": [".md", ".markdown", /*...*/],
        // 这里是引用的 monaco.editor.LanguageConfiguration 文件
        "configuration": "./language-configuration.json"
      }
    ],
    "grammars": [
      {
        "language": "markdown",
        "scopeName": "text.html.markdown",
        // 这里是引用的 IRawGrammar 文件
        "path": "./syntaxes/markdown.tmLanguage.json",
        // 这里是 IGrammarConfiguration 配置（需要将字符串的 language 转为数字id的 monaco.languages.getEncodedLanguageId(lang)）
        "embeddedLanguages": {
          "meta.embedded.block.html": "html",
          "source.js": "javascript",
          "source.css": "css",
        }
      }
    ],
  },
}

import { IGrammarConfiguration } from "./textmate/vscode-textmate/main";
type GrammarPkg = {
  contributes: {
    languages: {
      id: string;
      extensions: string[];
      aliases: string[];
      configuration: string;
    }[];
    grammars: ({
      language: string;
      scopeName: string;
      path: string;
    } & IGrammarConfiguration)[];
  };
};
export type GrammarInfo = {
  id: string;
  languageId: string;
  scopeName: string;
  language?: {
    id: string;
    extensions: string[];
    aliases: string[];
  };
  tmConfig?: IGrammarConfiguration;
  tmPath: string;
  configurationPath: string;
};
const basePath = "/libs/grammars";
const grammarsPath: string[] = [
  "bat",
  "clojure",
  "coffeescript",
  "cpp",
  "csharp",
  "css",
  // ...
];
const fixPath = (s: string) => s.replace(/\.\//, "");
// 不严格的 json 解析防止某些语法文件有注释、多个逗号之类的问题
const easyJsonDec = async (str: string) => new Function(`return ${str}`)();
export const loadRawJson = async (path: string) => {
  return await fetch(path, { cache: "force-cache" })
    .then((res) => res.text())
    .then((str) => easyJsonDec(str));
};
export const prepareGrammars = async () => {
  // 异步加载的形式，可以改成打包、预渲染、预处理等提前合并的手段避免这么多网络请求
  const grammarsPkgs: GrammarPkg[] = await Promise.all<GrammarPkg>(
    grammarsPath.map((p) =>
      fetch(`${basePath}/${p}/package.json`, { cache: "force-cache" }).then((res) => res.json())
    )
  );
  const grammarsOutput: GrammarInfo[] = [];
  grammarsPkgs.forEach(async (grammar, i) => {
    const folder = `${basePath}/${grammarsPath[i]}`;
    const { grammars, languages } = grammar.contributes;
    grammars.forEach(async (grammarConf) => {
      const tm = grammars.find((g) => g.scopeName == grammarConf.scopeName)!;
      let language =
        languages.length == 1
          ? languages[0]
          : languages.find((g) => g.id == tm.language);
      if (!language || !language.configuration) return;
      grammarsOutput.push({
        id: grammarConf.scopeName,
        languageId: language.id,
        scopeName: tm.scopeName,
        language,
        tmConfig: tm,
        tmPath: `${folder}/${fixPath(tm.path)}`,
        configurationPath: `${folder}/${fixPath(language.configuration)}`,
      });
    });
  });
  return grammarsOutput;
};