feat(compiler): basic codegen with source map support

packages/compiler-core/__tests__/codegen.spec.ts

@@ -0,0 +1,20 @@
+import { parse, generate } from '../src'
+import { SourceMapConsumer, RawSourceMap } from 'source-map'
+
+describe('compiler: codegen', () => {
+  test('basic source map support', async () => {
+    const ast = parse(`hello {{ world }}`)
+    const { code, map } = generate(ast, { module: false })
+    expect(code).toBe(`["hello ", world]`)
+
+    const consumer = await new SourceMapConsumer(map as RawSourceMap)
+    const pos = consumer.originalPositionFor({
+      line: 1,
+      column: 11
+    })
+    expect(pos).toMatchObject({
+      line: 1,
+      column: 6
+    })
+  })
+})

packages/compiler-core/__tests__/utils.spec.ts

@@ -1,14 +1,14 @@
 import { Position } from '../src/ast'
-import { getInnerRange, advancePositionBy } from '../src/utils'
+import { getInnerRange, advancePositionWithClone } from '../src/utils'
 
 function p(line: number, column: number, offset: number): Position {
   return { column, line, offset }
 }
 
-describe('advancePositionBy', () => {
+describe('advancePositionWithClone', () => {
   test('same line', () => {
     const pos = p(1, 1, 0)
-    const newPos = advancePositionBy(pos, 'foo\nbar', 2)
+    const newPos = advancePositionWithClone(pos, 'foo\nbar', 2)
 
     expect(newPos.column).toBe(3)
     expect(newPos.line).toBe(1)
@@ -17,7 +17,7 @@ describe('advancePositionBy', () => {
 
   test('same line', () => {
     const pos = p(1, 1, 0)
-    const newPos = advancePositionBy(pos, 'foo\nbar', 4)
+    const newPos = advancePositionWithClone(pos, 'foo\nbar', 4)
 
     expect(newPos.column).toBe(1)
     expect(newPos.line).toBe(2)
@@ -26,7 +26,7 @@ describe('advancePositionBy', () => {
 
   test('multiple lines', () => {
     const pos = p(1, 1, 0)
-    const newPos = advancePositionBy(pos, 'foo\nbar\nbaz', 10)
+    const newPos = advancePositionWithClone(pos, 'foo\nbar\nbaz', 10)
 
     expect(newPos.column).toBe(2)
     expect(newPos.line).toBe(3)

packages/compiler-core/package.json

@@ -9,7 +9,9 @@
     "dist"
   ],
   "types": "dist/compiler-core.d.ts",
-  "sideEffects": false,
+  "buildOptions": {
+    "formats": ["cjs"]
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vuejs/vue.git"
@@ -22,5 +24,8 @@
   "bugs": {
     "url": "https://github.com/vuejs/vue/issues"
   },
-  "homepage": "https://github.com/vuejs/vue/tree/dev/packages/compiler-core#readme"
+  "homepage": "https://github.com/vuejs/vue/tree/dev/packages/compiler-core#readme",
+  "dependencies": {
+    "source-map": "^0.7.3"
+  }
 }

packages/compiler-core/src/ast.ts

@@ -109,7 +109,7 @@ export interface ForNode extends Node {
 }
 
 export interface Position {
-  offset: number // from start of file (in SFCs)
+  offset: number // from start of file
   line: number
   column: number
 }

packages/compiler-core/src/codegen.ts

@@ -1 +1,165 @@
-// TODO
+import {
+  RootNode,
+  ChildNode,
+  ElementNode,
+  IfNode,
+  ForNode,
+  TextNode,
+  CommentNode,
+  ExpressionNode,
+  NodeTypes
+} from './ast'
+import { SourceMapGenerator, RawSourceMap } from 'source-map'
+import { advancePositionWithMutation } from './utils'
+
+export interface CodegenOptions {
+  // Assume ES module environment. If true, will generate import statements for
+  // runtime helpers; otherwise will grab the helpers from global `Vue`.
+  module?: boolean
+  // Filename for source map generation.
+  filename?: string
+}
+
+export interface CodegenResult {
+  code: string
+  map?: RawSourceMap
+}
+
+interface CodegenContext extends Required<CodegenOptions> {
+  source: string
+  code: string
+  line: number
+  column: number
+  offset: number
+  indent: number
+  identifiers: Set<string>
+  map?: SourceMapGenerator
+  push(generatedCode: string, astNode?: ChildNode): void
+}
+
+export function generate(
+  ast: RootNode,
+  options: CodegenOptions = {}
+): CodegenResult {
+  const context = createCodegenContext(ast, options)
+  if (context.module) {
+    // TODO inject import statements on RootNode
+    context.push(`export function render() {\n`)
+    context.indent++
+    context.push(`  return `)
+  }
+  if (ast.children.length === 1) {
+    genNode(ast.children[0], context)
+  } else {
+    genChildren(ast.children, context)
+  }
+  if (context.module) {
+    context.indent--
+    context.push(`\n}`)
+  }
+  return {
+    code: context.code,
+    map: context.map ? context.map.toJSON() : undefined
+  }
+}
+
+function createCodegenContext(
+  ast: RootNode,
+  options: CodegenOptions
+): CodegenContext {
+  const context: CodegenContext = {
+    module: true,
+    filename: `template.vue.html`,
+    ...options,
+    source: ast.loc.source,
+    code: ``,
+    column: 1,
+    line: 1,
+    offset: 0,
+    indent: 0,
+    identifiers: new Set(),
+    // lazy require source-map implementation, only in non-browser builds!
+    map: __BROWSER__
+      ? undefined
+      : new (require('source-map')).SourceMapGenerator(),
+    push(generatedCode, node) {
+      // TODO handle indent
+      context.code += generatedCode
+      if (context.map) {
+        if (node) {
+          context.map.addMapping({
+            source: context.filename,
+            original: {
+              line: node.loc.start.line,
+              column: node.loc.start.column - 1 // source-map column is 0 based
+            },
+            generated: {
+              line: context.line,
+              column: context.column - 1
+            }
+          })
+        }
+        advancePositionWithMutation(
+          context,
+          generatedCode,
+          generatedCode.length
+        )
+      }
+    }
+  }
+  return context
+}
+
+function genChildren(children: ChildNode[], context: CodegenContext) {
+  context.push(`[`)
+  for (let i = 0; i < children.length; i++) {
+    genNode(children[i], context)
+    if (i < children.length - 1) context.push(', ')
+  }
+  context.push(`]`)
+}
+
+function genNode(node: ChildNode, context: CodegenContext) {
+  switch (node.type) {
+    case NodeTypes.ELEMENT:
+      genElement(node, context)
+      break
+    case NodeTypes.TEXT:
+      genText(node, context)
+      break
+    case NodeTypes.EXPRESSION:
+      genExpression(node, context)
+      break
+    case NodeTypes.COMMENT:
+      genComment(node, context)
+      break
+    case NodeTypes.IF:
+      genIf(node, context)
+      break
+    case NodeTypes.FOR:
+      genFor(node, context)
+      break
+  }
+}
+
+function genElement(el: ElementNode, context: CodegenContext) {}
+
+function genText(node: TextNode | ExpressionNode, context: CodegenContext) {
+  context.push(JSON.stringify(node.content), node)
+}
+
+function genExpression(node: ExpressionNode, context: CodegenContext) {
+  if (!__BROWSER__) {
+    // TODO parse expression content and rewrite identifiers
+  }
+  context.push(node.content, node)
+}
+
+function genComment(node: CommentNode, context: CodegenContext) {
+  context.push(`<!--${node.content}-->`, node)
+}
+
+// control flow
+function genIf(node: IfNode, context: CodegenContext) {}
+
+function genFor(node: ForNode, context: CodegenContext) {}

packages/compiler-core/src/directives/vFor.ts

@@ -1,11 +1,12 @@
 import { createDirectiveTransform } from '../transform'
-import { NodeTypes, ExpressionNode, Node } from '../ast'
+import { NodeTypes, ExpressionNode } from '../ast'
 import { createCompilerError, ErrorCodes } from '../errors'
 import { getInnerRange } from '../utils'
 
 const forAliasRE = /([\s\S]*?)(?:(?<=\))|\s+)(?:in|of)\s+([\s\S]*)/
 const forIteratorRE = /,([^,\}\]]*)(?:,([^,\}\]]*))?$/
 const stripParensRE = /^\(|\)$/g
+
 export const transformFor = createDirectiveTransform(
   'for',
   (node, dir, context) => {
@@ -114,7 +115,7 @@ function parseAliasExpressions(source: string): AliasExpressions | null {
 
 function maybeCreateExpression(
   alias: AliasExpression | undefined,
-  node: Node
+  node: ExpressionNode
 ): ExpressionNode | undefined {
   if (alias) {
     return {

packages/compiler-core/src/index.ts

@@ -1,6 +1,5 @@
 export { parse, ParserOptions, TextModes } from './parse'
-export { transform, Transform, TransformContext } from './transform'
+export { transform, TransformOptions, Transform } from './transform'
+export { generate, CodegenOptions, CodegenResult } from './codegen'
 export { ErrorCodes } from './errors'
 export * from './ast'
-
-export { transformIf } from './directives/vIf'

packages/compiler-core/src/optimizations/mergeExpressions.ts

@@ -0,0 +1,2 @@
+// TODO merge adjacent text nodes and expressions into a single expression
+// e.g. <div>abc {{ d }} {{ e }}</div> should have a single expression node as child.

packages/compiler-core/src/parse.ts

@@ -1,5 +1,9 @@
 import { ErrorCodes, CompilerError, createCompilerError } from './errors'
-import { assert, advancePositionWithMutation } from './utils'
+import {
+  assert,
+  advancePositionWithMutation,
+  advancePositionWithClone
+} from './utils'
 import {
   Namespace,
   Namespaces,
@@ -799,6 +803,7 @@ function startsWith(source: string, searchString: string): boolean {
 
 function advanceBy(context: ParserContext, numberOfCharacters: number): void {
   const { source } = context
+  __DEV__ && assert(numberOfCharacters <= source.length)
   advancePositionWithMutation(context, source, numberOfCharacters)
   context.source = source.slice(numberOfCharacters)
 }
@@ -815,24 +820,11 @@ function getNewPosition(
   start: Position,
   numberOfCharacters: number
 ): Position {
-  const { originalSource } = context
+  return advancePositionWithClone(
-  const str = originalSource.slice(start.offset, numberOfCharacters)
+    start,
-  const lines = str.split(/\r?\n/)
+    context.originalSource.slice(start.offset, numberOfCharacters),
-
+    numberOfCharacters
-  const newPosition = {
+  )
-    column: start.column,
-    line: start.line,
-    offset: start.offset
-  }
-
-  newPosition.offset += numberOfCharacters
-  newPosition.line += lines.length - 1
-  newPosition.column =
-    lines.length === 1
-      ? start.column + numberOfCharacters
-      : Math.max(1, lines.pop()!.length)
-
-  return newPosition
 }
 
 function emitError(

packages/compiler-core/src/transform.ts

@@ -22,7 +22,7 @@ export interface TransformOptions {
   onError?: (error: CompilerError) => void
 }
 
-export interface TransformContext extends Required<TransformOptions> {
+interface TransformContext extends Required<TransformOptions> {
   parent: ParentNode
   ancestors: ParentNode[]
   childIndex: number

packages/compiler-core/src/utils.ts

@@ -5,21 +5,27 @@ export function getInnerRange(
   offset: number,
   length?: number
 ): SourceLocation {
+  __DEV__ && assert(offset <= loc.source.length)
   const source = loc.source.substr(offset, length)
   const newLoc: SourceLocation = {
     source,
-    start: advancePositionBy(loc.start, loc.source, offset),
+    start: advancePositionWithClone(loc.start, loc.source, offset),
     end: loc.end
   }
 
   if (length != null) {
-    newLoc.end = advancePositionBy(loc.start, loc.source, offset + length)
+    __DEV__ && assert(offset + length <= loc.source.length)
+    newLoc.end = advancePositionWithClone(
+      loc.start,
+      loc.source,
+      offset + length
+    )
   }
 
   return newLoc
 }
 
-export function advancePositionBy(
+export function advancePositionWithClone(
   pos: Position,
   source: string,
   numberOfCharacters: number
@@ -34,8 +40,6 @@ export function advancePositionWithMutation(
   source: string,
   numberOfCharacters: number
 ): Position {
-  __DEV__ && assert(numberOfCharacters <= source.length)
-
   let linesCount = 0
   let lastNewLinePos = -1
   for (let i = 0; i < numberOfCharacters; i++) {

packages/compiler-dom/package.json

@@ -13,7 +13,7 @@
   "sideEffects": false,
   "buildOptions": {
     "name": "VueDOMCompiler",
-    "formats": ["esm", "cjs", "global", "esm-browser"]
+    "formats": ["cjs", "global", "esm-browser"]
   },
   "repository": {
     "type": "git",

rollup.config.js

@@ -93,7 +93,8 @@ function createConfig(output, plugins = []) {
       compilerOptions: {
         declaration: shouldEmitDeclarations,
         declarationMap: shouldEmitDeclarations
-      }
+      },
+      exclude: ['**/__tests__']
     }
   })
   // we only need to check TS and generate declarations once for each build.

tsconfig.json

@@ -16,6 +16,7 @@
     "removeComments": false,
     "jsx": "react",
     "lib": ["esnext", "dom"],
+    "types": ["jest", "node"],
     "rootDir": ".",
     "paths": {
       "@vue/shared": ["packages/shared/src"],

yarn.lock

@@ -6475,6 +6475,11 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
+source-map@^0.7.3:
+  version "0.7.3"
+  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383"
+  integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==
+
 sourcemap-codec@^1.4.4:
   version "1.4.6"
   resolved "https://registry.yarnpkg.com/sourcemap-codec/-/sourcemap-codec-1.4.6.tgz#e30a74f0402bad09807640d39e971090a08ce1e9"