first commit

services/web/transform/cjs-to-esm/cjs-to-esm.mjs

@@ -0,0 +1,74 @@
+import minimist from 'minimist'
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+import Runner from 'jscodeshift/src/Runner.js'
+import { fileURLToPath } from 'node:url'
+import path from 'node:path'
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+// use minimist to get a list of files from the argv
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['usage'],
+})
+
+function printUsage() {
+  console.log(
+    'node scripts/esm-migration/cjs-to-esm.mjs [files] [--format] [--lint] [--usage]'
+  )
+  console.log(
+    'WARNING: this will only work in local development as important dependencies will be missing in production'
+  )
+  console.log('Options:')
+  console.log('   files: a list of files to convert')
+  console.log('--format: run prettier to fix formatting')
+  console.log('  --lint: run eslint to fix linting')
+  console.log(' --usage: show this help message')
+  process.exit(0)
+}
+
+const files = argv._
+
+if (argv.usage) {
+  printUsage()
+}
+
+if (!Array.isArray(files) || files.length === 0) {
+  console.error('You must provide a list of files to convert')
+  printUsage()
+  process.exit(1)
+}
+
+const promisifiedExec = promisify(exec)
+
+const cjsTransform = fileURLToPath(
+  import.meta.resolve('5to6-codemod/transforms/cjs.js')
+)
+const exportsTransform = fileURLToPath(
+  import.meta.resolve('5to6-codemod/transforms/exports.js')
+)
+const overleafTransform = fileURLToPath(
+  import.meta.resolve('./overleaf-es-codemod.js')
+)
+
+const config = {
+  output: __dirname,
+  silent: true,
+  print: false,
+  verbose: 0,
+  hoist: true,
+}
+
+await Runner.run(cjsTransform, files, config)
+await Runner.run(exportsTransform, files, config)
+await Runner.run(overleafTransform, files, config)
+
+const webRoot = fileURLToPath(new URL('../../', import.meta.url))
+
+for (const file of files) {
+  // move files with git mv
+  await promisifiedExec(`git mv ${file} ${file.replace('.js', '.mjs')}`)
+  const relativePath = path.relative(webRoot, file)
+  console.log(
+    `transformed ${relativePath} and renamed it to have a .mjs extension`
+  )
+}

services/web/transform/cjs-to-esm/overleaf-es-codemod.js

@@ -0,0 +1,230 @@
+// Performs a few useful codemod transformations for Overleaf's esm migration.
+// The transformations mostly address specific issues faced commonly in Overleaf's `web` service.
+// * Replaces `sandboxed-module` imports with `esmock` imports.
+// * Replaces `sandboxed-module` invocation with `esmock` invocation (Assumes `SandboxedModule.require` is used for the invocation).
+// * Fixes `mongodb-legacy` import to use `mongodb` import and extract `ObjectId` from the import.
+// * Replaces `require('path').join` with `path.join` (importing the path module if not already imported).
+// * Adds `const __dirname = fileURLToPath(new URL('.', import.meta.url))` if `__dirname` is used in the file.
+// * Adds `.js` or `.mjs` extension (as appropriate) to relative path imports.
+// call this with `jscodeshift -t overleaf-es-codemod.js <file>` or using the `cjs-to-esm.js` script (which does this as the final step before formatting).
+
+const fs = require('node:fs')
+const Path = require('node:path')
+
+module.exports = function (fileInfo, api) {
+  const j = api.jscodeshift
+  const root = j(fileInfo.source)
+  const body = root.get().value.program.body
+
+  /**
+   * Conditionally adds an import statement to the top of the file if it doesn't already exist.
+   * @param moduleName A plain text name for the module to import (e.g. 'node:path').
+   * @param specifier A jscodeshift specifier for the import statement (provides e.g. `{ promises }` from `import { promises } from 'fs'`.
+   * @param existingImportCheck A function that checks if a specific import statement is the one we're looking for.
+   */
+  function addImport(moduleName, specifier, existingImportCheck) {
+    // Add import path from 'path' at the top if not already present
+    const importDeclaration = j.importDeclaration(
+      specifier,
+      j.literal(moduleName)
+    )
+
+    if (!existingImportCheck) {
+      existingImportCheck = node => node.source.value === moduleName
+    }
+
+    const existingImport = body.find(
+      node => node.type === 'ImportDeclaration' && existingImportCheck(node)
+    )
+
+    if (!existingImport) {
+      const lastImportIndex = body.reduce((lastIndex, node, index) => {
+        return node.type === 'ImportDeclaration' ? index : lastIndex
+      }, -1)
+      body.splice(lastImportIndex, 0, importDeclaration)
+    }
+  }
+
+  // Replace sandboxed-module imports
+  root
+    .find(j.ImportDeclaration, {
+      source: { value: 'sandboxed-module' },
+    })
+    .forEach(path => {
+      path.node.source.value = 'esmock'
+      if (path.node.specifiers.length > 0 && path.node.specifiers[0].local) {
+        path.node.specifiers[0].local.name = 'esmock'
+      }
+    })
+
+  // Replace sandboxedModule.require calls with awaited esmock calls
+  root
+    .find(j.CallExpression, {
+      callee: {
+        object: { name: 'SandboxedModule' },
+        property: { name: 'require' },
+      },
+    })
+    .forEach(path => {
+      const args = path.node.arguments
+      if (args.length > 0) {
+        const firstArg = args[0]
+        const esmockArgs = [firstArg]
+
+        // Check if there's a second argument with a 'requires' property
+        if (args.length > 1 && args[1].type === 'ObjectExpression') {
+          const requiresProp = args[1].properties.find(
+            prop =>
+              prop.key.name === 'requires' || prop.key.value === 'requires'
+          )
+
+          if (requiresProp) {
+            // Move contents of 'requires' to top level
+            esmockArgs.push(requiresProp.value)
+          }
+        }
+
+        // Create the await expression with restructured arguments
+        const awaitExpression = j.awaitExpression(
+          j.callExpression(
+            j.memberExpression(j.identifier('esmock'), j.identifier('strict')),
+            esmockArgs
+          )
+        )
+
+        // Replace the original call with the await expression
+        j(path).replaceWith(awaitExpression)
+
+        // Find the closest function and make it async
+        let functionPath = path
+        while ((functionPath = functionPath.parent)) {
+          if (
+            functionPath.node.type === 'FunctionDeclaration' ||
+            functionPath.node.type === 'FunctionExpression' ||
+            functionPath.node.type === 'ArrowFunctionExpression'
+          ) {
+            functionPath.node.async = true
+            break
+          }
+        }
+      }
+    })
+
+  // Fix mongodb-legacy import
+  root
+    .find(j.ImportDeclaration, {
+      source: { value: 'mongodb-legacy' },
+      specifiers: [{ imported: { name: 'ObjectId' } }],
+    })
+    .forEach(path => {
+      // Create new import declaration
+      const newImport = j.importDeclaration(
+        [j.importDefaultSpecifier(j.identifier('mongodb'))],
+        j.literal('mongodb-legacy')
+      )
+
+      // Create new constant declaration
+      const newConst = j.variableDeclaration('const', [
+        j.variableDeclarator(
+          j.objectPattern([
+            j.property(
+              'init',
+              j.identifier('ObjectId'),
+              j.identifier('ObjectId')
+            ),
+          ]),
+          j.identifier('mongodb')
+        ),
+      ])
+
+      // Replace the old import with the new import and constant declaration
+      j(path).replaceWith(newImport)
+      path.insertAfter(newConst)
+    })
+
+  root
+    .find(j.CallExpression, {
+      callee: {
+        object: { callee: { name: 'require' }, arguments: [{ value: 'path' }] },
+        property: { name: 'join' },
+      },
+    })
+    .forEach(path => {
+      // Replace with path.join
+      j(path).replaceWith(
+        j.callExpression(
+          j.memberExpression(j.identifier('path'), j.identifier('join')),
+          path.node.arguments
+        )
+      )
+
+      // Add import path from 'path' at the top if not already presen
+      addImport(
+        'node:path',
+        [j.importDefaultSpecifier(j.identifier('path'))],
+        node =>
+          node.source.value === 'path' || node.source.value === 'node:path'
+      )
+    })
+
+  // Add const __dirname = fileURLToPath(new URL('.', import.meta.url)) if there is a usage of __dirname
+  const dirnameDeclaration = j.variableDeclaration('const', [
+    j.variableDeclarator(
+      j.identifier('__dirname'),
+      j.callExpression(j.identifier('fileURLToPath'), [
+        j.newExpression(j.identifier('URL'), [
+          j.literal('.'),
+          j.memberExpression(j.identifier('import'), j.identifier('meta.url')),
+        ]),
+      ])
+    ),
+  ])
+
+  const existingDirnameDeclaration = body.find(
+    node =>
+      node.type === 'VariableDeclaration' &&
+      node.declarations[0].id.name === '__dirname'
+  )
+  const firstDirnameUsage = root.find(j.Identifier, { name: '__dirname' }).at(0)
+
+  if (firstDirnameUsage.size() > 0 && !existingDirnameDeclaration) {
+    // Add import path from 'path' at the top if not already present
+    addImport(
+      'node:url',
+      [j.importSpecifier(j.identifier('fileURLToPath'))],
+      node => node.source.value === 'url' || node.source.value === 'node:url'
+    )
+
+    const lastImportIndex = body.reduce((lastIndex, node, index) => {
+      return node.type === 'ImportDeclaration' ? index : lastIndex
+    }, -1)
+
+    body.splice(lastImportIndex + 1, 0, dirnameDeclaration)
+  }
+
+  // Add extension to relative path imports
+  root
+    .find(j.ImportDeclaration)
+    .filter(path => path.node.source.value.startsWith('.'))
+    .forEach(path => {
+      const importPath = path.node.source.value
+      const fullPathJs = Path.resolve(
+        Path.dirname(fileInfo.path),
+        `${importPath}.js`
+      )
+      const fullPathMjs = Path.resolve(
+        Path.dirname(fileInfo.path),
+        `${importPath}.mjs`
+      )
+
+      if (fs.existsSync(fullPathJs)) {
+        path.node.source.value = `${importPath}.js`
+      } else if (fs.existsSync(fullPathMjs)) {
+        path.node.source.value = `${importPath}.mjs`
+      }
+    })
+
+  return root.toSource({
+    quote: 'single',
+  })
+}

services/web/transform/cjs-to-esm/transform-dir.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+  echo "Usage: transform-dir.sh <module_path>"
+  exit 1
+fi
+
+MODULE_PATH=$1
+
+while true; do
+  FILES=$(node scripts/esm-check-migration.mjs -f "$MODULE_PATH" -j | jq -r '.filesNotImportedViaCjs | join(" ")')
+  if [ -z "$FILES" ]; then
+    break
+  fi
+  node transform/cjs-to-esm/cjs-to-esm.mjs "$FILES"
+done
+
+make format_fix > /dev/null
+
+echo "All files processed."

services/web/transform/o-error/README.md

@@ -0,0 +1,5 @@
+# OError Transformer
+
+Run: `transform/o-error/transform.sh app ... `
+
+See: https://github.com/overleaf/web-internal/pull/2582 for discussion on next steps.

services/web/transform/o-error/transform.js

@@ -0,0 +1,126 @@
+function functionArgsFilter(j, path) {
+  if (path.get('params') && path.get('params').value[0]) {
+    return ['err', 'error'].includes(path.get('params').value[0].name)
+  } else {
+    return false
+  }
+}
+
+function isReturningFunctionCallWithError(path, errorVarName) {
+  return (
+    path.value.argument &&
+    path.value.argument.arguments &&
+    path.value.argument.arguments[0] &&
+    path.value.argument.arguments[0].name === errorVarName
+  )
+}
+
+function expressionIsLoggingError(path) {
+  return ['warn', 'error', 'err'].includes(
+    path.get('callee').get('property').value.name
+  )
+}
+
+function createTagErrorExpression(j, path, errorVarName) {
+  let message = 'error'
+  if (path.value.arguments.length >= 2) {
+    message = path.value.arguments[1].value || message
+  }
+
+  let info
+  try {
+    info = j.objectExpression(
+      // add properties from original logger info object to the
+      // OError info object, filtering out the err object itself,
+      // which is typically one of the args when doing intermediate
+      // error logging
+      // TODO: this can fail when the property name does not match
+      //       the variable name. e.g. { err: error } so need to check
+      //       both in the filter
+      path
+        .get('arguments')
+        .value[0].properties.filter(
+          property => property.key.name !== errorVarName
+        )
+    )
+  } catch (error) {
+    // if info retrieval fails it remains empty
+  }
+  const args = [j.identifier(errorVarName), j.literal(message)]
+  if (info) {
+    args.push(info)
+  }
+  return j.callExpression(
+    j.memberExpression(j.identifier('OError'), j.identifier('tag')),
+    args
+  )
+}
+
+function functionBodyProcessor(j, path) {
+  // the error variable should be the first parameter to the function
+  const errorVarName = path.get('params').value[0].name
+  j(path)
+    .find(j.IfStatement) // look for if statements
+    .filter(path =>
+      j(path)
+        // find returns inside the if statement where the error from
+        // the args is explicitly returned
+        .find(j.ReturnStatement)
+        .some(path => isReturningFunctionCallWithError(path, errorVarName))
+    )
+    .forEach(path => {
+      j(path)
+        .find(j.CallExpression, {
+          callee: {
+            object: { name: 'logger' },
+          },
+        })
+        .filter(path => expressionIsLoggingError(path))
+        .replaceWith(path => {
+          return createTagErrorExpression(j, path, errorVarName)
+        })
+    })
+}
+
+export default function transformer(file, api) {
+  const j = api.jscodeshift
+  let source = file.source
+  // apply transformer to declared functions
+  source = j(source)
+    .find(j.FunctionDeclaration)
+    .filter(path => functionArgsFilter(j, path))
+    .forEach(path => functionBodyProcessor(j, path))
+    .toSource()
+  // apply transformer to inline-functions
+  source = j(source)
+    .find(j.FunctionExpression)
+    .filter(path => functionArgsFilter(j, path))
+    .forEach(path => functionBodyProcessor(j, path))
+    .toSource()
+  // apply transformer to inline-arrow-functions
+  source = j(source)
+    .find(j.ArrowFunctionExpression)
+    .filter(path => functionArgsFilter(j, path))
+    .forEach(path => functionBodyProcessor(j, path))
+    .toSource()
+  // do a plain text search to see if OError is used but not imported
+  if (source.includes('OError') && !source.includes('@overleaf/o-error')) {
+    const root = j(source)
+    // assume the first variable declaration is an import
+    // TODO: this should check that there is actually a require/import here
+    //       but in most cases it will be
+    const imports = root.find(j.VariableDeclaration)
+    const importOError = "const OError = require('@overleaf/o-error')\n"
+    // if there were imports insert into list, format can re-order
+    if (imports.length) {
+      j(imports.at(0).get()).insertAfter(importOError)
+    }
+    // otherwise insert at beginning
+    else {
+      root.get().node.program.body.unshift(importOError)
+    }
+    source = root.toSource()
+  }
+
+  return source
+}

services/web/transform/o-error/transform.sh

@@ -0,0 +1,15 @@
+# run tranformer
+npx jscodeshift \
+  -t transform/o-error/transform.js \
+  --ignore-pattern=frontend/js/libraries.js \
+  --ignore-pattern=frontend/js/vendor \
+  "$1"
+# replace blank lines in staged changed with token
+git diff --ignore-all-space --ignore-blank-lines | sed 's/^\+$/\+REMOVE_ME_IM_A_BLANK_LINE/g' | git apply --reject --cached --ignore-space-change
+# stage changes with token instead of blank line
+git checkout .
+git add -A
+# delete line containing token in staged files
+git diff --cached --name-only | xargs sed -i '/^REMOVE_ME_IM_A_BLANK_LINE$/d'
+# fix format on modified files
+make format_fix