first commit

services/web/bin/cdn_upload

@@ -0,0 +1,73 @@
+#!/bin/bash
+set -eEu
+
+function upload_into_bucket() {
+  bucket=$1
+
+  # stylesheets
+  bin/cdn_upload_batch 'text/css' "$bucket" '.css' \
+    -x '.+(?<!\.css)$' &
+
+  # javascript files
+  bin/cdn_upload_batch 'application/javascript' "$bucket" '.js' \
+    -x '.+(?<!\.js)$' &
+
+  # the rest
+  bin/cdn_upload_batch '-' "$bucket" '-' \
+    -x '.+\.(css|js)$' &
+
+  wait
+}
+
+verify_upload_into_bucket() {
+  local bucket
+  local missing_from_bucket
+  bucket=$1
+  printf '\nINFO: Verifying file availability in %s.\n' "$bucket"
+  readarray -t missing_from_bucket < <(
+    comm -13 \
+      <(gsutil ls "${bucket}/public/**" | sed "s@${bucket}/@@" | sort) \
+      <(find /tmp/public /tmp/compressed -type f | sed '
+          # Remove absolute path prefix
+          s@^/tmp/@@;
+          # Undo the compressed/ directory separation that does not exist in the bucket
+          s@^compressed/@@
+        ' | sort)
+  )
+  if [[ ${#missing_from_bucket[@]} -eq 0 ]]; then
+    printf 'INFO: Verification successful: all local files have been found in bucket %s.\n' \
+      "$bucket"
+  else
+    printf >&2 'WARN: %d local file(s) not available in bucket %s:\n' \
+      ${#missing_from_bucket[@]} "$bucket"
+    printf >&2 ' - %s\n' "${missing_from_bucket[@]}"
+    return 1
+  fi
+}
+
+# Upload to staging CDN if branch is either 'main' or 'staging-main'
+if [[ "$BRANCH_NAME" == "main" ]] || [[ "$BRANCH_NAME" == "staging-main" ]]; then
+  tar --directory=/tmp/ -xf build.tar
+
+  # delete source maps
+  find /tmp/public -name '*.js.map' -delete
+
+  bin/compress_assets
+
+  upload_into_bucket "$CDN_STAG" &&
+    verify_upload_into_bucket "$CDN_STAG" || exit 3 &
+  pid_staging=$! # record pid of the detached process "upload && verify || exit 3")
+
+  pid_production=
+  # Only upload to production CDN if branch is
+  if [[ "$BRANCH_NAME" == "main" ]]; then
+    upload_into_bucket "$CDN_PROD" &&
+      verify_upload_into_bucket "$CDN_PROD" || exit 4 &
+    pid_production=$! # record pid of the detached process "upload && verify || exit 4")
+  fi
+
+  wait "$pid_staging" # wait for staging upload to finish, wait(1) will exit if the upload failed
+  if [[ -n "$pid_production" ]]; then
+    wait "$pid_production" # wait for production upload to finish (if started), wait(1) will exit if the upload failed
+  fi
+fi

services/web/bin/cdn_upload_batch

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+content_type=$1
+bucket=$2
+text_extension=$3
+shift 3
+content_type_options=""
+if [[ "$content_type" != "-" ]]; then
+  content_type_options="-h Content-Type:${content_type};charset=utf-8"
+fi
+
+# DOCS for gsutil -- it does not have long command line flags!
+## global flags
+# -h NAME:VALUE   add header, can occur multiples times
+# -m              upload with multiple threads
+## rsync flags
+# -c              use checksums for determining changed files (mtime is not stable)
+# -r              traverse into directories recursively
+# -x              Python regex for excluding files from the sync
+if [[ "$text_extension" == "-" || $(find /tmp/public -type f -name "*$text_extension" | wc -l) != "0" ]]; then
+  # Potentially skip upload of non-compressed .js/.css files.
+  # shellcheck disable=SC2086
+  gsutil \
+    -h "Cache-Control:public, max-age=31536000" \
+    ${content_type_options} \
+    -m \
+    rsync \
+    -c \
+    -r \
+    "$@" \
+    "/tmp/public/" \
+    "${bucket}/public/"
+fi
+
+# shellcheck disable=SC2086
+gsutil \
+  -h "Cache-Control:public, max-age=31536000" \
+  -h "Content-Encoding:gzip" \
+  ${content_type_options} \
+  -m \
+  rsync \
+  -c \
+  -r \
+  "$@" \
+  "/tmp/compressed/public/" \
+  "${bucket}/public/"

services/web/bin/check_extracted_translations

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Ensure all locales used in the frontend are tracked
+OUTPUT=data/dumpFolder/i18next-scanner
+trap 'rm -rf "$OUTPUT"' EXIT
+npx i18next-scanner --output "$OUTPUT"
+ACTUAL=frontend/extracted-translations.json
+EXPECTED="$OUTPUT/frontend/extracted-translations.json"
+if ! diff "$ACTUAL" "$EXPECTED"; then
+  cat <<MSG >&2
+
+services/web/frontend/extracted-translations.json is not up-to-date.
+
+---
+Try running:
+
+  internal$ bin/run web npm run extract-translations
+
+---
+MSG
+  exit 1
+fi

services/web/bin/compress_assets

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+SCRIPT_PATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
+
+pushd /tmp
+
+find public/ -type d | sed 's!^!compressed/!' | xargs mkdir --parents
+
+find public/ -type f -print0 \
+| xargs --null --max-args 20 --max-procs 10 "$SCRIPT_DIR/compress_batch_of_assets"
+
+popd

services/web/bin/compress_batch_of_assets

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+for file in "$@"; do
+  file_gzipped="compressed/$file"
+
+  gzip -9 --no-name --stdout "$file" > "$file_gzipped"
+
+  before=$(stat -c%s "$file")
+  after=$(stat -c%s "$file_gzipped")
+  if [[ "$after" -ge "$before" ]]; then
+    rm "$file_gzipped"
+  else
+    rm "$file"
+  fi
+done

services/web/bin/copy_external_pages

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# Branding
+mv app/views/external/robots.txt public/robots.txt
+mv app/views/external/googlebdb0f8f7f4a17241.html public/googlebdb0f8f7f4a17241.html

services/web/bin/lint_flag_res_send_usage

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+POTENTIAL_SEND_USAGE=$(\
+  grep \
+    --files-with-matches \
+    --recursive \
+      app.mjs \
+      app/ \
+      modules/*/app \
+      test/acceptance/ \
+      modules/*/test/acceptance/ \
+    --regex "\.send\b" \
+    --regex "\bsend(" \
+)
+HELPER_MODULE="app/src/infrastructure/Response.js"
+if [[ "$POTENTIAL_SEND_USAGE" == "$HELPER_MODULE" ]]; then
+  exit 0
+fi
+
+for file in ${POTENTIAL_SEND_USAGE}; do
+  if [[ "$file" == "$HELPER_MODULE" ]]; then
+    continue
+  fi
+
+  cat <<MSG >&2
+
+ERROR: $file contains a potential use of 'res.send'.
+
+---
+$(grep -n -C 3  "$file" --regex "\.send\b" --regex "\bsend(")
+---
+
+Using 'res.send' is prone to introducing XSS vulnerabilities.
+
+Consider using 'res.json' or one of the helpers in $HELPER_MODULE.
+
+If this is a false-positive, consider using a more specific name than 'send'
+ for your newly introduced function.
+
+Links:
+ - https://github.com/overleaf/internal/issues/6268
+
+MSG
+  exit 1
+done

services/web/bin/lint_locales

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+# Ensure all locale files are sorted.
+node scripts/translations/sort.js --check
+
+# Ensure all locales are still in use
+node scripts/translations/cleanupUnusedLocales.js --check
+
+# Ensure all locales use the same variables
+node scripts/translations/checkVariables.js --ignore-orphaned-translations
+
+# Ensure no locales contain single quotes.
+LOCALES_WITH_SINGLE_QUOTE=$(\
+  grep \
+    --files-with-matches \
+    --recursive locales/ \
+    --regex "'" \
+  || true
+)
+
+for file in ${LOCALES_WITH_SINGLE_QUOTE}; do
+  cat <<MSG >&2
+
+ERROR: $file contains a locale with a single quote.
+
+---
+$(grep "$file" --regex "'")
+---
+
+Using single quotes in locales can lead to Angular XSS.
+
+You will need to replace the quote with a similar looking character.
+’ (\u2019) is a good candidate.
+
+Links:
+ - https://en.wikipedia.org/wiki/Right_single_quotation_mark
+ - https://github.com/overleaf/issues/issues/4478
+
+MSG
+  exit 1
+done

services/web/bin/lint_pug_templates

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -e
+
+TEMPLATES_EXTENDING_META_BLOCK=$(\
+  grep \
+    --files-with-matches \
+    --recursive app/views modules/*/app/views \
+    --regex 'block append meta' \
+    --regex 'block prepend meta' \
+    --regex 'append meta' \
+    --regex 'prepend meta' \
+)
+
+for file in ${TEMPLATES_EXTENDING_META_BLOCK}; do
+    if ! grep "$file" --quiet --extended-regexp -e 'extends .+layout'; then
+      cat <<MSG >&2
+
+ERROR: $file is a partial template and extends 'block meta'.
+
+Using block append/prepend in a partial will duplicate the block contents into
+ the <body> due to a bug in pug.
+Putting meta tags in the <body> can lead to Angular XSS.
+
+You will need to refactor the partial and move the block into the top level
+ page template that extends the global layout.pug.
+
+MSG
+      exit 1
+    fi
+done

services/web/bin/push-translations-changes.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
+WEB_DIR=$(dirname "$SCRIPT_DIR")
+
+cd "$WEB_DIR"
+
+if [[ $(git status --porcelain=2 locales/) ]]; then
+  git add locales/*
+  git commit -m "auto update translation"
+  # Switch the cloudbuild clone from https to ssh authentication.
+  git remote set-url --push origin git@github.com:overleaf/internal.git
+  git push origin "HEAD:$BRANCH_NAME"
+else
+  echo 'No changes'
+fi

services/web/bin/routes

@@ -0,0 +1,134 @@
+#! /usr/bin/env node
+
+const acorn = require('acorn')
+const acornWalk = require('acorn-walk')
+const fs = require('fs')
+const _ = require('lodash')
+const glob = require('glob')
+const escodegen = require('escodegen')
+const print = console.log
+
+const Methods = new Set([
+  'get',
+  'head',
+  'post',
+  'put',
+  'delete',
+  'connect',
+  'options',
+  'trace',
+  'patch',
+])
+
+const isMethod = str => {
+  return Methods.has(str)
+}
+
+// Check if the expression is a call on a router, return data about it, or null
+const routerCall = callExpression => {
+  const callee = callExpression.callee
+  const property = callee.property
+  const args = callExpression.arguments
+  if (!callee.object || !callee.object.name) {
+    return false
+  }
+  const routerName = callee.object.name
+  if (
+    // Match known names for the Express routers: app, webRouter, whateverRouter, etc...
+    isMethod(property.name) &&
+    (routerName === 'app' || routerName.match('^.*[rR]outer$'))
+  ) {
+    return {
+      routerName,
+      method: property.name,
+      args,
+    }
+  } else {
+    return null
+  }
+}
+
+const formatMethodCall = expression => {
+  return escodegen.generate(expression, { format: { compact: true } })
+}
+
+const parseAndPrintRoutesSync = path => {
+  const content = fs.readFileSync(path)
+  // Walk the AST (Abstract Syntax Tree)
+  acornWalk.simple(
+    acorn.parse(content, { sourceType: 'module', ecmaVersion: 2020 }),
+    {
+      // We only care about call expression ( like `a.b()` )
+      CallExpression(node) {
+        const call = routerCall(node)
+        if (call) {
+          const firstArg = _.first(call.args)
+          try {
+            print(
+              `  ${formatRouterName(call.routerName)}\t .${call.method} \t: ${
+                firstArg.value
+              } => ${call.args.slice(1).map(formatMethodCall).join(' => ')}`
+            )
+          } catch (e) {
+            print('>> Error')
+            print(e)
+            print(JSON.stringify(call))
+            process.exit(1)
+          }
+        }
+      },
+    }
+  )
+}
+
+const routerNameMapping = {
+  privateApiRouter: 'privateApi',
+  publicApiRouter: 'publicApi',
+}
+const formatRouterName = name => {
+  return routerNameMapping[name] || name
+}
+
+const main = () => {
+  // Take an optional filter to apply to file names
+  const filter = process.argv[2] || null
+
+  if (filter && (filter === '--help' || filter === 'help')) {
+    print('')
+    print('  Usage: bin/routes [filter]')
+    print('  Examples:')
+    print('    bin/routes')
+    print('    bin/routes GitBridge')
+    print('')
+    process.exit(0)
+  }
+
+  // Find all routers
+  glob('*[rR]outer.*js', { matchBase: true }, (err, files) => {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    for (const file of files) {
+      if (file.match('^node_modules.*$') || file.match('.*/public/.*')) {
+        continue
+      }
+      // Restrict to the filter (if filter is present)
+      if (filter && !file.match(`.*${filter}.*`)) {
+        continue
+      }
+      print(`[${file}]`)
+      try {
+        parseAndPrintRoutesSync(file)
+      } catch (_e) {
+        print('>> Error parsing file')
+        continue
+      }
+    }
+    process.exit(0)
+  })
+}
+
+if (require.main === module) {
+  main()
+}

services/web/bin/run

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+pushd ..
+bin/run "$*"
+RV=$?
+popd || exit 1
+
+exit $RV

services/web/bin/sentry_upload

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+
+if [[ "$BRANCH_NAME" == "master" || "$BRANCH_NAME" == "main" ]]; then
+  rm -rf sentry_upload
+  mkdir sentry_upload
+  tar --directory sentry_upload -xf build.tar
+  cd sentry_upload/public
+
+  SENTRY_RELEASE=${COMMIT_SHA}
+  sentry-cli releases new "$SENTRY_RELEASE"
+  sentry-cli releases set-commits --auto "$SENTRY_RELEASE"
+  sentry-cli sourcemaps upload --release="$SENTRY_RELEASE" .
+  sentry-cli releases finalize "$SENTRY_RELEASE"
+
+  cd ../..
+  rm -rf sentry_upload
+fi