first commit

server-ce/init_scripts/000_check_for_old_bind_mounts_5.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -e
+
+POTENTIAL_OLD_PATHS="
+/etc/sharelatex
+/var/lib/sharelatex
+/var/log/sharelatex
+"
+
+OLD_ITEMS=""
+for path in ${POTENTIAL_OLD_PATHS}; do
+  if [[ -e "$path" ]]; then
+    OLD_ITEMS="$OLD_ITEMS $path"
+  fi
+done
+
+if [[ "$OLD_ITEMS" == "" ]]; then
+  exit 0
+fi
+
+OLD_ITEMS=$(echo "$OLD_ITEMS" | xargs -n1 | sed 's/^/      - /')
+N=$(echo "$OLD_ITEMS" | wc -l)
+cat <<EOF
+------------------------------------------------------------------------
+
+                   ShareLaTeX to Overleaf rebranding
+                   ---------------------------------
+
+  Starting with version 5.0, ShareLaTeX branded file system paths are no
+   longer supported as we are migrating to the Overleaf brand.
+
+  Your configuration still uses $N ShareLaTeX branded paths:
+$OLD_ITEMS
+
+  Please update the in-container bind-mount target to refer to the
+   Overleaf equivalent:
+    - /etc/sharelatex     -> /etc/overleaf
+    - /var/lib/sharelatex -> /var/lib/overleaf
+    - /var/log/sharelatex -> /var/log/overleaf
+
+  Overleaf toolkit setups:
+
+    github.com/overleaf/toolkit$ bin/upgrade
+
+
+  Legacy docker compose setups/Horizontal scaling setups:
+
+    before:
+
+      services:
+        sharelatex:
+          volumes:
+           - /my/docker-host/path:/var/lib/sharelatex
+
+    after:
+
+      services:
+        sharelatex:
+          volumes:
+           - /my/docker-host/path:/var/lib/overleaf
+
+
+  Other deployment methods:
+
+    Adapt the docker compose example or get in touch with support.
+
+
+  Server Pro: Please update SANDBOXED_COMPILES_HOST_DIR if needed.
+
+
+  Refusing to startup, exiting in 10s.
+
+------------------------------------------------------------------------
+EOF
+
+sleep 10
+exit 101

server-ce/init_scripts/000_check_for_old_env_vars_5.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+OLD_ITEMS=$(env | cut -d '=' -f1 | grep SHARELATEX | sed 's/^/      - /')
+
+if [[ "$OLD_ITEMS" == "" ]]; then
+  exit 0
+fi
+
+N=$(echo "$OLD_ITEMS" | wc -l)
+cat <<EOF
+------------------------------------------------------------------------
+
+                   ShareLaTeX to Overleaf rebranding
+                   ---------------------------------
+
+  Starting with version 5.0, ShareLaTeX branded variables are no
+   longer supported as we are migrating to the Overleaf brand.
+
+  Your configuration still uses $N ShareLaTeX environment variables:
+$OLD_ITEMS
+
+  Please either replace them with the "OVERLEAF_" prefix,
+   e.g. SHARELATEX_MONGO_URL -> OVERLEAF_MONGO_URL, or
+   remove old entries from your configuration.
+
+  You can use the following script for migrating your config.
+
+  Overleaf toolkit setups:
+
+    github.com/overleaf/toolkit$ bin/upgrade
+    github.com/overleaf/toolkit$ bin/rename-env-vars-5-0.sh
+
+
+  Legacy docker compose setups/Horizontal scaling setups:
+
+    github.com/overleaf/overleaf$ git pull
+    github.com/overleaf/overleaf$ server-ce/bin/rename-env-vars-5-0.sh
+
+    # When using a docker-compose.override.yml file (or other file name):
+    github.com/overleaf/overleaf$ server-ce/bin/rename-env-vars-5-0.sh docker-compose.override.yml
+
+
+  Other deployment methods:
+
+    Try using the docker compose script or get in touch with support.
+
+
+  Refusing to startup, exiting in 10s.
+
+------------------------------------------------------------------------
+EOF
+
+sleep 10
+exit 101

server-ce/init_scripts/100_generate_secrets.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e -o pipefail
+
+# generate secrets and defines them as environment variables
+# https://github.com/phusion/baseimage-docker#centrally-defining-your-own-environment-variables
+
+WEB_API_PASSWORD_FILE=/etc/container_environment/WEB_API_PASSWORD
+STAGING_PASSWORD_FILE=/etc/container_environment/STAGING_PASSWORD # HTTP auth for history-v1
+V1_HISTORY_PASSWORD_FILE=/etc/container_environment/V1_HISTORY_PASSWORD
+CRYPTO_RANDOM_FILE=/etc/container_environment/CRYPTO_RANDOM
+OT_JWT_AUTH_KEY_FILE=/etc/container_environment/OT_JWT_AUTH_KEY
+
+generate_secret () {
+  dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w 0 | rev | cut -b 2- | rev | tr -d '\n+/'
+}
+
+if [ ! -f "$WEB_API_PASSWORD_FILE" ] ||
+  [ ! -f "$STAGING_PASSWORD_FILE" ] ||
+  [ ! -f "$V1_HISTORY_PASSWORD_FILE" ] ||
+  [ ! -f "$CRYPTO_RANDOM_FILE" ] ||
+  [ ! -f "$OT_JWT_AUTH_KEY_FILE" ]
+then
+    echo "generating random secrets"
+
+    SECRET=$(generate_secret)
+    echo "${SECRET}" > ${WEB_API_PASSWORD_FILE}
+
+    SECRET=$(generate_secret)
+    echo "${SECRET}" > ${STAGING_PASSWORD_FILE}
+    echo "${SECRET}" > ${V1_HISTORY_PASSWORD_FILE}
+
+    SECRET=$(generate_secret)
+    echo "${SECRET}" > ${CRYPTO_RANDOM_FILE}
+
+    SECRET=$(generate_secret)
+    echo "${SECRET}" > ${OT_JWT_AUTH_KEY_FILE}
+fi
+

server-ce/init_scripts/100_make_overleaf_data_dirs.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+set -e
+
+mkdir -p /var/lib/overleaf/data
+chown www-data:www-data /var/lib/overleaf/data
+
+mkdir -p /var/lib/overleaf/data/user_files
+chown www-data:www-data /var/lib/overleaf/data/user_files
+
+mkdir -p /var/lib/overleaf/data/compiles
+chown www-data:www-data /var/lib/overleaf/data/compiles
+
+mkdir -p /var/lib/overleaf/data/output
+chown www-data:www-data /var/lib/overleaf/data/output
+
+mkdir -p /var/lib/overleaf/data/cache
+chown www-data:www-data /var/lib/overleaf/data/cache
+
+mkdir -p /var/lib/overleaf/data/template_files
+chown www-data:www-data /var/lib/overleaf/data/template_files
+
+mkdir -p /var/lib/overleaf/data/history
+chown www-data:www-data /var/lib/overleaf/data/history
+
+mkdir -p /var/lib/overleaf/tmp/projectHistories
+chown www-data:www-data /var/lib/overleaf/tmp/projectHistories
+
+mkdir -p /var/lib/overleaf/tmp/dumpFolder
+chown www-data:www-data /var/lib/overleaf/tmp/dumpFolder
+
+mkdir -p /var/lib/overleaf/tmp
+chown www-data:www-data /var/lib/overleaf/tmp
+
+mkdir -p /var/lib/overleaf/tmp/uploads
+chown www-data:www-data /var/lib/overleaf/tmp/uploads
+
+mkdir -p /var/lib/overleaf/tmp/dumpFolder
+chown www-data:www-data /var/lib/overleaf/tmp/dumpFolder

server-ce/init_scripts/100_restore_site_status.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# pre-shutdown scripts close the site by overriding the content of SITE_MAINTENANCE_FILE,
+# this script restores the original value on container restart
+SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown"
+
+if [ -f "${SITE_MAINTENANCE_FILE_BAK}" ]; then
+    mv -f "${SITE_MAINTENANCE_FILE_BAK}" "${SITE_MAINTENANCE_FILE}"
+    rm -f "${SITE_MAINTENANCE_FILE_BAK}"
+fi

server-ce/init_scripts/100_set_docker_host_ipaddress.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e -o pipefail
+
+# See the bottom of http://stackoverflow.com/questions/24319662/from-inside-of-a-docker-container-how-do-i-connect-to-the-localhost-of-the-mach
+echo "$(route -n | awk '/UG[ \t]/{print $2}') dockerhost" >> /etc/hosts

server-ce/init_scripts/200_nginx_config_template.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+## Generate nginx config files from templates,
+## with environment variables substituted
+
+nginx_dir='/etc/nginx'
+nginx_templates_dir="${nginx_dir}/templates"
+
+if ! [ -d "${nginx_templates_dir}" ]; then
+  echo "Nginx: no template directory found, skipping"
+  exit 0
+fi
+
+nginx_template_file="${nginx_templates_dir}/nginx.conf.template"
+nginx_config_file="${nginx_dir}/nginx.conf"
+
+if [ -f "${nginx_template_file}" ]; then
+  export NGINX_KEEPALIVE_TIMEOUT="${NGINX_KEEPALIVE_TIMEOUT:-65}"
+  export NGINX_WORKER_CONNECTIONS="${NGINX_WORKER_CONNECTIONS:-768}"
+  export NGINX_WORKER_PROCESSES="${NGINX_WORKER_PROCESSES:-4}"
+
+  echo "Nginx: generating config file from template"
+
+  # Note the single-quotes, they are important.
+  # This is a pass-list of env-vars that envsubst
+  # should operate on.
+  # shellcheck disable=SC2016
+  envsubst '
+    ${NGINX_KEEPALIVE_TIMEOUT}
+    ${NGINX_WORKER_CONNECTIONS}
+    ${NGINX_WORKER_PROCESSES}
+  ' \
+    < "${nginx_template_file}" \
+    > "${nginx_config_file}"
+
+  echo "Checking Nginx config"
+  nginx -t
+
+  echo "Nginx: reloading config"
+  service nginx reload
+fi

server-ce/init_scripts/300_delete_old_logs.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+
+# Up to version 2.5.0 the logs of the contacts service were written into a
+#  file that was not picked up by logrotate.
+# The service is stable and we can safely discard any logs.
+rm -vf /var/log/overleaf/contacts

server-ce/init_scripts/500_check_db_access.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+echo "Checking can connect to mongo and redis"
+cd /overleaf/services/web
+node modules/server-ce-scripts/scripts/check-mongodb.mjs
+node modules/server-ce-scripts/scripts/check-redis.mjs
+echo "All checks passed"

server-ce/init_scripts/900_run_web_migrations.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${OVERLEAF_IS_SERVER_PRO:-null}" == "true" ]]; then
+  environment="server-pro"
+else
+  environment="server-ce"
+fi
+
+echo "Running migrations for $environment"
+cd /overleaf/services/web
+npm run migrations -- migrate -t "$environment"
+echo "Finished migrations"

server-ce/init_scripts/910_check_texlive_images

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+
+echo "Checking texlive images"
+cd /overleaf/services/web
+node modules/server-ce-scripts/scripts/check-texlive-images.mjs

server-ce/init_scripts/910_initiate_doc_version_recovery

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source /etc/container_environment.sh
+source /etc/overleaf/env.sh
+
+LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
+RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-5.0.3.log
+
+echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
+cd /overleaf/services/history-v1
+LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
+
+function resyncAllProjectsInBackground() {
+  waitForService docstore 3016
+  waitForService document-updater 3003
+  waitForService filestore 3009
+  waitForService history-v1 3100
+  waitForService project-history 3054
+  waitForService web-api 4000
+
+  # Resync files that had their versions updated
+  while read -r project_id; do
+    echo "Resyncing project $project_id..."
+    curl -X POST --silent "http://127.0.0.1:3054/project/$project_id/resync?force=true"
+  done < "$RESYNCS_NEEDED_FILE"
+
+  # Resync files that have broken histories
+  /overleaf/bin/force-history-resyncs
+
+  echo "Finished resyncing history for all projects. Adding .done suffix to log file"
+  mv "$RESYNCS_NEEDED_FILE" "$RESYNCS_NEEDED_FILE.done"
+}
+
+function waitForService() {
+  local name=$1
+  local port=$2
+  while ! curl --fail --silent "http://127.0.0.1:$port/status"; do
+    echo "Waiting for $name service to start up"
+    sleep 10
+  done
+}
+
+if [ -f "$RESYNCS_NEEDED_FILE" ]; then
+  echo "Finished recovery of doc versions. Resyncing history for all projects in the background."
+  resyncAllProjectsInBackground &
+else
+  echo "No recovery of doc versions needed."
+fi