first commit
This commit is contained in:
53
services/git-bridge/.gitignore
vendored
Normal file
53
services/git-bridge/.gitignore
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# Let's not share anything because we're using Maven.
|
||||
|
||||
.idea
|
||||
*.iml
|
||||
|
||||
# User-specific stuff:
|
||||
.idea/workspace.xml
|
||||
.idea/tasks.xml
|
||||
.idea/dictionaries
|
||||
.idea/vcs.xml
|
||||
.idea/jsLibraryMappings.xml
|
||||
|
||||
# Sensitive or high-churn files:
|
||||
.idea/dataSources.ids
|
||||
.idea/dataSources.xml
|
||||
.idea/dataSources.local.xml
|
||||
.idea/sqlDataSources.xml
|
||||
.idea/dynamic.xml
|
||||
.idea/uiDesigner.xml
|
||||
|
||||
# Gradle:
|
||||
.idea/gradle.xml
|
||||
.idea/libraries
|
||||
|
||||
# Mongo Explorer plugin:
|
||||
.idea/mongoSettings.xml
|
||||
|
||||
## File-based project format:
|
||||
*.iws
|
||||
|
||||
## Plugin-specific files:
|
||||
|
||||
# IntelliJ
|
||||
/out/
|
||||
target/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Local configuration files
|
||||
conf/runtime.json
|
48
services/git-bridge/Dockerfile
Normal file
48
services/git-bridge/Dockerfile
Normal file
@@ -0,0 +1,48 @@
|
||||
# Dockerfile for git-bridge
|
||||
|
||||
FROM maven:3-amazoncorretto-21-debian AS base
|
||||
|
||||
RUN apt-get update && apt-get install -y make git sqlite3 \
|
||||
&& rm -rf /var/lib/apt/lists
|
||||
|
||||
COPY vendor/envsubst /opt/envsubst
|
||||
RUN chmod +x /opt/envsubst
|
||||
|
||||
RUN useradd --create-home node
|
||||
|
||||
FROM base AS builder
|
||||
|
||||
COPY . /app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN make package \
|
||||
# The name of the created jar contains the current version tag.
|
||||
# Rename it to a static path that can be used for copying.
|
||||
&& find /app/target \
|
||||
-name 'writelatex-git-bridge*jar-with-dependencies.jar' \
|
||||
-exec mv {} /git-bridge.jar \;
|
||||
|
||||
FROM amazoncorretto:21-alpine
|
||||
|
||||
RUN apk add --update --no-cache bash git sqlite procps htop net-tools jemalloc util-linux
|
||||
|
||||
ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2
|
||||
|
||||
RUN adduser -D node
|
||||
|
||||
COPY --from=builder /git-bridge.jar /
|
||||
|
||||
COPY vendor/envsubst /opt/envsubst
|
||||
RUN chmod +x /opt/envsubst
|
||||
|
||||
COPY conf/envsubst_template.json envsubst_template.json
|
||||
COPY start.sh start.sh
|
||||
COPY server-pro-start.sh server-pro-start.sh
|
||||
|
||||
RUN mkdir conf
|
||||
RUN chown node:node conf
|
||||
|
||||
USER node
|
||||
|
||||
CMD ["/start.sh"]
|
22
services/git-bridge/LICENSE
Normal file
22
services/git-bridge/LICENSE
Normal file
@@ -0,0 +1,22 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2014 Winston Li
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
40
services/git-bridge/Makefile
Normal file
40
services/git-bridge/Makefile
Normal file
@@ -0,0 +1,40 @@
|
||||
# git-bridge makefile
|
||||
|
||||
MVN_OPTS := --no-transfer-progress
|
||||
MVN_TARGET := target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar
|
||||
|
||||
runtime-conf:
|
||||
/opt/envsubst < conf/envsubst_template.json > conf/runtime.json
|
||||
|
||||
|
||||
run: $(MVN_TARGET) runtime-conf
|
||||
java $(GIT_BRIDGE_JVM_ARGS) -jar $(MVN_TARGET) conf/runtime.json
|
||||
|
||||
|
||||
$(MVN_TARGET): $(shell find src -type f) pom.xml
|
||||
mvn $(MVN_OPTS) package -DskipTests
|
||||
|
||||
build: $(MVN_TARGET)
|
||||
|
||||
|
||||
format:
|
||||
mvn $(MVN_OPTS) com.spotify.fmt:fmt-maven-plugin:check
|
||||
|
||||
|
||||
format_fix:
|
||||
mvn $(MVN_OPTS) com.spotify.fmt:fmt-maven-plugin:format
|
||||
|
||||
|
||||
test:
|
||||
mvn $(MVN_OPTS) test
|
||||
|
||||
|
||||
clean:
|
||||
mvn $(MVN_OPTS) clean
|
||||
|
||||
|
||||
package: clean
|
||||
mvn $(MVN_OPTS) package -DskipTests
|
||||
|
||||
|
||||
.PHONY: run package build clean test runtime-conf
|
136
services/git-bridge/README.md
Normal file
136
services/git-bridge/README.md
Normal file
@@ -0,0 +1,136 @@
|
||||
# writelatex-git-bridge
|
||||
|
||||
## Docker
|
||||
|
||||
The `Dockerfile` contains all the requirements for building and running the
|
||||
writelatex-git-bridge.
|
||||
|
||||
```bash
|
||||
# build the image
|
||||
docker build -t writelatex-git-bridge .
|
||||
|
||||
# run it with the demo config
|
||||
docker run -v `pwd`/conf/local.json:/conf/runtime.json writelatex-git-bridge
|
||||
```
|
||||
|
||||
## Native install
|
||||
|
||||
### Required packages
|
||||
|
||||
* `maven` (for building, running tests and packaging)
|
||||
* `jdk-8` (for compiling and running)
|
||||
|
||||
### Commands
|
||||
|
||||
To be run from the base directory:
|
||||
|
||||
**Build jar**:
|
||||
`mvn package`
|
||||
|
||||
**Run tests**:
|
||||
`mvn test`
|
||||
|
||||
**Clean**:
|
||||
`mvn clean`
|
||||
|
||||
To be run from the dev-environment:
|
||||
|
||||
**Build jar**:
|
||||
`bin/run git-bridge make package`
|
||||
|
||||
**Run tests**:
|
||||
`bin/run git-bridge make test`
|
||||
|
||||
**Clean**:
|
||||
`bin/run git-bridge make clean`
|
||||
|
||||
### Installation
|
||||
|
||||
Install dependencies:
|
||||
|
||||
```
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y maven
|
||||
sudo apt-get install -y openjdk-8-jdk
|
||||
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
|
||||
sudo update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/javac
|
||||
```
|
||||
|
||||
Create a config file according to the format below.
|
||||
|
||||
Run `mvn package` to build, test, and package it into a jar at `target/writelatex-git-bridge-1.0-SNAPSHOT-jar-with-dependencies.jar`.
|
||||
|
||||
Use `java -jar <path_to_jar> <path_to_config_file>` to run the server.
|
||||
|
||||
## Runtime Configuration
|
||||
|
||||
The configuration file is in `.json` format.
|
||||
|
||||
{
|
||||
"port" (int): the port number,
|
||||
"rootGitDirectory" (string): the directory in which to store
|
||||
git repos and the db/atts,
|
||||
"apiBaseUrl" (string): base url for the snapshot api,
|
||||
"username" (string, optional): username for http basic auth,
|
||||
"password" (string, optional): password for http basic auth,
|
||||
"postbackBaseUrl" (string): the postback url,
|
||||
"serviceName" (string): current name of writeLaTeX
|
||||
in case it ever changes,
|
||||
"oauth2Server" (string): oauth2 server,
|
||||
with protocol and
|
||||
without trailing slash,
|
||||
null or missing if oauth2 shouldn't be used
|
||||
},
|
||||
"repoStore" (object, optional): { configure the repo store
|
||||
"maxFileSize" (long, optional): maximum size of a file, inclusive
|
||||
},
|
||||
"swapStore" (object, optional): { the place to swap projects to.
|
||||
if null, type defaults to
|
||||
"noop"
|
||||
"type" (string): "s3", "memory", "noop" (not recommended),
|
||||
"awsAccessKey" (string, optional): only for s3,
|
||||
"awsSecret" (string, optional): only for s3,
|
||||
"s3BucketName" (string, optional): only for s3
|
||||
},
|
||||
"swapJob" (object, optional): { configure the project
|
||||
swapping job.
|
||||
if null, defaults to no-op
|
||||
"minProjects" (int64): lower bound on number of projects
|
||||
present. The swap job will never go
|
||||
below this, regardless of what the
|
||||
watermark shows. Regardless, if
|
||||
minProjects prevents an eviction,
|
||||
the swap job will WARN,
|
||||
"lowGiB" (int32): the low watermark for swapping,
|
||||
i.e. swap until disk usage is below this,
|
||||
"highGiB" (int32): the high watermark for swapping,
|
||||
i.e. start swapping when
|
||||
disk usage becomes this,
|
||||
"intervalMillis" (int64): amount of time in between running
|
||||
swap job and checking watermarks.
|
||||
3600000 is 1 hour
|
||||
}
|
||||
}
|
||||
|
||||
You have to restart the server for configuration changes to take effect.
|
||||
|
||||
|
||||
## Creating OAuth app
|
||||
|
||||
In dev-env, run the following command in mongo to create the oauth application
|
||||
for git-bridge.
|
||||
|
||||
```
|
||||
db.oauthApplications.insert({
|
||||
"clientSecret" : "v1.G5HHTXfxsJMmfFhSar9QhJLg/u4KpGpYOdPGwoKdZXk=",
|
||||
"grants" : [
|
||||
"password"
|
||||
],
|
||||
"id" : "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
|
||||
"name" : "Overleaf Git Bridge",
|
||||
"redirectUris" : [],
|
||||
"scopes" : [
|
||||
"git_bridge"
|
||||
]
|
||||
})
|
||||
```
|
31
services/git-bridge/conf/envsubst_template.json
Normal file
31
services/git-bridge/conf/envsubst_template.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"port": ${GIT_BRIDGE_PORT:-8000},
|
||||
"bindIp": "${GIT_BRIDGE_BIND_IP:-0.0.0.0}",
|
||||
"idleTimeout": ${GIT_BRIDGE_IDLE_TIMEOUT:-30000},
|
||||
"rootGitDirectory": "${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}",
|
||||
"allowedCorsOrigins": "${GIT_BRIDGE_ALLOWED_CORS_ORIGINS:-https://localhost}",
|
||||
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
|
||||
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
|
||||
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
|
||||
"oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}",
|
||||
"userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
|
||||
"repoStore": {
|
||||
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
|
||||
"maxFileSize": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE:-52428800}
|
||||
},
|
||||
"swapStore": {
|
||||
"type": "${GIT_BRIDGE_SWAPSTORE_TYPE:-noop}",
|
||||
"awsAccessKey": "${GIT_BRIDGE_SWAPSTORE_AWS_ACCESS_KEY}",
|
||||
"awsSecret": "${GIT_BRIDGE_SWAPSTORE_AWS_SECRET}",
|
||||
"s3BucketName": "${GIT_BRIDGE_SWAPSTORE_S3_BUCKET_NAME}",
|
||||
"awsRegion": "${GIT_BRIDGE_SWAPSTORE_AWS_REGION:-us-east-1}"
|
||||
},
|
||||
"swapJob": {
|
||||
"minProjects": ${GIT_BRIDGE_SWAPJOB_MIN_PROJECTS:-50},
|
||||
"lowGiB": ${GIT_BRIDGE_SWAPJOB_LOW_GIB:-128},
|
||||
"highGiB": ${GIT_BRIDGE_SWAPJOB_HIGH_GIB:-256},
|
||||
"intervalMillis": ${GIT_BRIDGE_SWAPJOB_INTERVAL_MILLIS:-3600000},
|
||||
"compressionMethod": "${GIT_BRIDGE_SWAPJOB_COMPRESSION_METHOD:-gzip}"
|
||||
},
|
||||
"sqliteHeapLimitBytes": ${GIT_BRIDGE_SQLITE_HEAP_LIMIT_BYTES:-0}
|
||||
}
|
30
services/git-bridge/conf/example_config.json
Normal file
30
services/git-bridge/conf/example_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"port": 8080,
|
||||
"bindIp": "127.0.0.1",
|
||||
"idleTimeout": 30000,
|
||||
"rootGitDirectory": "/tmp/wlgb",
|
||||
"allowedCorsOrigins": "https://localhost",
|
||||
"apiBaseUrl": "https://localhost/api/v0",
|
||||
"postbackBaseUrl": "https://localhost",
|
||||
"serviceName": "Overleaf",
|
||||
"oauth2Server": "https://localhost",
|
||||
"repoStore": {
|
||||
"maxFileNum": 2000,
|
||||
"maxFileSize": 52428800
|
||||
},
|
||||
"swapStore": {
|
||||
"type": "s3",
|
||||
"awsAccessKey": "asdf",
|
||||
"awsSecret": "asdf",
|
||||
"s3BucketName": "com.overleaf.testbucket",
|
||||
"awsRegion": "us-east-1"
|
||||
},
|
||||
"swapJob": {
|
||||
"minProjects": 50,
|
||||
"lowGiB": 128,
|
||||
"highGiB": 256,
|
||||
"intervalMillis": 3600000,
|
||||
"compressionMethod": "gzip"
|
||||
},
|
||||
"sqliteHeapLimitBytes": 512000000
|
||||
}
|
25
services/git-bridge/conf/local.json
Normal file
25
services/git-bridge/conf/local.json
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"port": 8000,
|
||||
"bindIp": "0.0.0.0",
|
||||
"idleTimeout": 30000,
|
||||
"rootGitDirectory": "/tmp/wlgb",
|
||||
"allowedCorsOrigins": "http://v2.overleaf.test",
|
||||
"apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
|
||||
"postbackBaseUrl": "http://git-bridge:8000",
|
||||
"serviceName": "Overleaf",
|
||||
"oauth2Server": "http://v2.overleaf.test:3000",
|
||||
"repoStore": {
|
||||
"maxFileNum": 2000,
|
||||
"maxFileSize": 52428800
|
||||
},
|
||||
"swapStore": {
|
||||
"type": "noop"
|
||||
},
|
||||
"swapJob": {
|
||||
"minProjects": 50,
|
||||
"lowGiB": 128,
|
||||
"highGiB": 256,
|
||||
"intervalMillis": 3600000,
|
||||
"compressionMethod": "gzip"
|
||||
}
|
||||
}
|
275
services/git-bridge/pom.xml
Normal file
275
services/git-bridge/pom.xml
Normal file
@@ -0,0 +1,275 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>uk.ac.ic.wlgitbridge</groupId>
|
||||
<artifactId>writelatex-git-bridge</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<java.version>21</java.version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.plugin.version>3.13.0</maven.compiler.plugin.version>
|
||||
<maven.surefire.plugin.version>2.12.4</maven.surefire.plugin.version>
|
||||
<maven.assembly.plugin.version>3.1.0</maven.assembly.plugin.version>
|
||||
<fmt.plugin.version>2.23</fmt.plugin.version>
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<jmock.junit4.version>2.8.4</jmock.junit4.version>
|
||||
<jetty.servlet.version>9.4.57.v20241219</jetty.servlet.version>
|
||||
<gson.version>2.9.0</gson.version>
|
||||
<async.http.client.version>3.0.1</async.http.client.version>
|
||||
<jgit.version>6.6.1.202309021850-r</jgit.version>
|
||||
<sqlite.jdbc.version>3.41.2.2</sqlite.jdbc.version>
|
||||
<joda.time.version>2.9.9</joda.time.version>
|
||||
<google.oauth.client.version>1.37.0</google.oauth.client.version>
|
||||
<google.http.client.version>1.23.0</google.http.client.version>
|
||||
<commons.lang3.version>3.17.0</commons.lang3.version>
|
||||
<logback.classic.version>1.2.13</logback.classic.version>
|
||||
<mockserver.version>5.12.0</mockserver.version>
|
||||
<mockito.version>5.12.0</mockito.version>
|
||||
<aws.java.sdk.version>1.12.780</aws.java.sdk.version>
|
||||
<jakarta.xml.bind.api.version>${jaxb.runtime.version}</jakarta.xml.bind.api.version>
|
||||
<jaxb.runtime.version>2.3.2</jaxb.runtime.version>
|
||||
<httpclient.version>4.5.14</httpclient.version>
|
||||
<commons.io.version>2.18.0</commons.io.version>
|
||||
<commons.compress.version>1.27.1</commons.compress.version>
|
||||
<simpleclient.version>0.10.0</simpleclient.version>
|
||||
<bouncycastle.crypto.version>1.70</bouncycastle.crypto.version>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-compiler-plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>${maven.compiler.plugin.version}</version>
|
||||
<configuration>
|
||||
<source>${java.version}</source>
|
||||
<target>${java.version}</target>
|
||||
<release>${java.version}</release>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<!-- Workaround, test loader crashes without this configuration option -->
|
||||
<!-- See: https://stackoverflow.com/questions/53010200/maven-surefire-could-not-find-forkedbooter-class -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>${maven.surefire.plugin.version}</version>
|
||||
<configuration>
|
||||
<argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-assembly-plugin -->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>${maven.assembly.plugin.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifest>
|
||||
<mainClass>uk.ac.ic.wlgitbridge.Main</mainClass>
|
||||
</manifest>
|
||||
</archive>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>com.spotify.fmt</groupId>
|
||||
<artifactId>fmt-maven-plugin</artifactId>
|
||||
<version>${fmt.plugin.version}</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<dependencies>
|
||||
<!-- https://mvnrepository.com/artifact/junit/junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>${junit.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.jmock/jmock-junit4 -->
|
||||
<dependency>
|
||||
<groupId>org.jmock</groupId>
|
||||
<artifactId>jmock-junit4</artifactId>
|
||||
<version>${jmock.junit4.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-servlet -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<version>${jetty.servlet.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.eclipse.jetty/jetty-server -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<version>${jetty.servlet.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
<version>${gson.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.asynchttpclient/async-http-client -->
|
||||
<dependency>
|
||||
<groupId>org.asynchttpclient</groupId>
|
||||
<artifactId>async-http-client</artifactId>
|
||||
<version>${async.http.client.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jgit</groupId>
|
||||
<artifactId>org.eclipse.jgit</artifactId>
|
||||
<version>${jgit.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.eclipse.jgit/org.eclipse.jgit.http.server -->
|
||||
<dependency>
|
||||
<groupId>org.eclipse.jgit</groupId>
|
||||
<artifactId>org.eclipse.jgit.http.server</artifactId>
|
||||
<version>${jgit.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc -->
|
||||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
<version>${sqlite.jdbc.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
|
||||
<dependency>
|
||||
<groupId>joda-time</groupId>
|
||||
<artifactId>joda-time</artifactId>
|
||||
<version>${joda.time.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.google.oauth-client/google-oauth-client -->
|
||||
<dependency>
|
||||
<groupId>com.google.oauth-client</groupId>
|
||||
<artifactId>google-oauth-client</artifactId>
|
||||
<version>${google.oauth.client.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client -->
|
||||
<dependency>
|
||||
<groupId>com.google.http-client</groupId>
|
||||
<artifactId>google-http-client</artifactId>
|
||||
<version>${google.http.client.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.google.http-client/google-http-client-gson -->
|
||||
<dependency>
|
||||
<groupId>com.google.http-client</groupId>
|
||||
<artifactId>google-http-client-gson</artifactId>
|
||||
<version>${google.http.client.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<version>${commons.lang3.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/ch.qos.logback/logback-classic -->
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>${logback.classic.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-netty -->
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-netty</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.mock-server/mockserver-junit-rule -->
|
||||
<dependency>
|
||||
<groupId>org.mock-server</groupId>
|
||||
<artifactId>mockserver-junit-rule</artifactId>
|
||||
<version>${mockserver.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.mockito/mockito-core -->
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>${mockito.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
<version>${aws.java.sdk.version}</version>
|
||||
</dependency>
|
||||
<!-- API, java.xml.bind module -->
|
||||
<dependency>
|
||||
<groupId>jakarta.xml.bind</groupId>
|
||||
<artifactId>jakarta.xml.bind-api</artifactId>
|
||||
<version>${jakarta.xml.bind.api.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Runtime, com.sun.xml.bind module -->
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jaxb</groupId>
|
||||
<artifactId>jaxb-runtime</artifactId>
|
||||
<version>${jaxb.runtime.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>${httpclient.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>${commons.io.version}</version>
|
||||
</dependency>
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-compress -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>${commons.compress.version}</version>
|
||||
</dependency>
|
||||
<!-- prometheus metrics -->
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient</artifactId>
|
||||
<version>${simpleclient.version}</version>
|
||||
</dependency>
|
||||
<!-- Hotspot JVM metrics -->
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_hotspot</artifactId>
|
||||
<version>${simpleclient.version}</version>
|
||||
</dependency>
|
||||
<!-- Expose metrics via a servlet -->
|
||||
<dependency>
|
||||
<groupId>io.prometheus</groupId>
|
||||
<artifactId>simpleclient_servlet</artifactId>
|
||||
<version>${simpleclient.version}</version>
|
||||
</dependency>
|
||||
<!-- Require by MockServerClient to load 'sun.security.x509' / 'sun.security.util' -->
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcprov-jdk15on</artifactId>
|
||||
<version>${bouncycastle.crypto.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcpkix-jdk15on</artifactId>
|
||||
<version>${bouncycastle.crypto.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
12
services/git-bridge/server-pro-start.sh
Executable file
12
services/git-bridge/server-pro-start.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script is meant to be run as root when the git bridge starts up in
|
||||
# Server Pro. It ensures that the data directory is created and owned by the
|
||||
# "node" user, which is the regular user git bridge runs as.
|
||||
|
||||
ROOT_DIR="${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}"
|
||||
mkdir -p "$ROOT_DIR"
|
||||
chown node:node "$ROOT_DIR"
|
||||
|
||||
# Drop privileges using setpriv to avoid spawning a new process
|
||||
exec setpriv --reuid=node --regid=node --init-groups /start.sh
|
@@ -0,0 +1,39 @@
|
||||
package uk.ac.ic.wlgitbridge;
|
||||
|
||||
import java.util.Arrays;
|
||||
import uk.ac.ic.wlgitbridge.application.GitBridgeApp;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 01/11/14.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This is the entry point into the Git Bridge.
|
||||
*
|
||||
* It is responsible for creating the {@link GitBridgeApp} and then running it.
|
||||
*
|
||||
* The {@link GitBridgeApp} parses args and creates the {@link GitBridgeServer}.
|
||||
*
|
||||
* The {@link GitBridgeServer} creates the {@link Bridge}, among other things.
|
||||
*
|
||||
* The {@link Bridge} is the heart of the Git Bridge. Start there, and follow
|
||||
* the links outwards (which lead back to the Git users and the postback from
|
||||
* the snapshot API) and inwards (which lead into the components of the Git
|
||||
* Bridge: the configurable repo store, db store, and swap store, along with
|
||||
* the project lock, the swap job, the snapshot API, the resource cache
|
||||
* and the postback manager).
|
||||
*/
|
||||
public class Main {
|
||||
|
||||
public static void main(String[] args) {
|
||||
Log.info("Git Bridge started with args: " + Arrays.toString(args));
|
||||
try {
|
||||
new GitBridgeApp(args).run();
|
||||
} catch (Throwable t) {
|
||||
/* So that we get a timestamp */
|
||||
Log.error("Fatal exception thrown to top level, exiting: ", t);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,93 @@
|
||||
package uk.ac.ic.wlgitbridge.application;
|
||||
|
||||
import java.io.IOException;
|
||||
import javax.servlet.ServletException;
|
||||
import uk.ac.ic.wlgitbridge.application.config.Config;
|
||||
import uk.ac.ic.wlgitbridge.application.exception.ArgsException;
|
||||
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
|
||||
import uk.ac.ic.wlgitbridge.server.GitBridgeServer;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 02/11/14.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Class that represents the application. Parses arguments and gives them to the
|
||||
* server, or dies with a usage message.
|
||||
*/
|
||||
public class GitBridgeApp implements Runnable {
|
||||
|
||||
public static final int EXIT_CODE_FAILED = 1;
|
||||
private static final String USAGE_MESSAGE = "usage: writelatex-git-bridge [config_file]";
|
||||
|
||||
private String configFilePath;
|
||||
Config config;
|
||||
private GitBridgeServer server;
|
||||
|
||||
/*
|
||||
* Constructs an instance of the WriteLatex-Git Bridge application.
|
||||
* @param args args from main, which should be in the format [config_file]
|
||||
*/
|
||||
public GitBridgeApp(String[] args) {
|
||||
try {
|
||||
parseArguments(args);
|
||||
loadConfigFile();
|
||||
Log.info("Config loaded: {}", config.getSanitisedString());
|
||||
} catch (ArgsException e) {
|
||||
printUsage();
|
||||
System.exit(EXIT_CODE_FAILED);
|
||||
} catch (ConfigFileException e) {
|
||||
Log.error(
|
||||
"The property for " + e.getMissingMember() + " is invalid. Check your config file.");
|
||||
System.exit(EXIT_CODE_FAILED);
|
||||
} catch (IOException e) {
|
||||
Log.error("Invalid config file. Check the file path.");
|
||||
System.exit(EXIT_CODE_FAILED);
|
||||
}
|
||||
try {
|
||||
server = new GitBridgeServer(config);
|
||||
} catch (ServletException e) {
|
||||
Log.error("Servlet exception when instantiating GitBridgeServer", e);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Starts the server with the port number and root directory path given in
|
||||
* the command-line arguments.
|
||||
*/
|
||||
@Override
|
||||
public void run() {
|
||||
server.start();
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
server.stop();
|
||||
}
|
||||
|
||||
/* Helper methods */
|
||||
|
||||
private void parseArguments(String[] args) throws ArgsException {
|
||||
checkArgumentsLength(args);
|
||||
parseConfigFilePath(args);
|
||||
}
|
||||
|
||||
private void checkArgumentsLength(String[] args) throws ArgsException {
|
||||
if (args.length < 1) {
|
||||
throw new ArgsException();
|
||||
}
|
||||
}
|
||||
|
||||
private void parseConfigFilePath(String[] args) throws ArgsException {
|
||||
configFilePath = args[0];
|
||||
}
|
||||
|
||||
private void loadConfigFile() throws ConfigFileException, IOException {
|
||||
Log.info("Loading config file at path: " + configFilePath);
|
||||
config = new Config(configFilePath);
|
||||
}
|
||||
|
||||
private void printUsage() {
|
||||
System.err.println(USAGE_MESSAGE);
|
||||
}
|
||||
}
|
@@ -0,0 +1,204 @@
|
||||
package uk.ac.ic.wlgitbridge.application.config;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Nullable;
|
||||
import uk.ac.ic.wlgitbridge.application.exception.ConfigFileException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStoreConfig;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJobConfig;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStoreConfig;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
|
||||
import uk.ac.ic.wlgitbridge.util.Instance;
|
||||
|
||||
/*
|
||||
* Created by Winston on 05/12/14.
|
||||
*/
|
||||
public class Config implements JSONSource {
|
||||
|
||||
static Config asSanitised(Config config) {
|
||||
return new Config(
|
||||
config.port,
|
||||
config.bindIp,
|
||||
config.idleTimeout,
|
||||
config.rootGitDirectory,
|
||||
config.allowedCorsOrigins,
|
||||
config.apiBaseURL,
|
||||
config.postbackURL,
|
||||
config.serviceName,
|
||||
config.oauth2Server,
|
||||
config.userPasswordEnabled,
|
||||
config.repoStore,
|
||||
SwapStoreConfig.sanitisedCopy(config.swapStore),
|
||||
config.swapJob,
|
||||
config.sqliteHeapLimitBytes);
|
||||
}
|
||||
|
||||
private int port;
|
||||
private String bindIp;
|
||||
private int idleTimeout;
|
||||
private String rootGitDirectory;
|
||||
private String[] allowedCorsOrigins;
|
||||
private String apiBaseURL;
|
||||
private String postbackURL;
|
||||
private String serviceName;
|
||||
@Nullable private String oauth2Server;
|
||||
private boolean userPasswordEnabled;
|
||||
@Nullable private RepoStoreConfig repoStore;
|
||||
@Nullable private SwapStoreConfig swapStore;
|
||||
@Nullable private SwapJobConfig swapJob;
|
||||
private int sqliteHeapLimitBytes = 0;
|
||||
|
||||
public Config(String configFilePath) throws ConfigFileException, IOException {
|
||||
this(new FileReader(configFilePath));
|
||||
}
|
||||
|
||||
Config(Reader reader) {
|
||||
fromJSON(new Gson().fromJson(reader, JsonElement.class));
|
||||
}
|
||||
|
||||
public Config(
|
||||
int port,
|
||||
String bindIp,
|
||||
int idleTimeout,
|
||||
String rootGitDirectory,
|
||||
String[] allowedCorsOrigins,
|
||||
String apiBaseURL,
|
||||
String postbackURL,
|
||||
String serviceName,
|
||||
String oauth2Server,
|
||||
boolean userPasswordEnabled,
|
||||
RepoStoreConfig repoStore,
|
||||
SwapStoreConfig swapStore,
|
||||
SwapJobConfig swapJob,
|
||||
int sqliteHeapLimitBytes) {
|
||||
this.port = port;
|
||||
this.bindIp = bindIp;
|
||||
this.idleTimeout = idleTimeout;
|
||||
this.rootGitDirectory = rootGitDirectory;
|
||||
this.allowedCorsOrigins = allowedCorsOrigins;
|
||||
this.apiBaseURL = apiBaseURL;
|
||||
this.postbackURL = postbackURL;
|
||||
this.serviceName = serviceName;
|
||||
this.oauth2Server = oauth2Server;
|
||||
this.userPasswordEnabled = userPasswordEnabled;
|
||||
this.repoStore = repoStore;
|
||||
this.swapStore = swapStore;
|
||||
this.swapJob = swapJob;
|
||||
this.sqliteHeapLimitBytes = sqliteHeapLimitBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fromJSON(JsonElement json) {
|
||||
JsonObject configObject = json.getAsJsonObject();
|
||||
port = getElement(configObject, "port").getAsInt();
|
||||
bindIp = getElement(configObject, "bindIp").getAsString();
|
||||
idleTimeout = getElement(configObject, "idleTimeout").getAsInt();
|
||||
rootGitDirectory = getElement(configObject, "rootGitDirectory").getAsString();
|
||||
String apiBaseURL = getElement(configObject, "apiBaseUrl").getAsString();
|
||||
if (!apiBaseURL.endsWith("/")) {
|
||||
apiBaseURL += "/";
|
||||
}
|
||||
this.apiBaseURL = apiBaseURL;
|
||||
serviceName = getElement(configObject, "serviceName").getAsString();
|
||||
final String rawAllowedCorsOrigins =
|
||||
getOptionalString(configObject, "allowedCorsOrigins").trim();
|
||||
if (rawAllowedCorsOrigins.isEmpty()) {
|
||||
allowedCorsOrigins = new String[] {};
|
||||
} else {
|
||||
allowedCorsOrigins = rawAllowedCorsOrigins.split(",");
|
||||
}
|
||||
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
|
||||
if (!postbackURL.endsWith("/")) {
|
||||
postbackURL += "/";
|
||||
}
|
||||
oauth2Server = getOptionalString(configObject, "oauth2Server");
|
||||
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
|
||||
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
|
||||
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
|
||||
swapJob = new Gson().fromJson(configObject.get("swapJob"), SwapJobConfig.class);
|
||||
if (configObject.has("sqliteHeapLimitBytes")) {
|
||||
sqliteHeapLimitBytes = getElement(configObject, "sqliteHeapLimitBytes").getAsInt();
|
||||
}
|
||||
}
|
||||
|
||||
public String getSanitisedString() {
|
||||
return Instance.prettyGson.toJson(Config.asSanitised(this));
|
||||
}
|
||||
|
||||
public int getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public String getBindIp() {
|
||||
return bindIp;
|
||||
}
|
||||
|
||||
public int getIdleTimeout() {
|
||||
return idleTimeout;
|
||||
}
|
||||
|
||||
public String getRootGitDirectory() {
|
||||
return rootGitDirectory;
|
||||
}
|
||||
|
||||
public int getSqliteHeapLimitBytes() {
|
||||
return this.sqliteHeapLimitBytes;
|
||||
}
|
||||
|
||||
public String[] getAllowedCorsOrigins() {
|
||||
return allowedCorsOrigins;
|
||||
}
|
||||
|
||||
public String getAPIBaseURL() {
|
||||
return apiBaseURL;
|
||||
}
|
||||
|
||||
public String getServiceName() {
|
||||
return serviceName;
|
||||
}
|
||||
|
||||
public String getPostbackURL() {
|
||||
return postbackURL;
|
||||
}
|
||||
|
||||
public boolean isUserPasswordEnabled() {
|
||||
return userPasswordEnabled;
|
||||
}
|
||||
|
||||
public String getOauth2Server() {
|
||||
return oauth2Server;
|
||||
}
|
||||
|
||||
public Optional<RepoStoreConfig> getRepoStore() {
|
||||
return Optional.ofNullable(repoStore);
|
||||
}
|
||||
|
||||
public Optional<SwapStoreConfig> getSwapStore() {
|
||||
return Optional.ofNullable(swapStore);
|
||||
}
|
||||
|
||||
public Optional<SwapJobConfig> getSwapJob() {
|
||||
return Optional.ofNullable(swapJob);
|
||||
}
|
||||
|
||||
private JsonElement getElement(JsonObject configObject, String name) {
|
||||
JsonElement element = configObject.get(name);
|
||||
if (element == null) {
|
||||
throw new RuntimeException(new ConfigFileException(name));
|
||||
}
|
||||
return element;
|
||||
}
|
||||
|
||||
private String getOptionalString(JsonObject configObject, String name) {
|
||||
JsonElement element = configObject.get(name);
|
||||
if (element == null || !element.isJsonPrimitive()) {
|
||||
return "";
|
||||
}
|
||||
return element.getAsString();
|
||||
}
|
||||
}
|
@@ -0,0 +1,6 @@
|
||||
package uk.ac.ic.wlgitbridge.application.exception;
|
||||
|
||||
/*
|
||||
* Created by Winston on 03/11/14.
|
||||
*/
|
||||
public class ArgsException extends Exception {}
|
@@ -0,0 +1,17 @@
|
||||
package uk.ac.ic.wlgitbridge.application.exception;
|
||||
|
||||
/*
|
||||
* Created by Winston on 05/12/14.
|
||||
*/
|
||||
public class ConfigFileException extends Exception {
|
||||
|
||||
private final String missingMember;
|
||||
|
||||
public ConfigFileException(String missingMember) {
|
||||
this.missingMember = missingMember;
|
||||
}
|
||||
|
||||
public String getMissingMember() {
|
||||
return missingMember;
|
||||
}
|
||||
}
|
@@ -0,0 +1,60 @@
|
||||
package uk.ac.ic.wlgitbridge.application.jetty;
|
||||
|
||||
import org.eclipse.jetty.util.log.Logger;
|
||||
|
||||
/*
|
||||
* Created by Winston on 03/11/14.
|
||||
*/
|
||||
public class NullLogger implements Logger {
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "null_logger";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void warn(String s, Object... objects) {}
|
||||
|
||||
@Override
|
||||
public void warn(Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public void warn(String s, Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public void info(String s, Object... objects) {}
|
||||
|
||||
@Override
|
||||
public void info(Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public void info(String s, Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public boolean isDebugEnabled() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setDebugEnabled(boolean b) {}
|
||||
|
||||
@Override
|
||||
public void debug(String s, Object... objects) {}
|
||||
|
||||
@Override
|
||||
public void debug(String s, long l) {}
|
||||
|
||||
@Override
|
||||
public void debug(Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public void debug(String s, Throwable throwable) {}
|
||||
|
||||
@Override
|
||||
public Logger getLogger(String s) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ignore(Throwable throwable) {}
|
||||
}
|
@@ -0,0 +1,709 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import org.eclipse.jgit.errors.RepositoryNotFoundException;
|
||||
import uk.ac.ic.wlgitbridge.application.config.Config;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
|
||||
import uk.ac.ic.wlgitbridge.bridge.gc.GcJob;
|
||||
import uk.ac.ic.wlgitbridge.bridge.gc.GcJobImpl;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.*;
|
||||
import uk.ac.ic.wlgitbridge.bridge.resource.ResourceCache;
|
||||
import uk.ac.ic.wlgitbridge.bridge.resource.UrlResourceCache;
|
||||
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
|
||||
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApiFacade;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
|
||||
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
import uk.ac.ic.wlgitbridge.data.ProjectLockImpl;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
|
||||
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.FileLimitExceededException;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PostbackManager;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.*;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* This is the heart of the Git Bridge. You plug in all the parts (project
|
||||
* lock, repo store, db store, swap store, snapshot api, resource cache and
|
||||
* postback manager) is called by Git user requests and Overleaf postback
|
||||
* requests.
|
||||
*
|
||||
* Follow these links to go "outward" (to input from Git users and Overleaf):
|
||||
*
|
||||
* 1. JGit hooks, which handle user Git requests:
|
||||
*
|
||||
* @see WLRepositoryResolver - used on all requests associate a repo with a
|
||||
* project name, or fail
|
||||
*
|
||||
* @see WLUploadPackFactory - used to handle clones and fetches
|
||||
*
|
||||
* @see WLReceivePackFactory - used to handle pushes by setting a hook
|
||||
* @see WriteLatexPutHook - the hook used to handle pushes
|
||||
*
|
||||
* 2. The Postback Servlet, which handles postbacks from the Overleaf app
|
||||
* to confirm that a project is pushed. If a postback is lost, it's fine, we
|
||||
* just update ourselves on the next access.
|
||||
*
|
||||
* @see PostbackHandler - the entry point for postbacks
|
||||
*
|
||||
* Follow these links to go "inward" (to the Git Bridge components):
|
||||
*
|
||||
* 1. The Project Lock, used to synchronise accesses to projects and shutdown
|
||||
* the Git Bridge gracefully by preventing further lock acquiring.
|
||||
*
|
||||
* @see ProjectLock - the interface used for the Project Lock
|
||||
* @see ProjectLockImpl - the default concrete implementation
|
||||
*
|
||||
* 2. The Repo Store, used to provide repository objects.
|
||||
*
|
||||
* The default implementation uses Git on the file system.
|
||||
*
|
||||
* @see RepoStore - the interface for the Repo Store
|
||||
* @see FSGitRepoStore - the default concrete implementation
|
||||
* @see ProjectRepo - an interface for an actual repo instance
|
||||
* @see GitProjectRepo - the default concrete implementation
|
||||
*
|
||||
* 3. The DB Store, used to store persistent data such as the latest version
|
||||
* of each project that we have (used for querying the Snapshot API), along
|
||||
* with caching remote blobs.
|
||||
*
|
||||
* The default implementation is SQLite based.
|
||||
*
|
||||
* @see DBStore - the interface for the DB store
|
||||
* @see SqliteDBStore - the default concrete implementation
|
||||
*
|
||||
* 4. The Swap Store, used to swap projects to when the disk goes over a
|
||||
* certain data usage.
|
||||
*
|
||||
* The default implementation tarbzips projects to/from Amazon S3.
|
||||
*
|
||||
* @see SwapStore - the interface for the Swap Store
|
||||
* @see S3SwapStore - the default concrete implementation
|
||||
*
|
||||
* 5. The Swap Job, which performs the actual swapping on the swap store based
|
||||
* on various configuration options.
|
||||
*
|
||||
* @see SwapJob - the interface for the Swap Job
|
||||
* @see SwapJobImpl - the default concrete implementation
|
||||
*
|
||||
* 6. The Snapshot API, which provides data from the Overleaf app.
|
||||
*
|
||||
* @see SnapshotApiFacade - wraps a concrete instance of the Snapshot API.
|
||||
* @see SnapshotApi - the interface for the Snapshot API.
|
||||
* @see NetSnapshotApi - the default concrete implementation
|
||||
*
|
||||
* 7. The Resource Cache, which provides the data for attachment resources from
|
||||
* URLs. It will generally fetch from the source on a cache miss.
|
||||
*
|
||||
* The default implementation uses the DB Store to maintain a mapping from
|
||||
* URLs to files in an actual repo.
|
||||
*
|
||||
* @see ResourceCache - the interface for the Resource Cache
|
||||
* @see UrlResourceCache - the default concrete implementation
|
||||
*
|
||||
* 8. The Postback Manager, which keeps track of pending postbacks. It stores a
|
||||
* mapping from project names to postback promises.
|
||||
*
|
||||
* @see PostbackManager - the class
|
||||
* @see PostbackPromise - the object waited on for a postback.
|
||||
*
|
||||
*/
|
||||
public class Bridge {
|
||||
|
||||
private final Config config;
|
||||
|
||||
private final ProjectLock lock;
|
||||
|
||||
private final RepoStore repoStore;
|
||||
private final DBStore dbStore;
|
||||
private final SwapStore swapStore;
|
||||
private final SwapJob swapJob;
|
||||
private final GcJob gcJob;
|
||||
|
||||
private final SnapshotApiFacade snapshotAPI;
|
||||
private final ResourceCache resourceCache;
|
||||
|
||||
private final PostbackManager postbackManager;
|
||||
|
||||
/*
|
||||
* Creates a Bridge from its configurable parts, which are the repo, db and
|
||||
* swap store, and the swap job config.
|
||||
*
|
||||
* This should be the method used to create a Bridge.
|
||||
* @param config The config to use
|
||||
* @param repoStore The repo store to use
|
||||
* @param dbStore The db store to use
|
||||
* @param swapStore The swap store to use
|
||||
* @param snapshotApi The snapshot api to use
|
||||
* @return The constructed Bridge.
|
||||
*/
|
||||
public static Bridge make(
|
||||
Config config,
|
||||
RepoStore repoStore,
|
||||
DBStore dbStore,
|
||||
SwapStore swapStore,
|
||||
SnapshotApi snapshotApi) {
|
||||
ProjectLock lock =
|
||||
new ProjectLockImpl((int threads) -> Log.info("Waiting for " + threads + " projects..."));
|
||||
return new Bridge(
|
||||
config,
|
||||
lock,
|
||||
repoStore,
|
||||
dbStore,
|
||||
swapStore,
|
||||
SwapJob.fromConfig(config.getSwapJob(), lock, repoStore, dbStore, swapStore),
|
||||
new GcJobImpl(repoStore, lock),
|
||||
new SnapshotApiFacade(snapshotApi),
|
||||
new UrlResourceCache(dbStore));
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates a bridge from all of its components, not just its configurable
|
||||
* parts. This is for substituting mock/stub components for testing.
|
||||
* It's also used by Bridge.make to actually construct the bridge.
|
||||
* @param lock the {@link ProjectLock} to use
|
||||
* @param repoStore the {@link RepoStore} to use
|
||||
* @param dbStore the {@link DBStore} to use
|
||||
* @param swapStore the {@link SwapStore} to use
|
||||
* @param swapJob the {@link SwapJob} to use
|
||||
* @param gcJob
|
||||
* @param snapshotAPI the {@link SnapshotApi} to use
|
||||
* @param resourceCache the {@link ResourceCache} to use
|
||||
*/
|
||||
Bridge(
|
||||
Config config,
|
||||
ProjectLock lock,
|
||||
RepoStore repoStore,
|
||||
DBStore dbStore,
|
||||
SwapStore swapStore,
|
||||
SwapJob swapJob,
|
||||
GcJob gcJob,
|
||||
SnapshotApiFacade snapshotAPI,
|
||||
ResourceCache resourceCache) {
|
||||
this.config = config;
|
||||
this.lock = lock;
|
||||
this.repoStore = repoStore;
|
||||
this.dbStore = dbStore;
|
||||
this.swapStore = swapStore;
|
||||
this.snapshotAPI = snapshotAPI;
|
||||
this.resourceCache = resourceCache;
|
||||
this.swapJob = swapJob;
|
||||
this.gcJob = gcJob;
|
||||
postbackManager = new PostbackManager();
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(this::doShutdown));
|
||||
repoStore.purgeNonexistentProjects(dbStore.getProjectNames());
|
||||
}
|
||||
|
||||
/*
|
||||
* This performs the graceful shutdown of the Bridge, which is called by the
|
||||
* shutdown hook. It acquires the project write lock, which prevents
|
||||
* work being done for new projects (which acquire the read lock).
|
||||
* Once it has the write lock, there are no readers left, so the git bridge
|
||||
* can shut down gracefully.
|
||||
*
|
||||
* It is also used by the tests.
|
||||
*/
|
||||
void doShutdown() {
|
||||
Log.info("Shutdown received.");
|
||||
Log.info("Stopping SwapJob");
|
||||
swapJob.stop();
|
||||
Log.info("Stopping GcJob");
|
||||
gcJob.stop();
|
||||
Log.info("Waiting for projects");
|
||||
lock.lockAll();
|
||||
Log.info("Bye");
|
||||
}
|
||||
|
||||
/*
|
||||
* Starts the swap job, which will begin checking whether projects should be
|
||||
* swapped with a configurable frequency.
|
||||
*/
|
||||
public void startBackgroundJobs() {
|
||||
swapJob.start();
|
||||
gcJob.start();
|
||||
}
|
||||
|
||||
public boolean healthCheck() {
|
||||
try {
|
||||
dbStore.getNumProjects();
|
||||
File rootDirectory = new File("/");
|
||||
if (!rootDirectory.exists()) {
|
||||
throw new Exception("bad filesystem state, root directory does not exist");
|
||||
}
|
||||
Log.debug("[HealthCheck] passed");
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
Log.error("[HealthCheck] FAILED!", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Performs a check of inconsistencies in the DB. This was used to upgrade
|
||||
* the schema.
|
||||
*/
|
||||
public void checkDB() {
|
||||
Log.info("Checking DB");
|
||||
File rootDir = repoStore.getRootDirectory();
|
||||
for (File f : rootDir.listFiles()) {
|
||||
if (f.getName().equals(".wlgb")) {
|
||||
continue;
|
||||
}
|
||||
String projName = f.getName();
|
||||
try (LockGuard __ = lock.lockGuard(projName)) {
|
||||
File dotGit = new File(f, ".git");
|
||||
if (!dotGit.exists()) {
|
||||
Log.warn("Project: {} has no .git", projName);
|
||||
continue;
|
||||
}
|
||||
ProjectState state = dbStore.getProjectState(projName);
|
||||
if (state != ProjectState.NOT_PRESENT) {
|
||||
continue;
|
||||
}
|
||||
Log.warn("Project: {} not in swap_store, adding", projName);
|
||||
dbStore.setLastAccessedTime(projName, new Timestamp(dotGit.lastModified()));
|
||||
} catch (CannotAcquireLockException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Synchronises the given repository with Overleaf.
|
||||
*
|
||||
* It acquires the project lock and calls
|
||||
* {@link #getUpdatedRepoCritical(Optional, String, GetDocResult)}.
|
||||
* @param oauth2 The oauth2 to use
|
||||
* @param projectName The name of the project
|
||||
* @throws IOException
|
||||
* @throws GitUserException
|
||||
*/
|
||||
public ProjectRepo getUpdatedRepo(Optional<Credential> oauth2, String projectName)
|
||||
throws IOException, GitUserException, CannotAcquireLockException {
|
||||
try (LockGuard __ = lock.lockGuard(projectName)) {
|
||||
Optional<GetDocResult> maybeDoc = snapshotAPI.getDoc(oauth2, projectName);
|
||||
if (!maybeDoc.isPresent()) {
|
||||
throw new RepositoryNotFoundException(projectName);
|
||||
}
|
||||
GetDocResult doc = maybeDoc.get();
|
||||
Log.debug("[{}] Updating repository", projectName);
|
||||
return getUpdatedRepoCritical(oauth2, projectName, doc);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Synchronises the given repository with Overleaf.
|
||||
*
|
||||
* Pre: the project lock must be acquired for the given repo.
|
||||
*
|
||||
* 1. Queries the project state for the given project name.
|
||||
* a. NOT_PRESENT = We've never seen it before, and the row for the
|
||||
* project doesn't even exist. The project definitely
|
||||
* exists because we would have aborted otherwise.
|
||||
* b. PRESENT = The project is on disk.
|
||||
* c. SWAPPED = The project is in the {@link SwapStore}
|
||||
*
|
||||
* If the project has never been cloned, it is git init'd. If the project
|
||||
* is in swap, it is restored to disk. Otherwise, the project was already
|
||||
* present.
|
||||
*
|
||||
* With the project present, snapshots are downloaded from the snapshot
|
||||
* API with {@link #updateProject(Optional, ProjectRepo)}.
|
||||
*
|
||||
* Then, the last accessed time of the project is set to the current time.
|
||||
* This is to support the LRU of the swap store.
|
||||
* @param oauth2
|
||||
* @param projectName The name of the project
|
||||
* @throws IOException
|
||||
* @throws GitUserException
|
||||
*/
|
||||
private ProjectRepo getUpdatedRepoCritical(
|
||||
Optional<Credential> oauth2, String projectName, GetDocResult doc)
|
||||
throws IOException, GitUserException {
|
||||
ProjectRepo repo;
|
||||
ProjectState state = dbStore.getProjectState(projectName);
|
||||
switch (state) {
|
||||
case NOT_PRESENT:
|
||||
Log.info("[{}] Repo not present", projectName);
|
||||
repo = repoStore.initRepo(projectName);
|
||||
break;
|
||||
case SWAPPED:
|
||||
swapJob.restore(projectName);
|
||||
repo = repoStore.getExistingRepo(projectName);
|
||||
break;
|
||||
default:
|
||||
repo = repoStore.getExistingRepo(projectName);
|
||||
}
|
||||
updateProject(oauth2, repo);
|
||||
dbStore.setLastAccessedTime(projectName, Timestamp.valueOf(LocalDateTime.now()));
|
||||
return repo;
|
||||
}
|
||||
|
||||
/*
|
||||
* The public call to push a project.
|
||||
*
|
||||
* It acquires the lock and calls {@link #pushCritical(
|
||||
* Optional,
|
||||
* String,
|
||||
* RawDirectory,
|
||||
* RawDirectory
|
||||
* )}, catching exceptions, logging, and rethrowing them.
|
||||
* @param oauth2 The oauth2 to use for the snapshot API
|
||||
* @param projectName The name of the project to push to
|
||||
* @param directoryContents The new contents of the project
|
||||
* @param oldDirectoryContents The old contents of the project
|
||||
* @param hostname
|
||||
* @throws SnapshotPostException
|
||||
* @throws IOException
|
||||
* @throws MissingRepositoryException
|
||||
* @throws ForbiddenException
|
||||
* @throws GitUserException
|
||||
*/
|
||||
public void push(
|
||||
Optional<Credential> oauth2,
|
||||
String projectName,
|
||||
RawDirectory directoryContents,
|
||||
RawDirectory oldDirectoryContents,
|
||||
String hostname)
|
||||
throws SnapshotPostException,
|
||||
IOException,
|
||||
MissingRepositoryException,
|
||||
ForbiddenException,
|
||||
GitUserException,
|
||||
CannotAcquireLockException {
|
||||
Log.debug("[{}] pushing to Overleaf", projectName);
|
||||
try (LockGuard __ = lock.lockGuard(projectName)) {
|
||||
Log.info("[{}] got project lock", projectName);
|
||||
pushCritical(oauth2, projectName, directoryContents, oldDirectoryContents);
|
||||
} catch (SevereSnapshotPostException e) {
|
||||
Log.warn("[" + projectName + "] Failed to put to Overleaf", e);
|
||||
throw e;
|
||||
} catch (SnapshotPostException e) {
|
||||
/* Stack trace should be printed further up */
|
||||
Log.warn(
|
||||
"[{}] Exception when waiting for postback: {}",
|
||||
projectName,
|
||||
e.getClass().getSimpleName());
|
||||
throw e;
|
||||
} catch (IOException e) {
|
||||
Log.warn("[{}] IOException on put: {}", projectName, e);
|
||||
throw e;
|
||||
}
|
||||
|
||||
gcJob.queueForGc(projectName);
|
||||
}
|
||||
|
||||
/*
|
||||
* Does the work of pushing to a project, assuming the project lock is held.
|
||||
* The {@link WriteLatexPutHook} is the original caller, and when we return
|
||||
* without throwing, the commit is committed.
|
||||
*
|
||||
* We start off by creating a postback key, which is given in the url when
|
||||
* the Overleaf app tries to access the atts.
|
||||
*
|
||||
* Then creates a {@link CandidateSnapshot} from the old and new project
|
||||
* contents. The
|
||||
* {@link CandidateSnapshot} is created using
|
||||
* {@link #createCandidateSnapshot(String, RawDirectory, RawDirectory)},
|
||||
* which creates the snapshot object and writes the push files to the
|
||||
* atts directory, which is served by the {@link PostbackHandler}.
|
||||
* The files are deleted at the end of a try-with-resources block.
|
||||
*
|
||||
* Then 3 things are used to make the push request to the snapshot API:
|
||||
* 1. The oauth2
|
||||
* 2. The candidate snapshot
|
||||
* 3. The postback key
|
||||
*
|
||||
* If the snapshot API reports this as not successful, we immediately throw
|
||||
* an {@link OutOfDateException}, which goes back to the user.
|
||||
*
|
||||
* Otherwise, we wait (with a timeout) on a promise from the postback
|
||||
* manager, which can throw back to the user.
|
||||
*
|
||||
* If this is successful, we approve the snapshot with
|
||||
* {@link #approveSnapshot(int, CandidateSnapshot)}, which updates our side
|
||||
* of the push: the latest version and the URL index store.
|
||||
*
|
||||
* Then, we set the last accessed time for the swap store.
|
||||
*
|
||||
* Finally, after we return, the push to the repo from the hook is
|
||||
* successful and the repo gets updated.
|
||||
*
|
||||
* @param oauth2
|
||||
* @param projectName
|
||||
* @param directoryContents
|
||||
* @param oldDirectoryContents
|
||||
* @throws IOException
|
||||
* @throws MissingRepositoryException
|
||||
* @throws ForbiddenException
|
||||
* @throws SnapshotPostException
|
||||
* @throws GitUserException
|
||||
*/
|
||||
private void pushCritical(
|
||||
Optional<Credential> oauth2,
|
||||
String projectName,
|
||||
RawDirectory directoryContents,
|
||||
RawDirectory oldDirectoryContents)
|
||||
throws IOException,
|
||||
MissingRepositoryException,
|
||||
ForbiddenException,
|
||||
SnapshotPostException,
|
||||
GitUserException {
|
||||
Optional<Long> maxFileNum = config.getRepoStore().flatMap(RepoStoreConfig::getMaxFileNum);
|
||||
if (maxFileNum.isPresent()) {
|
||||
long maxFileNum_ = maxFileNum.get();
|
||||
if (directoryContents.getFileTable().size() > maxFileNum_) {
|
||||
Log.warn(
|
||||
"[{}] Too many files: {}/{}",
|
||||
projectName,
|
||||
directoryContents.getFileTable().size(),
|
||||
maxFileNum_);
|
||||
throw new FileLimitExceededException(directoryContents.getFileTable().size(), maxFileNum_);
|
||||
}
|
||||
}
|
||||
Log.debug(
|
||||
"[{}] Pushing files ({} new, {} old)",
|
||||
projectName,
|
||||
directoryContents.getFileTable().size(),
|
||||
oldDirectoryContents.getFileTable().size());
|
||||
String postbackKey = postbackManager.makeKeyForProject(projectName);
|
||||
Log.debug("[{}] Created postback key: {}", projectName, postbackKey);
|
||||
try (CandidateSnapshot candidate =
|
||||
createCandidateSnapshot(projectName, directoryContents, oldDirectoryContents); ) {
|
||||
Log.debug("[{}] Candidate snapshot created: {}", projectName, candidate);
|
||||
PushResult result = snapshotAPI.push(oauth2, candidate, postbackKey);
|
||||
if (result.wasSuccessful()) {
|
||||
Log.debug("[{}] Push to Overleaf successful", projectName);
|
||||
Log.debug("[{}] Waiting for postback...", projectName);
|
||||
int versionID = postbackManager.waitForVersionIdOrThrow(projectName);
|
||||
Log.debug("[{}] Got version ID for push: {}", projectName, versionID);
|
||||
approveSnapshot(versionID, candidate);
|
||||
Log.debug("[{}] Approved version ID: {}", projectName, versionID);
|
||||
dbStore.setLastAccessedTime(projectName, Timestamp.valueOf(LocalDateTime.now()));
|
||||
} else {
|
||||
Log.warn("[{}] Went out of date while waiting for push", projectName);
|
||||
throw new OutOfDateException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* A public call that should originate from the {@link FileHandler}.
|
||||
*
|
||||
* The {@link FileHandler} serves atts to the Overleaf app during a push.
|
||||
* The Overleaf app includes the postback key in the request, which was
|
||||
* originally given on a push request.
|
||||
*
|
||||
* This method checks that the postback key matches, and throws if not.
|
||||
*
|
||||
* The FileHandler should not serve the file if this throws.
|
||||
* @param projectName The project name that this key belongs to
|
||||
* @param postbackKey The key
|
||||
* @throws InvalidPostbackKeyException If the key doesn't match
|
||||
*/
|
||||
public void checkPostbackKey(String projectName, String postbackKey)
|
||||
throws InvalidPostbackKeyException {
|
||||
postbackManager.checkPostbackKey(projectName, postbackKey);
|
||||
}
|
||||
|
||||
/*
|
||||
* A public call that originates from the postback thread
|
||||
* {@link PostbackContents#processPostback()}, i.e. once the Overleaf app
|
||||
* has fetched all the atts and has committed the push and is happy, it
|
||||
* calls back here, fulfilling the promise that the push
|
||||
* {@link #push(Optional, String, RawDirectory, RawDirectory, String)}
|
||||
* is waiting on.
|
||||
*
|
||||
* The Overleaf app will have invented a new version for the push, which is
|
||||
* passed to the promise for the original push request to update the app.
|
||||
* @param projectName The name of the project being pushed to
|
||||
* @param postbackKey The postback key being used
|
||||
* @param versionID the new version id to use
|
||||
* @throws UnexpectedPostbackException if the postback key is invalid
|
||||
*/
|
||||
public void postbackReceivedSuccessfully(String projectName, String postbackKey, int versionID)
|
||||
throws UnexpectedPostbackException {
|
||||
Log.debug(
|
||||
"[{}]" + " Postback received by postback thread, version: {}", projectName, versionID);
|
||||
postbackManager.postVersionIDForProject(projectName, versionID, postbackKey);
|
||||
}
|
||||
|
||||
/*
|
||||
* As with {@link #postbackReceivedSuccessfully(String, String, int)},
|
||||
* but with an exception instead.
|
||||
*
|
||||
* This is based on the JSON body of the postback from the Overleaf app.
|
||||
*
|
||||
* The most likely problem is an {@link OutOfDateException}.
|
||||
* @param projectName The name of the project
|
||||
* @param postbackKey The postback key being used
|
||||
* @param exception The exception encountered
|
||||
* @throws UnexpectedPostbackException If the postback key is invalid
|
||||
*/
|
||||
public void postbackReceivedWithException(
|
||||
String projectName, String postbackKey, SnapshotPostException exception)
|
||||
throws UnexpectedPostbackException {
|
||||
Log.warn("[{}] Postback received with exception", projectName);
|
||||
postbackManager.postExceptionForProject(projectName, exception, postbackKey);
|
||||
}
|
||||
|
||||
/*
|
||||
* Delete a project's data
|
||||
*/
|
||||
public void deleteProject(String projectName) {
|
||||
Log.info("[{}] deleting project", projectName);
|
||||
dbStore.deleteProject(projectName);
|
||||
try {
|
||||
repoStore.remove(projectName);
|
||||
} catch (IOException e) {
|
||||
Log.warn("Failed to delete repository for project {}: {}", projectName, e);
|
||||
}
|
||||
swapStore.remove(projectName);
|
||||
}
|
||||
|
||||
/* PRIVATE */
|
||||
|
||||
/*
|
||||
* Called by {@link #getUpdatedRepoCritical(Optional, String)}
|
||||
*
|
||||
* Does the actual work of getting the snapshots for a project from the
|
||||
* snapshot API and committing them to a repo.
|
||||
*
|
||||
* If any snapshots were found, sets the latest version for the project.
|
||||
*
|
||||
* @param oauth2
|
||||
* @param repo
|
||||
* @throws IOException
|
||||
* @throws GitUserException
|
||||
*/
|
||||
private void updateProject(Optional<Credential> oauth2, ProjectRepo repo)
|
||||
throws IOException, GitUserException {
|
||||
String projectName = repo.getProjectName();
|
||||
int latestVersionId = dbStore.getLatestVersionForProject(projectName);
|
||||
Deque<Snapshot> snapshots = snapshotAPI.getSnapshots(oauth2, projectName, latestVersionId);
|
||||
|
||||
makeCommitsFromSnapshots(repo, snapshots);
|
||||
|
||||
// TODO: in case crashes around here, add an
|
||||
// "updating_from_commit" column to the DB as a way to rollback the
|
||||
// any failed partial updates before re-trying
|
||||
// Also need to consider the empty state (a new git init'd repo being
|
||||
// the rollback target)
|
||||
if (!snapshots.isEmpty()) {
|
||||
dbStore.setLatestVersionForProject(projectName, snapshots.getLast().getVersionID());
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Called by {@link #updateProject(Optional, ProjectRepo)}.
|
||||
*
|
||||
* Performs the actual Git commits on the disk.
|
||||
*
|
||||
* Each commit adds files to the db store
|
||||
* ({@link ResourceCache#get(String, String, String, Map, Map, Optional)},
|
||||
* and then removes any files that were deleted.
|
||||
* @param repo The repository to commit to
|
||||
* @param snapshots The snapshots to commit
|
||||
* @throws IOException If an IOException occurred
|
||||
* @throws SizeLimitExceededException If one of the files was too big.
|
||||
*/
|
||||
private void makeCommitsFromSnapshots(ProjectRepo repo, Collection<Snapshot> snapshots)
|
||||
throws IOException, GitUserException {
|
||||
String name = repo.getProjectName();
|
||||
Optional<Long> maxSize = config.getRepoStore().flatMap(RepoStoreConfig::getMaxFileSize);
|
||||
for (Snapshot snapshot : snapshots) {
|
||||
RawDirectory directory = repo.getDirectory();
|
||||
Map<String, RawFile> fileTable = directory.getFileTable();
|
||||
List<RawFile> files = new ArrayList<>();
|
||||
files.addAll(snapshot.getSrcs());
|
||||
for (RawFile file : files) {
|
||||
long size = file.size();
|
||||
/* Can't throw in ifPresent... */
|
||||
if (maxSize.isPresent()) {
|
||||
long maxSize_ = maxSize.get();
|
||||
if (size >= maxSize_) {
|
||||
throw new SizeLimitExceededException(Optional.of(file.getPath()), size, maxSize_);
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<String, byte[]> fetchedUrls = new HashMap<>();
|
||||
for (SnapshotAttachment snapshotAttachment : snapshot.getAtts()) {
|
||||
files.add(
|
||||
resourceCache.get(
|
||||
name,
|
||||
snapshotAttachment.getUrl(),
|
||||
snapshotAttachment.getPath(),
|
||||
fileTable,
|
||||
fetchedUrls,
|
||||
maxSize));
|
||||
}
|
||||
Log.debug("[{}] Committing version ID: {}", name, snapshot.getVersionID());
|
||||
Collection<String> missingFiles =
|
||||
repo.commitAndGetMissing(
|
||||
new GitDirectoryContents(files, repoStore.getRootDirectory(), name, snapshot));
|
||||
dbStore.deleteFilesForProject(name, missingFiles.toArray(new String[missingFiles.size()]));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Called by
|
||||
* {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
|
||||
*
|
||||
* This call consists of 2 things: Creating the candidate snapshot,
|
||||
* and writing the atts to the atts directory.
|
||||
*
|
||||
* The candidate snapshot RAIIs away those atts (use try-with-resources).
|
||||
* @param projectName The name of the project
|
||||
* @param directoryContents The new directory contents
|
||||
* @param oldDirectoryContents The old directory contents
|
||||
* @return The {@link CandidateSnapshot} created
|
||||
* @throws IOException If an I/O exception occurred on writing
|
||||
*/
|
||||
private CandidateSnapshot createCandidateSnapshot(
|
||||
String projectName, RawDirectory directoryContents, RawDirectory oldDirectoryContents)
|
||||
throws IOException {
|
||||
CandidateSnapshot candidateSnapshot =
|
||||
new CandidateSnapshot(
|
||||
projectName,
|
||||
dbStore.getLatestVersionForProject(projectName),
|
||||
directoryContents,
|
||||
oldDirectoryContents);
|
||||
candidateSnapshot.writeServletFiles(repoStore.getRootDirectory());
|
||||
return candidateSnapshot;
|
||||
}
|
||||
|
||||
/*
|
||||
* Called by
|
||||
* {@link #pushCritical(Optional, String, RawDirectory, RawDirectory)}.
|
||||
*
|
||||
* This method approves a push by setting the latest version and removing
|
||||
* any deleted files from the db store (files were already added by the
|
||||
* resources cache).
|
||||
* @param versionID
|
||||
* @param candidateSnapshot
|
||||
*/
|
||||
private void approveSnapshot(int versionID, CandidateSnapshot candidateSnapshot) {
|
||||
List<String> deleted = candidateSnapshot.getDeleted();
|
||||
dbStore.setLatestVersionForProject(candidateSnapshot.getProjectName(), versionID);
|
||||
dbStore.deleteFilesForProject(
|
||||
candidateSnapshot.getProjectName(), deleted.toArray(new String[deleted.size()]));
|
||||
}
|
||||
}
|
@@ -0,0 +1,19 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class DBInitException extends RuntimeException {
|
||||
|
||||
public DBInitException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DBInitException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
|
||||
public DBInitException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
@@ -0,0 +1,48 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface DBStore {
|
||||
|
||||
int getNumProjects();
|
||||
|
||||
List<String> getProjectNames();
|
||||
|
||||
void setLatestVersionForProject(String project, int versionID);
|
||||
|
||||
int getLatestVersionForProject(String project);
|
||||
|
||||
void addURLIndexForProject(String projectName, String url, String path);
|
||||
|
||||
void deleteFilesForProject(String project, String... files);
|
||||
|
||||
String getPathForURLInProject(String projectName, String url);
|
||||
|
||||
String getOldestUnswappedProject();
|
||||
|
||||
void swap(String projectName, String compressionMethod);
|
||||
|
||||
void restore(String projectName);
|
||||
|
||||
String getSwapCompression(String projectName);
|
||||
|
||||
int getNumUnswappedProjects();
|
||||
|
||||
ProjectState getProjectState(String projectName);
|
||||
|
||||
/*
|
||||
* Sets the last accessed time for the given project name.
|
||||
* @param projectName the project's name
|
||||
* @param time the time, or null if the project is to be swapped
|
||||
*/
|
||||
void setLastAccessedTime(String projectName, Timestamp time);
|
||||
|
||||
/*
|
||||
* Delete the metadata associated with the given project.
|
||||
*/
|
||||
void deleteProject(String projectName);
|
||||
}
|
@@ -0,0 +1,10 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public enum ProjectState {
|
||||
NOT_PRESENT,
|
||||
PRESENT,
|
||||
SWAPPED
|
||||
}
|
@@ -0,0 +1,70 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.noop;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.util.List;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
|
||||
|
||||
public class NoopDbStore implements DBStore {
|
||||
|
||||
@Override
|
||||
public int getNumProjects() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getProjectNames() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLatestVersionForProject(String project, int versionID) {}
|
||||
|
||||
@Override
|
||||
public int getLatestVersionForProject(String project) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addURLIndexForProject(String projectName, String url, String path) {}
|
||||
|
||||
@Override
|
||||
public void deleteFilesForProject(String project, String... files) {}
|
||||
|
||||
@Override
|
||||
public String getPathForURLInProject(String projectName, String url) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOldestUnswappedProject() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumUnswappedProjects() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectState getProjectState(String projectName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLastAccessedTime(String projectName, Timestamp time) {}
|
||||
|
||||
@Override
|
||||
public void swap(String projectName, String compressionMethod) {}
|
||||
|
||||
@Override
|
||||
public void restore(String projectName) {}
|
||||
|
||||
@Override
|
||||
public String getSwapCompression(String projectName) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteProject(String projectName) {}
|
||||
}
|
@@ -0,0 +1,12 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public interface SQLQuery<T> extends SQLUpdate {
|
||||
|
||||
public T processResultSet(ResultSet resultSet) throws SQLException;
|
||||
}
|
@@ -0,0 +1,14 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public interface SQLUpdate {
|
||||
|
||||
String getSQL();
|
||||
|
||||
default void addParametersToStatement(PreparedStatement statement) throws SQLException {}
|
||||
}
|
@@ -0,0 +1,227 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import java.io.File;
|
||||
import java.sql.*;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBInitException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.query.*;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter.*;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create.*;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete.*;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert.*;
|
||||
|
||||
/*
|
||||
* Created by Winston on 17/11/14.
|
||||
*/
|
||||
public class SqliteDBStore implements DBStore {
|
||||
|
||||
private final Connection connection;
|
||||
private int heapLimitBytes = 0;
|
||||
|
||||
public SqliteDBStore(File dbFile) {
|
||||
this(dbFile, 0);
|
||||
}
|
||||
|
||||
public SqliteDBStore(File dbFile, int heapLimitBytes) {
|
||||
this.heapLimitBytes = heapLimitBytes;
|
||||
try {
|
||||
connection = openConnectionTo(dbFile);
|
||||
createTables();
|
||||
} catch (Throwable t) {
|
||||
throw new DBInitException(t);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumProjects() {
|
||||
return query(new GetNumProjects());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getProjectNames() {
|
||||
return query(new GetProjectNamesSQLQuery());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLatestVersionForProject(String projectName, int versionID) {
|
||||
update(new SetProjectSQLUpdate(projectName, versionID));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getLatestVersionForProject(String projectName) {
|
||||
return query(new GetLatestVersionForProjectSQLQuery(projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addURLIndexForProject(String projectName, String url, String path) {
|
||||
update(new AddURLIndexSQLUpdate(projectName, url, path));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFilesForProject(String projectName, String... paths) {
|
||||
update(new DeleteFilesForProjectSQLUpdate(projectName, paths));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPathForURLInProject(String projectName, String url) {
|
||||
return query(new GetPathForURLInProjectSQLQuery(projectName, url));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOldestUnswappedProject() {
|
||||
return query(new GetOldestProjectName());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumUnswappedProjects() {
|
||||
return query(new GetNumUnswappedProjects());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectState getProjectState(String projectName) {
|
||||
return query(new GetProjectState(projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLastAccessedTime(String projectName, Timestamp lastAccessed) {
|
||||
update(new SetProjectLastAccessedTime(projectName, lastAccessed));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void swap(String projectName, String compressionMethod) {
|
||||
update(new UpdateSwap(projectName, compressionMethod));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void restore(String projectName) {
|
||||
update(new UpdateRestore(projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSwapCompression(String projectName) {
|
||||
return query(new GetSwapCompression(projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteProject(String projectName) {
|
||||
update(new DeleteAllFilesInProjectSQLUpdate(projectName));
|
||||
update(new DeleteProjectSQLUpdate(projectName));
|
||||
}
|
||||
|
||||
private Connection openConnectionTo(File dbFile) {
|
||||
File parentDir = dbFile.getParentFile();
|
||||
if (!parentDir.exists() && !parentDir.mkdirs()) {
|
||||
throw new DBInitException(
|
||||
parentDir.getAbsolutePath()
|
||||
+ " directory didn't exist, "
|
||||
+ "and unable to create. Check your permissions.");
|
||||
}
|
||||
try {
|
||||
Class.forName("org.sqlite.JDBC");
|
||||
} catch (ClassNotFoundException e) {
|
||||
throw new DBInitException(e);
|
||||
}
|
||||
try {
|
||||
return DriverManager.getConnection("jdbc:sqlite:" + dbFile.getAbsolutePath());
|
||||
} catch (SQLException e) {
|
||||
throw new DBInitException("Unable to connect to DB", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void createTables() {
|
||||
/* Migrations */
|
||||
/* We need to eat exceptions from here */
|
||||
try {
|
||||
doUpdate(new SetSoftHeapLimitPragma(this.heapLimitBytes));
|
||||
} catch (SQLException ignore) {
|
||||
}
|
||||
try {
|
||||
doUpdate(new ProjectsAddLastAccessed());
|
||||
} catch (SQLException ignore) {
|
||||
}
|
||||
try {
|
||||
doUpdate(new ProjectsAddSwapTime());
|
||||
} catch (SQLException ignore) {
|
||||
}
|
||||
try {
|
||||
doUpdate(new ProjectsAddRestoreTime());
|
||||
} catch (SQLException ignore) {
|
||||
}
|
||||
try {
|
||||
doUpdate(new ProjectsAddSwapCompression());
|
||||
} catch (SQLException ignore) {
|
||||
}
|
||||
|
||||
/* Create tables (if they don't exist) */
|
||||
Stream.of(
|
||||
new CreateProjectsTableSQLUpdate(),
|
||||
new CreateProjectsIndexLastAccessed(),
|
||||
new CreateURLIndexStoreSQLUpdate(),
|
||||
new CreateIndexURLIndexStore())
|
||||
.forEach(this::update);
|
||||
|
||||
/* In the case of needing to change the schema, we need to check that
|
||||
migrations didn't just fail */
|
||||
Preconditions.checkState(query(new LastAccessedColumnExists()));
|
||||
Preconditions.checkState(query(new SwapTimeColumnExists()));
|
||||
Preconditions.checkState(query(new RestoreTimeColumnExists()));
|
||||
Preconditions.checkState(query(new SwapCompressionColumnExists()));
|
||||
}
|
||||
|
||||
private void update(SQLUpdate update) {
|
||||
try {
|
||||
doUpdate(update);
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T query(SQLQuery<T> query) {
|
||||
try {
|
||||
return doQuery(query);
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void doUpdate(SQLUpdate update) throws SQLException {
|
||||
PreparedStatement statement = null;
|
||||
try {
|
||||
statement = connection.prepareStatement(update.getSQL());
|
||||
update.addParametersToStatement(statement);
|
||||
statement.executeUpdate();
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try {
|
||||
statement.close();
|
||||
} catch (Throwable t) {
|
||||
throw new SQLException(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private <T> T doQuery(SQLQuery<T> query) throws SQLException {
|
||||
PreparedStatement statement = null;
|
||||
ResultSet results = null;
|
||||
try {
|
||||
statement = connection.prepareStatement(query.getSQL());
|
||||
query.addParametersToStatement(statement);
|
||||
results = statement.executeQuery();
|
||||
return query.processResultSet(results);
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
if (statement != null) {
|
||||
statement.close();
|
||||
}
|
||||
if (results != null) {
|
||||
results.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,40 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class GetLatestVersionForProjectSQLQuery implements SQLQuery<Integer> {
|
||||
|
||||
private static final String GET_VERSION_IDS_FOR_PROJECT_NAME =
|
||||
"SELECT `version_id` FROM `projects` WHERE `name` = ?";
|
||||
|
||||
private final String projectName;
|
||||
|
||||
public GetLatestVersionForProjectSQLQuery(String projectName) {
|
||||
this.projectName = projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer processResultSet(ResultSet resultSet) throws SQLException {
|
||||
int versionID = 0;
|
||||
while (resultSet.next()) {
|
||||
versionID = resultSet.getInt("version_id");
|
||||
}
|
||||
return versionID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_VERSION_IDS_FOR_PROJECT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,26 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class GetNumProjects implements SQLQuery<Integer> {
|
||||
|
||||
private static final String GET_NUM_PROJECTS = "SELECT COUNT(*)\n" + " FROM `projects`";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_NUM_PROJECTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
return resultSet.getInt("COUNT(*)");
|
||||
}
|
||||
throw new IllegalStateException("Count always returns results");
|
||||
}
|
||||
}
|
@@ -0,0 +1,27 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class GetNumUnswappedProjects implements SQLQuery<Integer> {
|
||||
|
||||
private static final String GET_NUM_UNSWAPPED_PROJECTS =
|
||||
"SELECT COUNT(*)\n" + " FROM `projects`\n" + " WHERE `last_accessed` IS NOT NULL";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_NUM_UNSWAPPED_PROJECTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
return resultSet.getInt("COUNT(*)");
|
||||
}
|
||||
throw new IllegalStateException("Count always returns results");
|
||||
}
|
||||
}
|
@@ -0,0 +1,29 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class GetOldestProjectName implements SQLQuery<String> {
|
||||
|
||||
private static final String GET_OLDEST_PROJECT_NAME =
|
||||
"SELECT `name`, MIN(`last_accessed`)\n"
|
||||
+ " FROM `projects` \n"
|
||||
+ " WHERE `last_accessed` IS NOT NULL;";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_OLDEST_PROJECT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
return resultSet.getString("name");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
@@ -0,0 +1,43 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class GetPathForURLInProjectSQLQuery implements SQLQuery<String> {
|
||||
|
||||
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME =
|
||||
"SELECT `path` " + "FROM `url_index_store` " + "WHERE `project_name` = ? " + "AND `url` = ?";
|
||||
|
||||
private final String projectName;
|
||||
private final String url;
|
||||
|
||||
public GetPathForURLInProjectSQLQuery(String projectName, String url) {
|
||||
this.projectName = projectName;
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String processResultSet(ResultSet resultSet) throws SQLException {
|
||||
String path = null;
|
||||
while (resultSet.next()) {
|
||||
path = resultSet.getString("path");
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_URL_INDEXES_FOR_PROJECT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
statement.setString(2, url);
|
||||
}
|
||||
}
|
@@ -0,0 +1,29 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by Winston on 21/02/15.
|
||||
*/
|
||||
public class GetProjectNamesSQLQuery implements SQLQuery<List<String>> {
|
||||
|
||||
private static final String GET_URL_INDEXES_FOR_PROJECT_NAME = "SELECT `name` FROM `projects`";
|
||||
|
||||
@Override
|
||||
public List<String> processResultSet(ResultSet resultSet) throws SQLException {
|
||||
List<String> projectNames = new ArrayList<>();
|
||||
while (resultSet.next()) {
|
||||
projectNames.add(resultSet.getString("name"));
|
||||
}
|
||||
return projectNames;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_URL_INDEXES_FOR_PROJECT_NAME;
|
||||
}
|
||||
}
|
@@ -0,0 +1,43 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.ProjectState;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class GetProjectState implements SQLQuery<ProjectState> {
|
||||
|
||||
private static final String GET_PROJECT_STATE =
|
||||
"SELECT `last_accessed`\n" + " FROM `projects`\n" + " WHERE `name` = ?";
|
||||
|
||||
private final String projectName;
|
||||
|
||||
public GetProjectState(String projectName) {
|
||||
this.projectName = projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_PROJECT_STATE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectState processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
if (resultSet.getTimestamp("last_accessed") == null) {
|
||||
return ProjectState.SWAPPED;
|
||||
}
|
||||
return ProjectState.PRESENT;
|
||||
}
|
||||
return ProjectState.NOT_PRESENT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,36 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
public class GetSwapCompression implements SQLQuery<String> {
|
||||
private static final String GET_SWAP_COMPRESSION =
|
||||
"SELECT `swap_compression` FROM `projects` WHERE `name` = ?";
|
||||
|
||||
private final String projectName;
|
||||
|
||||
public GetSwapCompression(String projectName) {
|
||||
this.projectName = projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String processResultSet(ResultSet resultSet) throws SQLException {
|
||||
String compression = null;
|
||||
while (resultSet.next()) {
|
||||
compression = resultSet.getString("swap_compression");
|
||||
}
|
||||
return compression;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return GET_SWAP_COMPRESSION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,28 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
/*
|
||||
* Created by winston on 04/09/2016.
|
||||
*/
|
||||
public class LastAccessedColumnExists implements SQLQuery<Boolean> {
|
||||
|
||||
private static final String LAST_ACCESSED_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return LAST_ACCESSED_COLUMN_EXISTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
if (resultSet.getString(2).equals("last_accessed")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,24 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
public class RestoreTimeColumnExists implements SQLQuery<Boolean> {
|
||||
private static final String RESTORE_TIME_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return RESTORE_TIME_COLUMN_EXISTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
if (resultSet.getString(2).equals("restore_time")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,24 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
public class SwapCompressionColumnExists implements SQLQuery<Boolean> {
|
||||
private static final String SWAP_COMPRESSION_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return SWAP_COMPRESSION_COLUMN_EXISTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
if (resultSet.getString(2).equals("swap_compression")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,24 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.query;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLQuery;
|
||||
|
||||
public class SwapTimeColumnExists implements SQLQuery<Boolean> {
|
||||
private static final String SWAP_TIME_COLUMN_EXISTS = "PRAGMA table_info(`projects`)";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return SWAP_TIME_COLUMN_EXISTS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean processResultSet(ResultSet resultSet) throws SQLException {
|
||||
while (resultSet.next()) {
|
||||
if (resultSet.getString(2).equals("swap_time")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,17 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by winston on 03/09/2016.
|
||||
*/
|
||||
public class ProjectsAddLastAccessed implements SQLUpdate {
|
||||
|
||||
private static final String PROJECTS_ADD_LAST_ACCESSED =
|
||||
"ALTER TABLE `projects`\n" + "ADD COLUMN `last_accessed` DATETIME NULL DEFAULT 0";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return PROJECTS_ADD_LAST_ACCESSED;
|
||||
}
|
||||
}
|
@@ -0,0 +1,13 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class ProjectsAddRestoreTime implements SQLUpdate {
|
||||
private static final String PROJECTS_ADD_RESTORE_TIME =
|
||||
"ALTER TABLE `projects`\n" + "ADD COLUMN `restore_time` DATETIME NULL;\n";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return PROJECTS_ADD_RESTORE_TIME;
|
||||
}
|
||||
}
|
@@ -0,0 +1,13 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class ProjectsAddSwapCompression implements SQLUpdate {
|
||||
private static final String PROJECTS_ADD_SWAP_COMPRESSION =
|
||||
"ALTER TABLE `projects`\n" + "ADD COLUMN `swap_compression` VARCHAR NULL;\n";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return PROJECTS_ADD_SWAP_COMPRESSION;
|
||||
}
|
||||
}
|
@@ -0,0 +1,13 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class ProjectsAddSwapTime implements SQLUpdate {
|
||||
private static final String PROJECTS_ADD_SWAP_TIME =
|
||||
"ALTER TABLE `projects`\n" + "ADD COLUMN `swap_time` DATETIME NULL;\n";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return PROJECTS_ADD_SWAP_TIME;
|
||||
}
|
||||
}
|
@@ -0,0 +1,16 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.alter;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class SetSoftHeapLimitPragma implements SQLUpdate {
|
||||
private int heapLimitBytes = 0;
|
||||
|
||||
public SetSoftHeapLimitPragma(int heapLimitBytes) {
|
||||
this.heapLimitBytes = heapLimitBytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return "PRAGMA soft_heap_limit=" + this.heapLimitBytes + ";";
|
||||
}
|
||||
}
|
@@ -0,0 +1,18 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 21/02/15.
|
||||
*/
|
||||
public class CreateIndexURLIndexStore implements SQLUpdate {
|
||||
|
||||
public static final String CREATE_INDEX_URL_INDEX_STORE =
|
||||
"CREATE UNIQUE INDEX IF NOT EXISTS `project_path_index` "
|
||||
+ "ON `url_index_store`(`project_name`, `path`);\n";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return CREATE_INDEX_URL_INDEX_STORE;
|
||||
}
|
||||
}
|
@@ -0,0 +1,18 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class CreateProjectsIndexLastAccessed implements SQLUpdate {
|
||||
|
||||
private static final String CREATE_PROJECTS_INDEX_LAST_ACCESSED =
|
||||
"CREATE INDEX IF NOT EXISTS `projects_index_last_accessed`\n"
|
||||
+ " ON `projects`(`last_accessed`)";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return CREATE_PROJECTS_INDEX_LAST_ACCESSED;
|
||||
}
|
||||
}
|
@@ -0,0 +1,25 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class CreateProjectsTableSQLUpdate implements SQLUpdate {
|
||||
|
||||
private static final String CREATE_PROJECTS_TABLE =
|
||||
"CREATE TABLE IF NOT EXISTS `projects` (\n"
|
||||
+ " `name` VARCHAR NOT NULL DEFAULT '',\n"
|
||||
+ " `version_id` INT NOT NULL DEFAULT 0,\n"
|
||||
+ " `last_accessed` DATETIME NULL DEFAULT 0,\n"
|
||||
+ " `swap_time` DATETIME NULL,\n"
|
||||
+ " `restore_time` DATETIME NULL,\n"
|
||||
+ " `swap_compression` VARCHAR NULL,\n"
|
||||
+ " PRIMARY KEY (`name`)\n"
|
||||
+ ")";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return CREATE_PROJECTS_TABLE;
|
||||
}
|
||||
}
|
@@ -0,0 +1,27 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.create;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class CreateURLIndexStoreSQLUpdate implements SQLUpdate {
|
||||
|
||||
private static final String CREATE_URL_INDEX_STORE =
|
||||
"CREATE TABLE IF NOT EXISTS `url_index_store` (\n"
|
||||
+ " `project_name` varchar(10) NOT NULL DEFAULT '',\n"
|
||||
+ " `url` text NOT NULL,\n"
|
||||
+ " `path` text NOT NULL,\n"
|
||||
+ " PRIMARY KEY (`project_name`,`url`),\n"
|
||||
+ " CONSTRAINT `url_index_store_ibfk_1` "
|
||||
+ "FOREIGN KEY (`project_name`) "
|
||||
+ "REFERENCES `projects` (`name`) "
|
||||
+ "ON DELETE CASCADE "
|
||||
+ "ON UPDATE CASCADE\n"
|
||||
+ ");\n";
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return CREATE_URL_INDEX_STORE;
|
||||
}
|
||||
}
|
@@ -0,0 +1,23 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class DeleteAllFilesInProjectSQLUpdate implements SQLUpdate {
|
||||
private final String projectName;
|
||||
|
||||
public DeleteAllFilesInProjectSQLUpdate(String projectName) {
|
||||
this.projectName = projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return "DELETE FROM `url_index_store` WHERE `project_name` = ?";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,43 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class DeleteFilesForProjectSQLUpdate implements SQLUpdate {
|
||||
|
||||
private static final String DELETE_URL_INDEXES_FOR_PROJECT_NAME =
|
||||
"DELETE FROM `url_index_store` " + "WHERE `project_name` = ? AND path IN (";
|
||||
|
||||
private final String projectName;
|
||||
private final String[] paths;
|
||||
|
||||
public DeleteFilesForProjectSQLUpdate(String projectName, String... paths) {
|
||||
this.projectName = projectName;
|
||||
this.paths = paths;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
StringBuilder sb = new StringBuilder(DELETE_URL_INDEXES_FOR_PROJECT_NAME);
|
||||
for (int i = 0; i < paths.length; i++) {
|
||||
sb.append("?");
|
||||
if (i < paths.length - 1) {
|
||||
sb.append(", ");
|
||||
}
|
||||
}
|
||||
sb.append(");\n");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
for (int i = 0; i < paths.length; i++) {
|
||||
statement.setString(i + 2, paths[i]);
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,23 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.delete;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class DeleteProjectSQLUpdate implements SQLUpdate {
|
||||
private final String projectName;
|
||||
|
||||
public DeleteProjectSQLUpdate(String projectName) {
|
||||
this.projectName = projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return "DELETE FROM `projects` WHERE `name` = ?";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,41 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class AddURLIndexSQLUpdate implements SQLUpdate {
|
||||
|
||||
private static final String ADD_URL_INDEX =
|
||||
"INSERT OR REPLACE INTO `url_index_store`("
|
||||
+ "`project_name`, "
|
||||
+ "`url`, "
|
||||
+ "`path`"
|
||||
+ ") VALUES "
|
||||
+ "(?, ?, ?)\n";
|
||||
|
||||
private final String projectName;
|
||||
private final String url;
|
||||
private final String path;
|
||||
|
||||
public AddURLIndexSQLUpdate(String projectName, String url, String path) {
|
||||
this.projectName = projectName;
|
||||
this.url = url;
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return ADD_URL_INDEX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
statement.setString(2, url);
|
||||
statement.setString(3, path);
|
||||
}
|
||||
}
|
@@ -0,0 +1,34 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class SetProjectLastAccessedTime implements SQLUpdate {
|
||||
|
||||
private static final String SET_PROJECT_LAST_ACCESSED_TIME =
|
||||
"UPDATE `projects`\n" + "SET `last_accessed` = ?\n" + "WHERE `name` = ?";
|
||||
|
||||
private final String projectName;
|
||||
private final Timestamp lastAccessed;
|
||||
|
||||
public SetProjectLastAccessedTime(String projectName, Timestamp lastAccessed) {
|
||||
this.projectName = projectName;
|
||||
this.lastAccessed = lastAccessed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return SET_PROJECT_LAST_ACCESSED_TIME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setTimestamp(1, lastAccessed);
|
||||
statement.setString(2, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,35 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class SetProjectSQLUpdate implements SQLUpdate {
|
||||
|
||||
private static final String SET_PROJECT =
|
||||
"INSERT OR REPLACE "
|
||||
+ "INTO `projects`(`name`, `version_id`, `last_accessed`) "
|
||||
+ "VALUES (?, ?, DATETIME('now'));\n";
|
||||
|
||||
private final String projectName;
|
||||
private final int versionID;
|
||||
|
||||
public SetProjectSQLUpdate(String projectName, int versionID) {
|
||||
this.projectName = projectName;
|
||||
this.versionID = versionID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return SET_PROJECT;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setString(1, projectName);
|
||||
statement.setInt(2, versionID);
|
||||
}
|
||||
}
|
@@ -0,0 +1,37 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class UpdateRestore implements SQLUpdate {
|
||||
private static final String UPDATE_RESTORE =
|
||||
"UPDATE `projects`\n"
|
||||
+ "SET `last_accessed` = ?,\n"
|
||||
+ " `swap_time` = NULL,\n"
|
||||
+ " `restore_time` = ?,\n"
|
||||
+ " `swap_compression` = NULL\n"
|
||||
+ "WHERE `name` = ?;\n";
|
||||
|
||||
private final String projectName;
|
||||
private final Timestamp now;
|
||||
|
||||
public UpdateRestore(String projectName) {
|
||||
this.projectName = projectName;
|
||||
this.now = Timestamp.valueOf(LocalDateTime.now());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return UPDATE_RESTORE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setTimestamp(1, now);
|
||||
statement.setTimestamp(2, now);
|
||||
statement.setString(3, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,39 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.db.sqlite.update.insert;
|
||||
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.LocalDateTime;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.sqlite.SQLUpdate;
|
||||
|
||||
public class UpdateSwap implements SQLUpdate {
|
||||
private static final String UPDATE_SWAP =
|
||||
"UPDATE `projects`\n"
|
||||
+ "SET `last_accessed` = NULL,\n"
|
||||
+ " `swap_time` = ?,\n"
|
||||
+ " `restore_time` = NULL,\n"
|
||||
+ " `swap_compression` = ?\n"
|
||||
+ "WHERE `name` = ?;\n";
|
||||
|
||||
private final String projectName;
|
||||
private final String compression;
|
||||
private final Timestamp now;
|
||||
|
||||
public UpdateSwap(String projectName, String compression) {
|
||||
this.projectName = projectName;
|
||||
this.compression = compression;
|
||||
this.now = Timestamp.valueOf(LocalDateTime.now());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSQL() {
|
||||
return UPDATE_SWAP;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addParametersToStatement(PreparedStatement statement) throws SQLException {
|
||||
statement.setTimestamp(1, now);
|
||||
statement.setString(2, compression);
|
||||
statement.setString(3, projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,29 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.gc;
|
||||
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
/*
|
||||
* Is started by the bridge. Every time a project is updated, we queue it for
|
||||
* GC which executes every hour or so.
|
||||
*
|
||||
* We don't queue it into a more immediate Executor because there is no way to
|
||||
* know if a call to {@link Bridge#updateProject(Optional, ProjectRepo)},
|
||||
* which releases the lock, is going to call
|
||||
* {@link Bridge#push(Optional, String, RawDirectory, RawDirectory, String)}.
|
||||
*
|
||||
* We don't want the GC to run in between an update and a push.
|
||||
*/
|
||||
public interface GcJob {
|
||||
|
||||
void start();
|
||||
|
||||
void stop();
|
||||
|
||||
void onPreGc(Runnable preGc);
|
||||
|
||||
void onPostGc(Runnable postGc);
|
||||
|
||||
void queueForGc(String projectName);
|
||||
|
||||
CompletableFuture<Void> waitForRun();
|
||||
}
|
@@ -0,0 +1,130 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.gc;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.ProjectRepo;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.TimerUtils;
|
||||
|
||||
/*
|
||||
* Implementation of {@link GcJob} using its own Timer and a synchronized
|
||||
* queue.
|
||||
*/
|
||||
public class GcJobImpl implements GcJob {
|
||||
|
||||
private final RepoStore repoStore;
|
||||
private final ProjectLock locks;
|
||||
|
||||
private final long intervalMs;
|
||||
private final Timer timer;
|
||||
|
||||
private final Set<String> gcQueue;
|
||||
|
||||
/*
|
||||
* Hooks in case they are needed, e.g. for testing.
|
||||
*/
|
||||
private AtomicReference<Runnable> preGc;
|
||||
private AtomicReference<Runnable> postGc;
|
||||
|
||||
/* We need to iterate over and empty it after every run */
|
||||
private final Lock jobWaitersLock;
|
||||
private final List<CompletableFuture<Void>> jobWaiters;
|
||||
|
||||
public GcJobImpl(RepoStore repoStore, ProjectLock locks, long intervalMs) {
|
||||
this.repoStore = repoStore;
|
||||
this.locks = locks;
|
||||
this.intervalMs = intervalMs;
|
||||
timer = new Timer();
|
||||
gcQueue = Collections.newSetFromMap(new ConcurrentHashMap<>());
|
||||
preGc = new AtomicReference<>(() -> {});
|
||||
postGc = new AtomicReference<>(() -> {});
|
||||
jobWaitersLock = new ReentrantLock();
|
||||
jobWaiters = new ArrayList<>();
|
||||
}
|
||||
|
||||
public GcJobImpl(RepoStore repoStore, ProjectLock locks) {
|
||||
this(repoStore, locks, TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
Log.info("Starting GC job to run every [{}] ms", intervalMs);
|
||||
timer.scheduleAtFixedRate(TimerUtils.makeTimerTask(this::doGC), intervalMs, intervalMs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
Log.info("Stopping GC job");
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPreGc(Runnable preGc) {
|
||||
this.preGc.set(preGc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPostGc(Runnable postGc) {
|
||||
this.postGc.set(postGc);
|
||||
}
|
||||
|
||||
/*
|
||||
* Needs to be callable from any thread.
|
||||
* @param projectName
|
||||
*/
|
||||
@Override
|
||||
public void queueForGc(String projectName) {
|
||||
gcQueue.add(projectName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> waitForRun() {
|
||||
CompletableFuture<Void> ret = new CompletableFuture<>();
|
||||
jobWaitersLock.lock();
|
||||
try {
|
||||
jobWaiters.add(ret);
|
||||
} finally {
|
||||
jobWaitersLock.unlock();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void doGC() {
|
||||
Log.info("GC job running");
|
||||
int numGcs = 0;
|
||||
preGc.get().run();
|
||||
for (Iterator<String> it = gcQueue.iterator(); it.hasNext(); it.remove(), ++numGcs) {
|
||||
String proj = it.next();
|
||||
Log.debug("[{}] Running GC job on project", proj);
|
||||
try (LockGuard __ = locks.lockGuard(proj)) {
|
||||
try {
|
||||
ProjectRepo repo = repoStore.getExistingRepo(proj);
|
||||
repo.runGC();
|
||||
repo.deleteIncomingPacks();
|
||||
} catch (IOException e) {
|
||||
Log.warn("[{}] Failed to GC project", proj);
|
||||
}
|
||||
} catch (CannotAcquireLockException e) {
|
||||
Log.warn("[{}] Cannot acquire project lock, skipping GC", proj);
|
||||
}
|
||||
}
|
||||
Log.info("GC job finished, num gcs: {}", numGcs);
|
||||
jobWaitersLock.lock();
|
||||
try {
|
||||
jobWaiters.forEach(w -> w.complete(null));
|
||||
} finally {
|
||||
jobWaitersLock.unlock();
|
||||
}
|
||||
postGc.get().run();
|
||||
}
|
||||
}
|
@@ -0,0 +1,9 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.lock;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public interface LockGuard extends AutoCloseable {
|
||||
|
||||
void close();
|
||||
}
|
@@ -0,0 +1,24 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.lock;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
|
||||
/*
|
||||
* Project Lock class.
|
||||
*
|
||||
* The locks should be re-entrant. For example, we are usually holding the lock
|
||||
* when a project must be restored, which tries to acquire the lock again.
|
||||
*/
|
||||
public interface ProjectLock {
|
||||
|
||||
void lockAll();
|
||||
|
||||
void lockForProject(String projectName) throws CannotAcquireLockException;
|
||||
|
||||
void unlockForProject(String projectName);
|
||||
|
||||
/* RAII hahaha */
|
||||
default LockGuard lockGuard(String projectName) throws CannotAcquireLockException {
|
||||
lockForProject(projectName);
|
||||
return () -> unlockForProject(projectName);
|
||||
}
|
||||
}
|
@@ -0,0 +1,164 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import static uk.ac.ic.wlgitbridge.util.Util.deleteInDirectoryApartFrom;
|
||||
|
||||
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.Project;
|
||||
import uk.ac.ic.wlgitbridge.util.Tar;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public class FSGitRepoStore implements RepoStore {
|
||||
|
||||
private static final long DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024;
|
||||
|
||||
private final String repoStorePath;
|
||||
|
||||
private final File rootDirectory;
|
||||
|
||||
private final long maxFileSize;
|
||||
|
||||
private final Function<File, Long> fsSizer;
|
||||
|
||||
public FSGitRepoStore(String repoStorePath, Optional<Long> maxFileSize) {
|
||||
this(
|
||||
repoStorePath,
|
||||
maxFileSize.orElse(DEFAULT_MAX_FILE_SIZE),
|
||||
d -> d.getTotalSpace() - d.getFreeSpace());
|
||||
}
|
||||
|
||||
public FSGitRepoStore(String repoStorePath, long maxFileSize, Function<File, Long> fsSizer) {
|
||||
this.repoStorePath = repoStorePath;
|
||||
rootDirectory = initRootGitDirectory(repoStorePath);
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.fsSizer = fsSizer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRepoStorePath() {
|
||||
return repoStorePath;
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getRootDirectory() {
|
||||
return rootDirectory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectRepo initRepo(String project) throws IOException {
|
||||
GitProjectRepo ret = GitProjectRepo.fromName(project);
|
||||
ret.initRepo(this);
|
||||
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectRepo getExistingRepo(String project) throws IOException {
|
||||
GitProjectRepo ret = GitProjectRepo.fromName(project);
|
||||
ret.useExistingRepository(this);
|
||||
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectRepo useJGitRepo(Repository repo, ObjectId commitId) {
|
||||
GitProjectRepo ret = GitProjectRepo.fromJGitRepo(repo);
|
||||
return new WalkOverrideGitRepo(ret, Optional.of(maxFileSize), Optional.of(commitId));
|
||||
}
|
||||
|
||||
/* TODO: Perhaps we should just delete bad directories on the fly. */
|
||||
@Override
|
||||
public void purgeNonexistentProjects(Collection<String> existingProjectNames) {
|
||||
List<String> excludedFromDeletion = new ArrayList<>(existingProjectNames);
|
||||
excludedFromDeletion.add(".wlgb");
|
||||
deleteInDirectoryApartFrom(rootDirectory, excludedFromDeletion.toArray(new String[] {}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public long totalSize() {
|
||||
return fsSizer.apply(rootDirectory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream bzip2Project(String projectName, long[] sizePtr) throws IOException {
|
||||
Project.checkValidProjectName(projectName);
|
||||
Log.debug("[{}] bzip2 project", projectName);
|
||||
return Tar.bz2.zip(getDotGitForProject(projectName), sizePtr);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream gzipProject(String projectName, long[] sizePtr) throws IOException {
|
||||
Project.checkValidProjectName(projectName);
|
||||
Log.debug("[{}] gzip project", projectName);
|
||||
return Tar.gzip.zip(getDotGitForProject(projectName), sizePtr);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void gcProject(String projectName) throws IOException {
|
||||
Project.checkValidProjectName(projectName);
|
||||
ProjectRepo repo = getExistingRepo(projectName);
|
||||
repo.runGC();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String projectName) throws IOException {
|
||||
Project.checkValidProjectName(projectName);
|
||||
FileUtils.deleteDirectory(new File(rootDirectory, projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unbzip2Project(String projectName, InputStream dataStream) throws IOException {
|
||||
Preconditions.checkArgument(
|
||||
Project.isValidProjectName(projectName), "[%s] invalid project name: ", projectName);
|
||||
Preconditions.checkState(
|
||||
getDirForProject(projectName).mkdirs(),
|
||||
"[%s] directories for " + "evicted project already exist",
|
||||
projectName);
|
||||
Log.debug("[{}] un-bzip2 project", projectName);
|
||||
Tar.bz2.unzip(dataStream, getDirForProject(projectName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void ungzipProject(String projectName, InputStream dataStream) throws IOException {
|
||||
Preconditions.checkArgument(
|
||||
Project.isValidProjectName(projectName), "[%s] invalid project name: ", projectName);
|
||||
Preconditions.checkState(
|
||||
getDirForProject(projectName).mkdirs(),
|
||||
"[%s] directories for " + "evicted project already exist",
|
||||
projectName);
|
||||
Log.debug("[{}] un-gzip project", projectName);
|
||||
Tar.gzip.unzip(dataStream, getDirForProject(projectName));
|
||||
}
|
||||
|
||||
private File getDirForProject(String projectName) {
|
||||
Project.checkValidProjectName(projectName);
|
||||
return Paths.get(rootDirectory.getAbsolutePath()).resolve(projectName).toFile();
|
||||
}
|
||||
|
||||
private File getDotGitForProject(String projectName) {
|
||||
Project.checkValidProjectName(projectName);
|
||||
return Paths.get(rootDirectory.getAbsolutePath()).resolve(projectName).resolve(".git").toFile();
|
||||
}
|
||||
|
||||
private File initRootGitDirectory(String rootGitDirectoryPath) {
|
||||
File rootGitDirectory = new File(rootGitDirectoryPath);
|
||||
rootGitDirectory.mkdirs();
|
||||
Preconditions.checkArgument(
|
||||
rootGitDirectory.isDirectory(),
|
||||
"given root git directory " + "is not a directory: %s",
|
||||
rootGitDirectory.getAbsolutePath());
|
||||
return rootGitDirectory;
|
||||
}
|
||||
}
|
@@ -0,0 +1,231 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.FileVisitor;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.*;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.eclipse.jgit.api.Git;
|
||||
import org.eclipse.jgit.api.ResetCommand;
|
||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||
import org.eclipse.jgit.lib.PersonIdent;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.Project;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Class representing a Git repository.
|
||||
*
|
||||
* It stores the projectName and repo separately because the hooks need to be
|
||||
* able to construct one of these without knowing whether the repo exists yet.
|
||||
*
|
||||
* It can then be passed to the Bridge, which will either
|
||||
* {@link #initRepo(RepoStore)} for a never-seen-before repo, or
|
||||
* {@link #useExistingRepository(RepoStore)} for an existing repo.
|
||||
*
|
||||
* Make sure to acquire the project lock before calling methods here.
|
||||
*/
|
||||
public class GitProjectRepo implements ProjectRepo {
|
||||
|
||||
private final String projectName;
|
||||
private Optional<Repository> repository;
|
||||
|
||||
public static GitProjectRepo fromJGitRepo(Repository repo) {
|
||||
return new GitProjectRepo(repo.getWorkTree().getName(), Optional.of(repo));
|
||||
}
|
||||
|
||||
public static GitProjectRepo fromName(String projectName) {
|
||||
return new GitProjectRepo(projectName, Optional.empty());
|
||||
}
|
||||
|
||||
GitProjectRepo(String projectName, Optional<Repository> repository) {
|
||||
Preconditions.checkArgument(Project.isValidProjectName(projectName));
|
||||
this.projectName = projectName;
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getProjectName() {
|
||||
return projectName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initRepo(RepoStore repoStore) throws IOException {
|
||||
initRepositoryField(repoStore);
|
||||
Preconditions.checkState(repository.isPresent());
|
||||
Repository repo = this.repository.get();
|
||||
// TODO: assert that this is a fresh repo. At the moment, we can't be
|
||||
// sure whether the repo to be init'd doesn't exist or is just fresh
|
||||
// and we crashed / aborted while committing
|
||||
if (repo.getObjectDatabase().exists()) return;
|
||||
repo.create();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useExistingRepository(RepoStore repoStore) throws IOException {
|
||||
initRepositoryField(repoStore);
|
||||
Preconditions.checkState(repository.isPresent());
|
||||
Preconditions.checkState(repository.get().getObjectDatabase().exists());
|
||||
}
|
||||
|
||||
@Override
|
||||
public RawDirectory getDirectory() throws IOException, GitUserException {
|
||||
Preconditions.checkState(repository.isPresent());
|
||||
return new RepositoryObjectTreeWalker(repository.get()).getDirectoryContents(Optional.empty());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> commitAndGetMissing(GitDirectoryContents contents) throws IOException {
|
||||
try {
|
||||
return doCommitAndGetMissing(contents);
|
||||
} catch (GitAPIException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void runGC() throws IOException {
|
||||
Preconditions.checkState(repository.isPresent(), "Repo is not present");
|
||||
File dir = getProjectDir();
|
||||
Preconditions.checkState(dir.isDirectory());
|
||||
Log.debug("[{}] Running git gc", projectName);
|
||||
Process proc = new ProcessBuilder("git", "gc").directory(dir).start();
|
||||
int exitCode;
|
||||
try {
|
||||
exitCode = proc.waitFor();
|
||||
Log.debug("Exit: {}", exitCode);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
if (exitCode != 0) {
|
||||
Log.warn("[{}] Git gc failed", dir.getAbsolutePath());
|
||||
Log.warn(IOUtils.toString(proc.getInputStream(), StandardCharsets.UTF_8));
|
||||
Log.warn(IOUtils.toString(proc.getErrorStream(), StandardCharsets.UTF_8));
|
||||
throw new IOException("git gc error");
|
||||
}
|
||||
Log.debug("[{}] git gc successful", projectName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteIncomingPacks() throws IOException {
|
||||
Log.debug("[{}] Checking for garbage `incoming` files", projectName);
|
||||
Files.walkFileTree(
|
||||
getDotGitDir().toPath(),
|
||||
new FileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
|
||||
throws IOException {
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
|
||||
throws IOException {
|
||||
File file_ = file.toFile();
|
||||
String name = file_.getName();
|
||||
if (name.startsWith("incoming_") && name.endsWith(".pack")) {
|
||||
Log.debug("Deleting garbage `incoming` file: {}", file_);
|
||||
Preconditions.checkState(file_.delete());
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
|
||||
Preconditions.checkNotNull(file);
|
||||
Preconditions.checkNotNull(exc);
|
||||
Log.warn("Failed to visit file: " + file, exc);
|
||||
return FileVisitResult.TERMINATE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
|
||||
Preconditions.checkNotNull(dir);
|
||||
if (exc != null) {
|
||||
return FileVisitResult.TERMINATE;
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getProjectDir() {
|
||||
return getJGitRepository().getDirectory().getParentFile();
|
||||
}
|
||||
|
||||
public void resetHard() throws IOException {
|
||||
Git git = new Git(getJGitRepository());
|
||||
try {
|
||||
git.reset().setMode(ResetCommand.ResetType.HARD).call();
|
||||
} catch (GitAPIException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Repository getJGitRepository() {
|
||||
return repository.get();
|
||||
}
|
||||
|
||||
public File getDotGitDir() {
|
||||
return getJGitRepository().getWorkTree();
|
||||
}
|
||||
|
||||
private void initRepositoryField(RepoStore repoStore) throws IOException {
|
||||
Preconditions.checkNotNull(repoStore);
|
||||
Preconditions.checkArgument(Project.isValidProjectName(projectName));
|
||||
Preconditions.checkState(!repository.isPresent());
|
||||
repository = Optional.of(createJGitRepository(repoStore, projectName));
|
||||
}
|
||||
|
||||
private Repository createJGitRepository(RepoStore repoStore, String projName) throws IOException {
|
||||
File repoDir = new File(repoStore.getRootDirectory(), projName);
|
||||
return new FileRepositoryBuilder().setWorkTree(repoDir).build();
|
||||
}
|
||||
|
||||
private Collection<String> doCommitAndGetMissing(GitDirectoryContents contents)
|
||||
throws IOException, GitAPIException {
|
||||
Preconditions.checkState(repository.isPresent());
|
||||
Repository repo = getJGitRepository();
|
||||
resetHard();
|
||||
String name = getProjectName();
|
||||
Log.debug("[{}] Writing commit", name);
|
||||
contents.write();
|
||||
Git git = new Git(getJGitRepository());
|
||||
Log.debug("[{}] Getting missing files", name);
|
||||
Set<String> missingFiles = git.status().call().getMissing();
|
||||
for (String missing : missingFiles) {
|
||||
Log.debug("[{}] Git rm {}", name, missing);
|
||||
git.rm().setCached(true).addFilepattern(missing).call();
|
||||
}
|
||||
Log.debug("[{}] Calling Git add", name);
|
||||
git.add().setWorkingTreeIterator(new NoGitignoreIterator(repo)).addFilepattern(".").call();
|
||||
Log.debug("[{}] Calling Git commit", name);
|
||||
git.commit()
|
||||
.setAuthor(
|
||||
new PersonIdent(
|
||||
contents.getUserName(),
|
||||
contents.getUserEmail(),
|
||||
contents.getWhen(),
|
||||
TimeZone.getDefault()))
|
||||
.setMessage(contents.getCommitMessage())
|
||||
.call();
|
||||
Log.debug(
|
||||
"[{}] Deleting files in directory: {}", name, contents.getDirectory().getAbsolutePath());
|
||||
Util.deleteInDirectoryApartFrom(contents.getDirectory(), ".git");
|
||||
return missingFiles;
|
||||
}
|
||||
}
|
@@ -0,0 +1,73 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Field;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.treewalk.AbstractTreeIterator;
|
||||
import org.eclipse.jgit.treewalk.FileTreeIterator;
|
||||
import org.eclipse.jgit.treewalk.WorkingTreeIterator;
|
||||
import org.eclipse.jgit.treewalk.WorkingTreeOptions;
|
||||
import org.eclipse.jgit.util.FS;
|
||||
|
||||
/*
|
||||
* Created by winston on 08/10/2016.
|
||||
*/
|
||||
public class NoGitignoreIterator extends FileTreeIterator {
|
||||
|
||||
private static final Field ignoreNodeField;
|
||||
|
||||
static {
|
||||
try {
|
||||
ignoreNodeField = WorkingTreeIterator.class.getDeclaredField("ignoreNode");
|
||||
} catch (NoSuchFieldException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
ignoreNodeField.setAccessible(true);
|
||||
}
|
||||
|
||||
public NoGitignoreIterator(Repository repo) {
|
||||
super(repo);
|
||||
}
|
||||
|
||||
public NoGitignoreIterator(Repository repo, FileModeStrategy fileModeStrategy) {
|
||||
super(repo, fileModeStrategy);
|
||||
}
|
||||
|
||||
public NoGitignoreIterator(File root, FS fs, WorkingTreeOptions options) {
|
||||
super(root, fs, options);
|
||||
}
|
||||
|
||||
public NoGitignoreIterator(
|
||||
File root, FS fs, WorkingTreeOptions options, FileModeStrategy fileModeStrategy) {
|
||||
super(root, fs, options, fileModeStrategy);
|
||||
}
|
||||
|
||||
protected NoGitignoreIterator(FileTreeIterator p, File root, FS fs) {
|
||||
super(p, root, fs);
|
||||
}
|
||||
|
||||
protected NoGitignoreIterator(
|
||||
WorkingTreeIterator p, File root, FS fs, FileModeStrategy fileModeStrategy) {
|
||||
super(p, root, fs, fileModeStrategy);
|
||||
}
|
||||
|
||||
// Note: the `list` is a list of top-level entities in this directory,
|
||||
// not a full list of files in the tree.
|
||||
@Override
|
||||
protected void init(Entry[] list) {
|
||||
super.init(list);
|
||||
try {
|
||||
ignoreNodeField.set(this, null);
|
||||
} catch (IllegalAccessException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// When entering a sub-directory, create a new instance of this class,
|
||||
// so we can also ignore gitignore specifications in sub-directories
|
||||
@Override
|
||||
protected AbstractTreeIterator enterSubtree() {
|
||||
String fullPath = getDirectory().getAbsolutePath() + "/" + current().getName();
|
||||
return new NoGitignoreIterator(this, new File(fullPath), fs, fileModeStrategy);
|
||||
}
|
||||
}
|
@@ -0,0 +1,34 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface ProjectRepo {
|
||||
|
||||
String getProjectName();
|
||||
|
||||
void initRepo(RepoStore repoStore) throws IOException;
|
||||
|
||||
void useExistingRepository(RepoStore repoStore) throws IOException;
|
||||
|
||||
RawDirectory getDirectory() throws IOException, GitUserException;
|
||||
|
||||
Collection<String> commitAndGetMissing(GitDirectoryContents gitDirectoryContents)
|
||||
throws IOException, GitUserException;
|
||||
|
||||
void runGC() throws IOException;
|
||||
|
||||
void deleteIncomingPacks() throws IOException;
|
||||
|
||||
File getProjectDir();
|
||||
|
||||
Repository getJGitRepository();
|
||||
}
|
@@ -0,0 +1,82 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Collection;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface RepoStore {
|
||||
|
||||
/* Still need to get rid of these two methods.
|
||||
Main dependency: GitRepoStore needs a Repository which needs a directory.
|
||||
Instead, use a visitor or something. */
|
||||
String getRepoStorePath();
|
||||
|
||||
File getRootDirectory();
|
||||
|
||||
ProjectRepo initRepo(String project) throws IOException;
|
||||
|
||||
ProjectRepo getExistingRepo(String project) throws IOException;
|
||||
|
||||
ProjectRepo useJGitRepo(Repository repo, ObjectId commitId);
|
||||
|
||||
void purgeNonexistentProjects(Collection<String> existingProjectNames);
|
||||
|
||||
long totalSize();
|
||||
|
||||
/*
|
||||
* Tars and bzip2s the .git directory of the given project. Throws an
|
||||
* IOException if the project doesn't exist. The returned stream is a copy
|
||||
* of the original .git directory, which must be deleted using remove().
|
||||
*/
|
||||
InputStream bzip2Project(String projectName, long[] sizePtr) throws IOException;
|
||||
|
||||
default InputStream bzip2Project(String projectName) throws IOException {
|
||||
return bzip2Project(projectName, null);
|
||||
}
|
||||
|
||||
/*
|
||||
* Tars and gzips the .git directory of the given project. Throws an
|
||||
* IOException if the project doesn't exist. The returned stream is a copy
|
||||
* of the original .git directory, which must be deleted using remove().
|
||||
*/
|
||||
InputStream gzipProject(String projectName, long[] sizePtr) throws IOException;
|
||||
|
||||
default InputStream gzipProject(String projectName) throws IOException {
|
||||
return gzipProject(projectName, null);
|
||||
}
|
||||
|
||||
void gcProject(String projectName) throws IOException;
|
||||
|
||||
/*
|
||||
* Called after {@link #bzip2Project(String, long[])}'s has been safely
|
||||
* uploaded to the swap store. Removes all traces of the project from disk,
|
||||
* i.e. not just its .git, but the whole project's git directory.
|
||||
* @param projectName
|
||||
* @throws IOException
|
||||
*/
|
||||
void remove(String projectName) throws IOException;
|
||||
|
||||
/*
|
||||
* Unbzip2s the given data stream into a .git directory for projectName.
|
||||
* Creates the project's git directory.
|
||||
* If projectName already exists, throws an IOException.
|
||||
* @param projectName the name of the project, e.g. abc123
|
||||
* @param dataStream the data stream containing the bzipped contents.
|
||||
*/
|
||||
void unbzip2Project(String projectName, InputStream dataStream) throws IOException;
|
||||
|
||||
/*
|
||||
* Ungzips the given data stream into a .git directory for projectName.
|
||||
* Creates the project's git directory.
|
||||
* If projectName already exists, throws an IOException.
|
||||
* @param projectName the name of the project, e.g. abc123
|
||||
* @param dataStream the data stream containing the gzip contents.
|
||||
*/
|
||||
void ungzipProject(String projectName, InputStream dataStream) throws IOException;
|
||||
}
|
@@ -0,0 +1,27 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import java.util.Optional;
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
/*
|
||||
* Created by winston on 02/07/2017.
|
||||
*/
|
||||
public class RepoStoreConfig {
|
||||
|
||||
@Nullable private final Long maxFileSize;
|
||||
|
||||
@Nullable private final Long maxFileNum;
|
||||
|
||||
public RepoStoreConfig(Long maxFileSize, Long maxFileNum) {
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.maxFileNum = maxFileNum;
|
||||
}
|
||||
|
||||
public Optional<Long> getMaxFileSize() {
|
||||
return Optional.ofNullable(maxFileSize);
|
||||
}
|
||||
|
||||
public Optional<Long> getMaxFileNum() {
|
||||
return Optional.ofNullable(maxFileNum);
|
||||
}
|
||||
}
|
@@ -0,0 +1,89 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.repo;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.GitDirectoryContents;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.git.util.RepositoryObjectTreeWalker;
|
||||
|
||||
/*
|
||||
* This class takes a GitProjectRepo and delegates all calls to it.
|
||||
*
|
||||
* The purpose is to insert a file size check in {@link #getDirectory()}.
|
||||
*
|
||||
* We delegate instead of subclass because we can't override the static
|
||||
* constructors in {@link GitProjectRepo}.
|
||||
*/
|
||||
public class WalkOverrideGitRepo implements ProjectRepo {
|
||||
|
||||
private final GitProjectRepo gitRepo;
|
||||
|
||||
private final Optional<Long> maxFileSize;
|
||||
|
||||
private final Optional<ObjectId> commitId;
|
||||
|
||||
public WalkOverrideGitRepo(
|
||||
GitProjectRepo gitRepo, Optional<Long> maxFileSize, Optional<ObjectId> commitId) {
|
||||
this.gitRepo = gitRepo;
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.commitId = commitId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getProjectName() {
|
||||
return gitRepo.getProjectName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initRepo(RepoStore repoStore) throws IOException {
|
||||
gitRepo.initRepo(repoStore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void useExistingRepository(RepoStore repoStore) throws IOException {
|
||||
gitRepo.useExistingRepository(repoStore);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RawDirectory getDirectory() throws IOException, GitUserException {
|
||||
Repository repo = gitRepo.getJGitRepository();
|
||||
RepositoryObjectTreeWalker walker;
|
||||
if (commitId.isPresent()) {
|
||||
walker = new RepositoryObjectTreeWalker(repo, commitId.get());
|
||||
} else {
|
||||
walker = new RepositoryObjectTreeWalker(repo);
|
||||
}
|
||||
return walker.getDirectoryContents(maxFileSize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> commitAndGetMissing(GitDirectoryContents gitDirectoryContents)
|
||||
throws GitUserException, IOException {
|
||||
return gitRepo.commitAndGetMissing(gitDirectoryContents);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void runGC() throws IOException {
|
||||
gitRepo.runGC();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteIncomingPacks() throws IOException {
|
||||
gitRepo.deleteIncomingPacks();
|
||||
}
|
||||
|
||||
@Override
|
||||
public File getProjectDir() {
|
||||
return gitRepo.getProjectDir();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Repository getJGitRepository() {
|
||||
return gitRepo.getJGitRepository();
|
||||
}
|
||||
}
|
@@ -0,0 +1,22 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.resource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface ResourceCache {
|
||||
|
||||
RawFile get(
|
||||
String projectName,
|
||||
String url,
|
||||
String newPath,
|
||||
Map<String, RawFile> fileTable,
|
||||
Map<String, byte[]> fetchedUrls,
|
||||
Optional<Long> maxFileSize)
|
||||
throws IOException, SizeLimitExceededException;
|
||||
}
|
@@ -0,0 +1,139 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.resource;
|
||||
|
||||
import static org.asynchttpclient.Dsl.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RepositoryFile;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.SizeLimitExceededException;
|
||||
import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClient;
|
||||
import uk.ac.ic.wlgitbridge.io.http.ning.NingHttpClientFacade;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public class UrlResourceCache implements ResourceCache {
|
||||
|
||||
private final DBStore dbStore;
|
||||
|
||||
private final NingHttpClientFacade http;
|
||||
|
||||
UrlResourceCache(DBStore dbStore, NingHttpClientFacade http) {
|
||||
this.dbStore = dbStore;
|
||||
this.http = http;
|
||||
}
|
||||
|
||||
public UrlResourceCache(DBStore dbStore) {
|
||||
this(dbStore, new NingHttpClient(asyncHttpClient()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public RawFile get(
|
||||
String projectName,
|
||||
String url,
|
||||
String newPath,
|
||||
Map<String, RawFile> fileTable,
|
||||
Map<String, byte[]> fetchedUrls,
|
||||
Optional<Long> maxFileSize)
|
||||
throws IOException, SizeLimitExceededException {
|
||||
String path = dbStore.getPathForURLInProject(projectName, getCacheKeyFromUrl(url));
|
||||
byte[] contents;
|
||||
if (path == null) {
|
||||
path = newPath;
|
||||
contents = fetch(projectName, url, path, maxFileSize);
|
||||
fetchedUrls.put(url, contents);
|
||||
} else {
|
||||
Log.debug("Found (" + projectName + "): " + url);
|
||||
Log.debug("At (" + projectName + "): " + path);
|
||||
contents = fetchedUrls.get(url);
|
||||
if (contents == null) {
|
||||
RawFile rawFile = fileTable.get(path);
|
||||
if (rawFile == null) {
|
||||
Log.warn(
|
||||
"File "
|
||||
+ path
|
||||
+ " was not in the current commit, "
|
||||
+ "or the git tree, yet path was not null. "
|
||||
+ "File url is: "
|
||||
+ url);
|
||||
contents = fetch(projectName, url, path, maxFileSize);
|
||||
} else {
|
||||
contents = rawFile.getContents();
|
||||
}
|
||||
}
|
||||
}
|
||||
return new RepositoryFile(newPath, contents);
|
||||
}
|
||||
|
||||
private byte[] fetch(
|
||||
String projectName, final String url, String path, Optional<Long> maxFileSize)
|
||||
throws FailedConnectionException, SizeLimitExceededException {
|
||||
byte[] contents;
|
||||
Log.debug("GET -> " + url);
|
||||
try {
|
||||
contents =
|
||||
http.get(
|
||||
url,
|
||||
hs -> {
|
||||
List<String> contentLengths = hs.getAll("Content-Length");
|
||||
if (!maxFileSize.isPresent()) {
|
||||
return true;
|
||||
}
|
||||
if (contentLengths.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
long contentLength = Long.parseLong(contentLengths.get(0));
|
||||
long maxFileSize_ = maxFileSize.get();
|
||||
if (contentLength <= maxFileSize_) {
|
||||
return true;
|
||||
}
|
||||
throw new SizeLimitExceededException(
|
||||
Optional.of(path), contentLength, maxFileSize_);
|
||||
});
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof SizeLimitExceededException) {
|
||||
throw (SizeLimitExceededException) cause;
|
||||
}
|
||||
Log.warn(
|
||||
"ExecutionException when fetching project: "
|
||||
+ projectName
|
||||
+ ", url: "
|
||||
+ url
|
||||
+ ", path: "
|
||||
+ path,
|
||||
e);
|
||||
throw new FailedConnectionException();
|
||||
}
|
||||
if (maxFileSize.isPresent() && contents.length > maxFileSize.get()) {
|
||||
throw new SizeLimitExceededException(Optional.of(path), contents.length, maxFileSize.get());
|
||||
}
|
||||
dbStore.addURLIndexForProject(projectName, getCacheKeyFromUrl(url), path);
|
||||
return contents;
|
||||
}
|
||||
|
||||
/*
|
||||
* Construct a suitable cache key from the given file URL.
|
||||
*
|
||||
* The file URL returned by the web service may contain a token parameter
|
||||
* used for authentication. This token changes for every request, so we
|
||||
* need to strip it from the query string before using the URL as a cache
|
||||
* key.
|
||||
*/
|
||||
private String getCacheKeyFromUrl(String url) {
|
||||
// We're not doing proper URL parsing here, but it should be enough to
|
||||
// remove the token without touching the important parts of the URL.
|
||||
//
|
||||
// The URL looks like:
|
||||
//
|
||||
// https://history.overleaf.com/api/projects/:project_id/blobs/:hash?token=:token&_path=:path
|
||||
return url.replaceAll("token=[^&]*", "token=REMOVED");
|
||||
}
|
||||
}
|
@@ -0,0 +1,47 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.snapshot;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocRequest;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionRequest;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersRequest;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PushRequest;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public class NetSnapshotApi implements SnapshotApi {
|
||||
|
||||
@Override
|
||||
public CompletableFuture<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName) {
|
||||
return new GetDocRequest(opt(oauth2), projectName).request();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<GetForVersionResult> getForVersion(
|
||||
Optional<Credential> oauth2, String projectName, int versionId) {
|
||||
return new GetForVersionRequest(opt(oauth2), projectName, versionId).request();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<GetSavedVersResult> getSavedVers(
|
||||
Optional<Credential> oauth2, String projectName) {
|
||||
return new GetSavedVersRequest(opt(oauth2), projectName).request();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<PushResult> push(
|
||||
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey) {
|
||||
return new PushRequest(opt(oauth2), candidateSnapshot, postbackKey).request();
|
||||
}
|
||||
|
||||
private static Credential opt(Optional<Credential> oauth2) {
|
||||
return oauth2.orElse(null);
|
||||
}
|
||||
}
|
@@ -0,0 +1,49 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.snapshot;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface SnapshotApi {
|
||||
|
||||
CompletableFuture<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName);
|
||||
|
||||
CompletableFuture<GetForVersionResult> getForVersion(
|
||||
Optional<Credential> oauth2, String projectName, int versionId);
|
||||
|
||||
CompletableFuture<GetSavedVersResult> getSavedVers(
|
||||
Optional<Credential> oauth2, String projectName);
|
||||
|
||||
CompletableFuture<PushResult> push(
|
||||
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey);
|
||||
|
||||
static <T> T getResult(CompletableFuture<T> result)
|
||||
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
|
||||
try {
|
||||
return result.join();
|
||||
} catch (CompletionException e) {
|
||||
try {
|
||||
throw e.getCause();
|
||||
} catch (MissingRepositoryException
|
||||
| FailedConnectionException
|
||||
| ForbiddenException
|
||||
| RuntimeException r) {
|
||||
throw r;
|
||||
} catch (Throwable __) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,121 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.snapshot;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import uk.ac.ic.wlgitbridge.data.CandidateSnapshot;
|
||||
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.MissingRepositoryException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.exception.FailedConnectionException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getdoc.GetDocResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.GetForVersionResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.GetSavedVersResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.PushResult;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InvalidProjectException;
|
||||
|
||||
/*
|
||||
* Created by winston on 02/07/2017.
|
||||
*/
|
||||
public class SnapshotApiFacade {
|
||||
|
||||
private final SnapshotApi api;
|
||||
|
||||
public SnapshotApiFacade(SnapshotApi api) {
|
||||
this.api = api;
|
||||
}
|
||||
|
||||
public boolean projectExists(Optional<Credential> oauth2, String projectName)
|
||||
throws FailedConnectionException, GitUserException {
|
||||
try {
|
||||
SnapshotApi.getResult(api.getDoc(oauth2, projectName)).getVersionID();
|
||||
return true;
|
||||
} catch (InvalidProjectException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public Optional<GetDocResult> getDoc(Optional<Credential> oauth2, String projectName)
|
||||
throws FailedConnectionException, GitUserException {
|
||||
try {
|
||||
GetDocResult doc = SnapshotApi.getResult(api.getDoc(oauth2, projectName));
|
||||
doc.getVersionID();
|
||||
return Optional.of(doc);
|
||||
} catch (InvalidProjectException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
public Deque<Snapshot> getSnapshots(
|
||||
Optional<Credential> oauth2, String projectName, int afterVersionId)
|
||||
throws GitUserException, FailedConnectionException {
|
||||
List<SnapshotInfo> snapshotInfos =
|
||||
getSnapshotInfosAfterVersion(oauth2, projectName, afterVersionId);
|
||||
List<SnapshotData> snapshotDatas = getMatchingSnapshotData(oauth2, projectName, snapshotInfos);
|
||||
return combine(snapshotInfos, snapshotDatas);
|
||||
}
|
||||
|
||||
public PushResult push(
|
||||
Optional<Credential> oauth2, CandidateSnapshot candidateSnapshot, String postbackKey)
|
||||
throws MissingRepositoryException, FailedConnectionException, ForbiddenException {
|
||||
return SnapshotApi.getResult(api.push(oauth2, candidateSnapshot, postbackKey));
|
||||
}
|
||||
|
||||
private List<SnapshotInfo> getSnapshotInfosAfterVersion(
|
||||
Optional<Credential> oauth2, String projectName, int version)
|
||||
throws FailedConnectionException, GitUserException {
|
||||
SortedSet<SnapshotInfo> versions = new TreeSet<>();
|
||||
CompletableFuture<GetDocResult> getDoc = api.getDoc(oauth2, projectName);
|
||||
CompletableFuture<GetSavedVersResult> savedVers = api.getSavedVers(oauth2, projectName);
|
||||
GetDocResult latestDoc = SnapshotApi.getResult(getDoc);
|
||||
int latest = latestDoc.getVersionID();
|
||||
// Handle edge-case for projects with no changes, that were imported
|
||||
// to v2. In which case both `latest` and `version` will be zero.
|
||||
// See: https://github.com/overleaf/writelatex-git-bridge/pull/50
|
||||
if (latest > version || (latest == 0 && version == 0)) {
|
||||
for (SnapshotInfo snapshotInfo : SnapshotApi.getResult(savedVers).getSavedVers()) {
|
||||
if (snapshotInfo.getVersionId() > version) {
|
||||
versions.add(snapshotInfo);
|
||||
}
|
||||
}
|
||||
versions.add(
|
||||
new SnapshotInfo(
|
||||
latest, latestDoc.getCreatedAt(), latestDoc.getName(), latestDoc.getEmail()));
|
||||
}
|
||||
return new ArrayList<>(versions);
|
||||
}
|
||||
|
||||
private List<SnapshotData> getMatchingSnapshotData(
|
||||
Optional<Credential> oauth2, String projectName, List<SnapshotInfo> snapshotInfos)
|
||||
throws FailedConnectionException, ForbiddenException {
|
||||
List<CompletableFuture<GetForVersionResult>> firedRequests =
|
||||
fireDataRequests(oauth2, projectName, snapshotInfos);
|
||||
List<SnapshotData> snapshotDataList = new ArrayList<>();
|
||||
for (CompletableFuture<GetForVersionResult> fired : firedRequests) {
|
||||
snapshotDataList.add(fired.join().getSnapshotData());
|
||||
}
|
||||
return snapshotDataList;
|
||||
}
|
||||
|
||||
private List<CompletableFuture<GetForVersionResult>> fireDataRequests(
|
||||
Optional<Credential> oauth2, String projectName, List<SnapshotInfo> snapshotInfos) {
|
||||
return snapshotInfos.stream()
|
||||
.map(snap -> api.getForVersion(oauth2, projectName, snap.getVersionId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private static Deque<Snapshot> combine(
|
||||
List<SnapshotInfo> snapshotInfos, List<SnapshotData> snapshotDatas) {
|
||||
Deque<Snapshot> snapshots = new LinkedList<>();
|
||||
Iterator<SnapshotInfo> infos = snapshotInfos.iterator();
|
||||
Iterator<SnapshotData> datas = snapshotDatas.iterator();
|
||||
while (infos.hasNext()) {
|
||||
snapshots.add(new Snapshot(infos.next(), datas.next()));
|
||||
}
|
||||
return snapshots;
|
||||
}
|
||||
}
|
@@ -0,0 +1,19 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.job;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class NoopSwapJob implements SwapJob {
|
||||
|
||||
@Override
|
||||
public void start() {}
|
||||
|
||||
@Override
|
||||
public void stop() {}
|
||||
|
||||
@Override
|
||||
public void evict(String projName) {}
|
||||
|
||||
@Override
|
||||
public void restore(String projName) {}
|
||||
}
|
@@ -0,0 +1,124 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.job;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface SwapJob {
|
||||
|
||||
enum CompressionMethod {
|
||||
Bzip2,
|
||||
Gzip
|
||||
}
|
||||
|
||||
static CompressionMethod stringToCompressionMethod(String compressionString) {
|
||||
if (compressionString == null) {
|
||||
return null;
|
||||
}
|
||||
CompressionMethod result;
|
||||
switch (compressionString) {
|
||||
case "gzip":
|
||||
result = CompressionMethod.Gzip;
|
||||
break;
|
||||
case "bzip2":
|
||||
result = CompressionMethod.Bzip2;
|
||||
break;
|
||||
default:
|
||||
result = null;
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static String compressionMethodAsString(CompressionMethod compressionMethod) {
|
||||
if (compressionMethod == null) {
|
||||
return null;
|
||||
}
|
||||
String result;
|
||||
switch (compressionMethod) {
|
||||
case Gzip:
|
||||
result = "gzip";
|
||||
break;
|
||||
case Bzip2:
|
||||
result = "bzip2";
|
||||
break;
|
||||
default:
|
||||
result = null;
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static SwapJob fromConfig(
|
||||
Optional<SwapJobConfig> cfg,
|
||||
ProjectLock lock,
|
||||
RepoStore repoStore,
|
||||
DBStore dbStore,
|
||||
SwapStore swapStore) {
|
||||
if (!cfg.isPresent()) {
|
||||
return new NoopSwapJob();
|
||||
}
|
||||
if (!swapStore.isSafe() && !cfg.get().getAllowUnsafeStores()) {
|
||||
Log.warn(
|
||||
"Swap store '{}' is not safe; disabling swap job", swapStore.getClass().getSimpleName());
|
||||
return new NoopSwapJob();
|
||||
}
|
||||
return new SwapJobImpl(cfg.get(), lock, repoStore, dbStore, swapStore);
|
||||
}
|
||||
|
||||
/*
|
||||
* Starts the swap job, which should schedule an attempted swap at the given
|
||||
* configured interval (config["swapJob"]["intervalMillis"]
|
||||
*/
|
||||
void start();
|
||||
|
||||
/*
|
||||
* Stops the stop job.
|
||||
*/
|
||||
void stop();
|
||||
|
||||
/*
|
||||
* Called by the swap job when a project should be evicted.
|
||||
*
|
||||
* Pre:
|
||||
* 1. projName must be in repoStore
|
||||
* 2. projName should not be in swapStore
|
||||
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
|
||||
*
|
||||
* Acquires the project lock and performs an eviction of projName.
|
||||
*
|
||||
* Post:
|
||||
* 1. projName should not in repoStore
|
||||
* 2. projName must be in swapStore
|
||||
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
|
||||
* @param projName
|
||||
* @throws IOException
|
||||
*/
|
||||
void evict(String projName) throws IOException;
|
||||
|
||||
/*
|
||||
* Called on a project when it must be restored.
|
||||
*
|
||||
* Pre:
|
||||
* 1. projName should not be in repoStore
|
||||
* 2. projName must be in swapStore
|
||||
* 3. projName must be SWAPPED in dbStore (last_accessed is null)
|
||||
*
|
||||
* Acquires the project lock and restores projName.
|
||||
*
|
||||
* Post:
|
||||
* 1. projName must be in repoStore
|
||||
* 2. projName should not in swapStore
|
||||
* 3. projName should be PRESENT in dbStore (last_accessed is not null)
|
||||
* @param projName
|
||||
* @throws IOException
|
||||
*/
|
||||
void restore(String projName) throws IOException;
|
||||
}
|
@@ -0,0 +1,63 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.job;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.job.SwapJob.CompressionMethod;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class SwapJobConfig {
|
||||
|
||||
private final int minProjects;
|
||||
private final int lowGiB;
|
||||
private final int highGiB;
|
||||
private final long intervalMillis;
|
||||
private final String compressionMethod;
|
||||
private final boolean allowUnsafeStores;
|
||||
|
||||
public SwapJobConfig(
|
||||
int minProjects,
|
||||
int lowGiB,
|
||||
int highGiB,
|
||||
long intervalMillis,
|
||||
String compressionMethod,
|
||||
boolean allowUnsafeStores) {
|
||||
this.minProjects = minProjects;
|
||||
this.lowGiB = lowGiB;
|
||||
this.highGiB = highGiB;
|
||||
this.intervalMillis = intervalMillis;
|
||||
this.compressionMethod = compressionMethod;
|
||||
this.allowUnsafeStores = allowUnsafeStores;
|
||||
}
|
||||
|
||||
public int getMinProjects() {
|
||||
return minProjects;
|
||||
}
|
||||
|
||||
public int getLowGiB() {
|
||||
return lowGiB;
|
||||
}
|
||||
|
||||
public int getHighGiB() {
|
||||
return highGiB;
|
||||
}
|
||||
|
||||
public long getIntervalMillis() {
|
||||
return intervalMillis;
|
||||
}
|
||||
|
||||
public boolean getAllowUnsafeStores() {
|
||||
return allowUnsafeStores;
|
||||
}
|
||||
|
||||
public SwapJob.CompressionMethod getCompressionMethod() {
|
||||
CompressionMethod result = SwapJob.stringToCompressionMethod(compressionMethod);
|
||||
if (result == null) {
|
||||
Log.info(
|
||||
"SwapJobConfig: un-supported compressionMethod '{}', default to 'bzip2'",
|
||||
compressionMethod);
|
||||
result = CompressionMethod.Bzip2;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
@@ -0,0 +1,245 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.job;
|
||||
|
||||
import com.google.api.client.repackaged.com.google.common.base.Preconditions;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Timer;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import uk.ac.ic.wlgitbridge.bridge.db.DBStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.LockGuard;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.bridge.swap.store.SwapStore;
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.TimerUtils;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public class SwapJobImpl implements SwapJob {
|
||||
|
||||
private static final long GiB = (1l << 30);
|
||||
|
||||
int minProjects;
|
||||
long lowWatermarkBytes;
|
||||
long highWatermarkBytes;
|
||||
Duration interval;
|
||||
|
||||
private final ProjectLock lock;
|
||||
private final RepoStore repoStore;
|
||||
private final DBStore dbStore;
|
||||
private final SwapStore swapStore;
|
||||
private final CompressionMethod compressionMethod;
|
||||
|
||||
private final Timer timer;
|
||||
|
||||
final AtomicInteger swaps;
|
||||
|
||||
public SwapJobImpl(
|
||||
SwapJobConfig cfg,
|
||||
ProjectLock lock,
|
||||
RepoStore repoStore,
|
||||
DBStore dbStore,
|
||||
SwapStore swapStore) {
|
||||
this(
|
||||
cfg.getMinProjects(),
|
||||
GiB * cfg.getLowGiB(),
|
||||
GiB * cfg.getHighGiB(),
|
||||
Duration.ofMillis(cfg.getIntervalMillis()),
|
||||
cfg.getCompressionMethod(),
|
||||
lock,
|
||||
repoStore,
|
||||
dbStore,
|
||||
swapStore);
|
||||
}
|
||||
|
||||
SwapJobImpl(
|
||||
int minProjects,
|
||||
long lowWatermarkBytes,
|
||||
long highWatermarkBytes,
|
||||
Duration interval,
|
||||
CompressionMethod method,
|
||||
ProjectLock lock,
|
||||
RepoStore repoStore,
|
||||
DBStore dbStore,
|
||||
SwapStore swapStore) {
|
||||
this.minProjects = minProjects;
|
||||
this.lowWatermarkBytes = lowWatermarkBytes;
|
||||
this.highWatermarkBytes = highWatermarkBytes;
|
||||
this.interval = interval;
|
||||
this.compressionMethod = method;
|
||||
this.lock = lock;
|
||||
this.repoStore = repoStore;
|
||||
this.dbStore = dbStore;
|
||||
this.swapStore = swapStore;
|
||||
timer = new Timer();
|
||||
swaps = new AtomicInteger(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
timer.schedule(TimerUtils.makeTimerTask(this::doSwap), 0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
private void doSwap() {
|
||||
try {
|
||||
doSwap_();
|
||||
} catch (Throwable t) {
|
||||
Log.warn("Exception thrown during swap job", t);
|
||||
}
|
||||
timer.schedule(TimerUtils.makeTimerTask(this::doSwap), interval.toMillis());
|
||||
}
|
||||
|
||||
private void doSwap_() {
|
||||
ArrayList<String> exceptionProjectNames = new ArrayList<String>();
|
||||
|
||||
Log.debug("Running swap number {}", swaps.get() + 1);
|
||||
long totalSize = repoStore.totalSize();
|
||||
Log.debug("Size is {}/{} (high)", totalSize, highWatermarkBytes);
|
||||
if (totalSize < highWatermarkBytes) {
|
||||
Log.debug("No need to swap.");
|
||||
swaps.incrementAndGet();
|
||||
return;
|
||||
}
|
||||
int numProjects = dbStore.getNumProjects();
|
||||
// while we have too many projects on disk
|
||||
while ((totalSize = repoStore.totalSize()) > lowWatermarkBytes
|
||||
&& (numProjects = dbStore.getNumUnswappedProjects()) > minProjects) {
|
||||
// check if we've had too many exceptions so far
|
||||
if (exceptionProjectNames.size() >= 20) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String s : exceptionProjectNames) {
|
||||
sb.append(s);
|
||||
sb.append(' ');
|
||||
}
|
||||
Log.error(
|
||||
"Too many exceptions while running swap, giving up on this run: {}", sb.toString());
|
||||
break;
|
||||
}
|
||||
// get the oldest project and try to swap it
|
||||
String projectName = dbStore.getOldestUnswappedProject();
|
||||
try {
|
||||
evict(projectName);
|
||||
} catch (Exception e) {
|
||||
Log.warn("[{}] Exception while swapping, mark project and move on", projectName, e);
|
||||
// NOTE: this is something of a hack. If a project fails to swap we get stuck in a
|
||||
// loop where `dbStore.getOldestUnswappedProject()` gives the same failing project over and
|
||||
// over again,
|
||||
// which fills up the disk with errors. By touching the access time we can mark the project
|
||||
// as a
|
||||
// non-candidate for swapping. Ideally we should be checking the logs for these log events
|
||||
// and fixing
|
||||
// whatever is wrong with the project
|
||||
dbStore.setLastAccessedTime(projectName, Timestamp.valueOf(LocalDateTime.now()));
|
||||
exceptionProjectNames.add(projectName);
|
||||
}
|
||||
}
|
||||
if (totalSize > lowWatermarkBytes) {
|
||||
Log.warn("Finished swapping, but total size is still too high.");
|
||||
}
|
||||
Log.debug(
|
||||
"Size: {}/{} (low), "
|
||||
+ "{} (high), "
|
||||
+ "projects on disk: {}/{}, "
|
||||
+ "min projects on disk: {}",
|
||||
totalSize,
|
||||
lowWatermarkBytes,
|
||||
highWatermarkBytes,
|
||||
numProjects,
|
||||
dbStore.getNumProjects(),
|
||||
minProjects);
|
||||
swaps.incrementAndGet();
|
||||
}
|
||||
|
||||
/*
|
||||
* @see SwapJob#evict(String) for high-level description.
|
||||
*
|
||||
* 1. Acquires the project lock.
|
||||
* 2. Gets a bz2 stream and size of a project from the repo store, or throws
|
||||
* 3. Uploads the bz2 stream and size to the projName in the swapStore.
|
||||
* 4. Sets the last accessed time in the dbStore to null, which makes our
|
||||
* state SWAPPED
|
||||
* 5. Removes the project from the repo store.
|
||||
* @param projName
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void evict(String projName) throws IOException {
|
||||
Preconditions.checkNotNull(projName, "projName was null");
|
||||
Log.info("Evicting project: {}", projName);
|
||||
try (LockGuard __ = lock.lockGuard(projName)) {
|
||||
try {
|
||||
repoStore.gcProject(projName);
|
||||
} catch (Exception e) {
|
||||
Log.error("[{}] Exception while running gc on project: {}", projName, e);
|
||||
}
|
||||
long[] sizePtr = new long[1];
|
||||
try (InputStream blob = getBlobStream(projName, sizePtr)) {
|
||||
swapStore.upload(projName, blob, sizePtr[0]);
|
||||
String compression = SwapJob.compressionMethodAsString(compressionMethod);
|
||||
if (compression == null) {
|
||||
throw new RuntimeException("invalid compression method, should not happen");
|
||||
}
|
||||
dbStore.swap(projName, compression);
|
||||
repoStore.remove(projName);
|
||||
}
|
||||
} catch (CannotAcquireLockException e) {
|
||||
Log.warn("[{}] Cannot acquire project lock, skipping swap", projName);
|
||||
return;
|
||||
}
|
||||
Log.info("Evicted project: {}", projName);
|
||||
}
|
||||
|
||||
private InputStream getBlobStream(String projName, long[] sizePtr) throws IOException {
|
||||
if (compressionMethod == CompressionMethod.Gzip) {
|
||||
return repoStore.gzipProject(projName, sizePtr);
|
||||
} else if (compressionMethod == CompressionMethod.Bzip2) {
|
||||
return repoStore.bzip2Project(projName, sizePtr);
|
||||
} else {
|
||||
throw new RuntimeException("invalid compression method, should not happen");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* @see SwapJob#restore(String) for high-level description.
|
||||
*
|
||||
* 1. Acquires the project lock.
|
||||
* 2. Gets a bz2 stream for the project from the swapStore.
|
||||
* 3. Fully downloads and places the bz2 stream back in the repo store.
|
||||
* 4. Sets the last accessed time in the dbStore to now, which makes our
|
||||
* state PRESENT and the last project to be evicted.
|
||||
* @param projName
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void restore(String projName) throws IOException {
|
||||
try (LockGuard __ = lock.lockGuard(projName)) {
|
||||
try (InputStream zipped = swapStore.openDownloadStream(projName)) {
|
||||
String compression = dbStore.getSwapCompression(projName);
|
||||
if (compression == null) {
|
||||
throw new RuntimeException(
|
||||
"Missing compression method during restore, should not happen");
|
||||
}
|
||||
if ("gzip".equals(compression)) {
|
||||
repoStore.ungzipProject(projName, zipped);
|
||||
} else if ("bzip2".equals(compression)) {
|
||||
repoStore.unbzip2Project(projName, zipped);
|
||||
}
|
||||
swapStore.remove(projName);
|
||||
dbStore.restore(projName);
|
||||
}
|
||||
} catch (CannotAcquireLockException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,49 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.store;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
/*
|
||||
* Created by winston on 23/08/2016.
|
||||
*/
|
||||
public class InMemorySwapStore implements SwapStore {
|
||||
|
||||
private final Map<String, byte[]> store;
|
||||
|
||||
public InMemorySwapStore() {
|
||||
store = new HashMap<>();
|
||||
}
|
||||
|
||||
public InMemorySwapStore(SwapStoreConfig __) {
|
||||
this();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(String projectName, InputStream uploadStream, long contentLength)
|
||||
throws IOException {
|
||||
store.put(projectName, IOUtils.toByteArray(uploadStream, contentLength));
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream openDownloadStream(String projectName) {
|
||||
byte[] buf = store.get(projectName);
|
||||
if (buf == null) {
|
||||
throw new IllegalArgumentException("no such project in swap store: " + projectName);
|
||||
}
|
||||
return new ByteArrayInputStream(buf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String projectName) {
|
||||
store.remove(projectName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSafe() {
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,28 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.store;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class NoopSwapStore implements SwapStore {
|
||||
|
||||
public NoopSwapStore(SwapStoreConfig __) {}
|
||||
|
||||
@Override
|
||||
public void upload(String projectName, InputStream uploadStream, long contentLength) {}
|
||||
|
||||
@Override
|
||||
public InputStream openDownloadStream(String projectName) {
|
||||
return new ByteArrayInputStream(new byte[0]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String projectName) {}
|
||||
|
||||
@Override
|
||||
public boolean isSafe() {
|
||||
return false;
|
||||
}
|
||||
}
|
@@ -0,0 +1,64 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.store;
|
||||
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.*;
|
||||
import java.io.InputStream;
|
||||
|
||||
/*
|
||||
* Created by winston on 21/08/2016.
|
||||
*/
|
||||
public class S3SwapStore implements SwapStore {
|
||||
|
||||
private final AmazonS3 s3;
|
||||
|
||||
private final String bucketName;
|
||||
|
||||
public S3SwapStore(SwapStoreConfig cfg) {
|
||||
this(cfg.getAwsAccessKey(), cfg.getAwsSecret(), cfg.getS3BucketName(), cfg.getAwsRegion());
|
||||
}
|
||||
|
||||
S3SwapStore(String accessKey, String secret, String bucketName, String region) {
|
||||
String regionToUse = null;
|
||||
if (region == null) {
|
||||
regionToUse = "us-east-1";
|
||||
} else {
|
||||
regionToUse = region;
|
||||
}
|
||||
s3 =
|
||||
AmazonS3ClientBuilder.standard()
|
||||
.withRegion(regionToUse)
|
||||
.withCredentials(
|
||||
new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secret)))
|
||||
.build();
|
||||
this.bucketName = bucketName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void upload(String projectName, InputStream uploadStream, long contentLength) {
|
||||
ObjectMetadata metadata = new ObjectMetadata();
|
||||
metadata.setContentLength(contentLength);
|
||||
PutObjectRequest put = new PutObjectRequest(bucketName, projectName, uploadStream, metadata);
|
||||
PutObjectResult res = s3.putObject(put);
|
||||
}
|
||||
|
||||
@Override
|
||||
public InputStream openDownloadStream(String projectName) {
|
||||
GetObjectRequest get = new GetObjectRequest(bucketName, projectName);
|
||||
S3Object res = s3.getObject(get);
|
||||
return res.getObjectContent();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String projectName) {
|
||||
DeleteObjectRequest del = new DeleteObjectRequest(bucketName, projectName);
|
||||
s3.deleteObject(del);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isSafe() {
|
||||
return true;
|
||||
}
|
||||
}
|
@@ -0,0 +1,43 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.store;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public interface SwapStore {
|
||||
|
||||
Map<String, Function<SwapStoreConfig, SwapStore>> swapStores =
|
||||
new HashMap<String, Function<SwapStoreConfig, SwapStore>>() {
|
||||
|
||||
{
|
||||
put("noop", NoopSwapStore::new);
|
||||
put("memory", InMemorySwapStore::new);
|
||||
put("s3", S3SwapStore::new);
|
||||
}
|
||||
};
|
||||
|
||||
static SwapStore fromConfig(Optional<SwapStoreConfig> cfg) {
|
||||
SwapStoreConfig cfg_ = cfg.orElse(SwapStoreConfig.NOOP);
|
||||
String type = cfg_.getType();
|
||||
return swapStores.get(type).apply(cfg_);
|
||||
}
|
||||
|
||||
void upload(String projectName, InputStream uploadStream, long contentLength) throws IOException;
|
||||
|
||||
InputStream openDownloadStream(String projectName);
|
||||
|
||||
void remove(String projectName);
|
||||
|
||||
/*
|
||||
* Returns true if the swap store safely persists swapped projects.
|
||||
*
|
||||
* Fake swap stores should return false.
|
||||
*/
|
||||
boolean isSafe();
|
||||
}
|
@@ -0,0 +1,64 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.swap.store;
|
||||
|
||||
/*
|
||||
* Created by winston on 24/08/2016.
|
||||
*/
|
||||
public class SwapStoreConfig {
|
||||
|
||||
public static final SwapStoreConfig NOOP = new SwapStoreConfig("noop", null, null, null, null);
|
||||
|
||||
private String type;
|
||||
private String awsAccessKey;
|
||||
private String awsSecret;
|
||||
private String s3BucketName;
|
||||
private String awsRegion;
|
||||
|
||||
public SwapStoreConfig() {}
|
||||
|
||||
public SwapStoreConfig(
|
||||
String awsAccessKey, String awsSecret, String s3BucketName, String awsRegion) {
|
||||
this("s3", awsAccessKey, awsSecret, s3BucketName, awsRegion);
|
||||
}
|
||||
|
||||
SwapStoreConfig(
|
||||
String type, String awsAccessKey, String awsSecret, String s3BucketName, String awsRegion) {
|
||||
this.type = type;
|
||||
this.awsAccessKey = awsAccessKey;
|
||||
this.awsSecret = awsSecret;
|
||||
this.s3BucketName = s3BucketName;
|
||||
this.awsRegion = awsRegion;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public String getAwsAccessKey() {
|
||||
return awsAccessKey;
|
||||
}
|
||||
|
||||
public String getAwsSecret() {
|
||||
return awsSecret;
|
||||
}
|
||||
|
||||
public String getS3BucketName() {
|
||||
return s3BucketName;
|
||||
}
|
||||
|
||||
public String getAwsRegion() {
|
||||
return awsRegion;
|
||||
}
|
||||
|
||||
public SwapStoreConfig sanitisedCopy() {
|
||||
return new SwapStoreConfig(
|
||||
type,
|
||||
awsAccessKey == null ? null : "<awsAccessKey>",
|
||||
awsSecret == null ? null : "<awsSecret>",
|
||||
s3BucketName,
|
||||
awsRegion);
|
||||
}
|
||||
|
||||
public static SwapStoreConfig sanitisedCopy(SwapStoreConfig swapStore) {
|
||||
return swapStore == null ? null : swapStore.sanitisedCopy();
|
||||
}
|
||||
}
|
@@ -0,0 +1,16 @@
|
||||
package uk.ac.ic.wlgitbridge.bridge.util;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
/*
|
||||
* Created by winston on 01/07/2017.
|
||||
*/
|
||||
public class CastUtil {
|
||||
|
||||
public static int assumeInt(long l) {
|
||||
Preconditions.checkArgument(
|
||||
l <= (long) Integer.MAX_VALUE && l >= (long) Integer.MIN_VALUE,
|
||||
l + " cannot fit inside an int");
|
||||
return (int) l;
|
||||
}
|
||||
}
|
@@ -0,0 +1,130 @@
|
||||
package uk.ac.ic.wlgitbridge.data;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Created by Winston on 16/11/14.
|
||||
*/
|
||||
public class CandidateSnapshot implements AutoCloseable {
|
||||
|
||||
private final String projectName;
|
||||
private final int currentVersion;
|
||||
private final List<ServletFile> files;
|
||||
private final List<String> deleted;
|
||||
private File attsDirectory;
|
||||
|
||||
public CandidateSnapshot(
|
||||
String projectName,
|
||||
int currentVersion,
|
||||
RawDirectory directoryContents,
|
||||
RawDirectory oldDirectoryContents) {
|
||||
this.projectName = projectName;
|
||||
this.currentVersion = currentVersion;
|
||||
files = diff(directoryContents, oldDirectoryContents);
|
||||
deleted = deleted(directoryContents, oldDirectoryContents);
|
||||
}
|
||||
|
||||
private List<ServletFile> diff(
|
||||
RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
|
||||
List<ServletFile> files = new LinkedList<ServletFile>();
|
||||
Map<String, RawFile> fileTable = directoryContents.getFileTable();
|
||||
Map<String, RawFile> oldFileTable = oldDirectoryContents.getFileTable();
|
||||
for (Entry<String, RawFile> entry : fileTable.entrySet()) {
|
||||
RawFile file = entry.getValue();
|
||||
files.add(new ServletFile(file, oldFileTable.get(file.getPath())));
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private List<String> deleted(RawDirectory directoryContents, RawDirectory oldDirectoryContents) {
|
||||
List<String> deleted = new LinkedList<String>();
|
||||
Map<String, RawFile> fileTable = directoryContents.getFileTable();
|
||||
for (Entry<String, RawFile> entry : oldDirectoryContents.getFileTable().entrySet()) {
|
||||
String path = entry.getKey();
|
||||
RawFile newFile = fileTable.get(path);
|
||||
if (newFile == null) {
|
||||
deleted.add(path);
|
||||
}
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
public void writeServletFiles(File rootGitDirectory) throws IOException {
|
||||
attsDirectory = new File(rootGitDirectory, ".wlgb/atts/" + projectName);
|
||||
for (ServletFile file : files) {
|
||||
if (file.isChanged()) {
|
||||
file.writeToDiskWithName(attsDirectory, file.getUniqueIdentifier());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteServletFiles() throws IOException {
|
||||
if (attsDirectory != null) {
|
||||
Util.deleteDirectory(attsDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
public JsonElement getJsonRepresentation(String postbackKey) {
|
||||
String projectURL = Util.getPostbackURL() + "api/" + projectName;
|
||||
JsonObject jsonObject = new JsonObject();
|
||||
jsonObject.addProperty("latestVerId", currentVersion);
|
||||
jsonObject.add("files", getFilesAsJson(projectURL, postbackKey));
|
||||
jsonObject.addProperty("postbackUrl", projectURL + "/" + postbackKey + "/postback");
|
||||
return jsonObject;
|
||||
}
|
||||
|
||||
private JsonArray getFilesAsJson(String projectURL, String postbackKey) {
|
||||
JsonArray filesArray = new JsonArray();
|
||||
for (ServletFile file : files) {
|
||||
filesArray.add(getFileAsJson(file, projectURL, postbackKey));
|
||||
}
|
||||
return filesArray;
|
||||
}
|
||||
|
||||
private JsonObject getFileAsJson(ServletFile file, String projectURL, String postbackKey) {
|
||||
JsonObject jsonFile = new JsonObject();
|
||||
jsonFile.addProperty("name", file.getPath());
|
||||
if (file.isChanged()) {
|
||||
String identifier = file.getUniqueIdentifier();
|
||||
String url = projectURL + "/" + identifier + "?key=" + postbackKey;
|
||||
jsonFile.addProperty("url", url);
|
||||
}
|
||||
return jsonFile;
|
||||
}
|
||||
|
||||
public String getProjectName() {
|
||||
return projectName;
|
||||
}
|
||||
|
||||
public List<String> getDeleted() {
|
||||
return deleted;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("VersionId: ");
|
||||
sb.append(currentVersion);
|
||||
sb.append(", files: ");
|
||||
sb.append(files);
|
||||
sb.append(", deleted: ");
|
||||
sb.append(deleted);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
deleteServletFiles();
|
||||
}
|
||||
}
|
@@ -0,0 +1,9 @@
|
||||
package uk.ac.ic.wlgitbridge.data;
|
||||
|
||||
public class CannotAcquireLockException extends Exception {
|
||||
String projectName;
|
||||
|
||||
public CannotAcquireLockException() {
|
||||
super("Another operation is in progress. Please try again later.");
|
||||
}
|
||||
}
|
@@ -0,0 +1,9 @@
|
||||
package uk.ac.ic.wlgitbridge.data;
|
||||
|
||||
/*
|
||||
* Created by Winston on 21/02/15.
|
||||
*/
|
||||
public interface LockAllWaiter {
|
||||
|
||||
void threadsRemaining(int threads);
|
||||
}
|
@@ -0,0 +1,96 @@
|
||||
package uk.ac.ic.wlgitbridge.data;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
import uk.ac.ic.wlgitbridge.bridge.lock.ProjectLock;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 20/11/14.
|
||||
*/
|
||||
public class ProjectLockImpl implements ProjectLock {
|
||||
|
||||
private final Map<String, Lock> projectLocks;
|
||||
private final ReentrantReadWriteLock rwlock;
|
||||
private final Lock rlock;
|
||||
private final ReentrantReadWriteLock.WriteLock wlock;
|
||||
private LockAllWaiter waiter;
|
||||
private boolean waiting;
|
||||
|
||||
public ProjectLockImpl() {
|
||||
projectLocks = new HashMap<String, Lock>();
|
||||
rwlock = new ReentrantReadWriteLock();
|
||||
rlock = rwlock.readLock();
|
||||
wlock = rwlock.writeLock();
|
||||
waiting = false;
|
||||
}
|
||||
|
||||
public ProjectLockImpl(LockAllWaiter waiter) {
|
||||
this();
|
||||
setWaiter(waiter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void lockForProject(String projectName) throws CannotAcquireLockException {
|
||||
Log.debug("[{}] taking project lock", projectName);
|
||||
Lock projectLock = getLockForProjectName(projectName);
|
||||
try {
|
||||
if (!projectLock.tryLock(5, TimeUnit.SECONDS)) {
|
||||
Log.debug("[{}] failed to acquire project lock", projectName);
|
||||
throw new CannotAcquireLockException();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
Log.debug("[{}] taking reentrant lock", projectName);
|
||||
rlock.lock();
|
||||
Log.debug("[{}] taken locks", projectName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unlockForProject(String projectName) {
|
||||
Log.debug("[{}] releasing project lock", projectName);
|
||||
getLockForProjectName(projectName).unlock();
|
||||
Log.debug("[{}] releasing reentrant lock", projectName);
|
||||
rlock.unlock();
|
||||
Log.debug("[{}] released locks", projectName);
|
||||
if (waiting) {
|
||||
Log.debug("[{}] waiting for remaining threads", projectName);
|
||||
trySignal();
|
||||
}
|
||||
}
|
||||
|
||||
private void trySignal() {
|
||||
int threads = rwlock.getReadLockCount();
|
||||
Log.debug("-> waiting for {} threads", threads);
|
||||
if (waiter != null && threads > 0) {
|
||||
waiter.threadsRemaining(threads);
|
||||
}
|
||||
Log.debug("-> finished waiting for threads");
|
||||
}
|
||||
|
||||
public void lockAll() {
|
||||
Log.debug("-> locking all threads");
|
||||
waiting = true;
|
||||
trySignal();
|
||||
Log.debug("-> locking reentrant write lock");
|
||||
wlock.lock();
|
||||
}
|
||||
|
||||
private synchronized Lock getLockForProjectName(String projectName) {
|
||||
Lock lock = projectLocks.get(projectName);
|
||||
if (lock == null) {
|
||||
lock = new ReentrantLock();
|
||||
projectLocks.put(projectName, lock);
|
||||
}
|
||||
return lock;
|
||||
}
|
||||
|
||||
public void setWaiter(LockAllWaiter waiter) {
|
||||
this.waiter = waiter;
|
||||
}
|
||||
}
|
@@ -0,0 +1,48 @@
|
||||
package uk.ac.ic.wlgitbridge.data;
|
||||
|
||||
import java.util.UUID;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawFile;
|
||||
|
||||
/*
|
||||
* Created by Winston on 21/02/15.
|
||||
*/
|
||||
public class ServletFile extends RawFile {
|
||||
|
||||
private final RawFile file;
|
||||
private final boolean changed;
|
||||
private String uuid;
|
||||
|
||||
public ServletFile(RawFile file, RawFile oldFile) {
|
||||
this.file = file;
|
||||
this.uuid = UUID.randomUUID().toString();
|
||||
changed = !equals(oldFile);
|
||||
}
|
||||
|
||||
public String getUniqueIdentifier() {
|
||||
return uuid;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPath() {
|
||||
return file.getPath();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getContents() {
|
||||
return file.getContents();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() {
|
||||
return getContents().length;
|
||||
}
|
||||
|
||||
public boolean isChanged() {
|
||||
return changed;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getPath();
|
||||
}
|
||||
}
|
@@ -0,0 +1,76 @@
|
||||
package uk.ac.ic.wlgitbridge.data.filestore;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import uk.ac.ic.wlgitbridge.data.model.Snapshot;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Created by Winston on 14/11/14.
|
||||
*/
|
||||
public class GitDirectoryContents {
|
||||
|
||||
private final List<RawFile> files;
|
||||
private final File gitDirectory;
|
||||
private final String userName;
|
||||
private final String userEmail;
|
||||
private final String commitMessage;
|
||||
private final Date when;
|
||||
|
||||
public GitDirectoryContents(
|
||||
List<RawFile> files,
|
||||
File rootGitDirectory,
|
||||
String projectName,
|
||||
String userName,
|
||||
String userEmail,
|
||||
String commitMessage,
|
||||
Date when) {
|
||||
this.files = files;
|
||||
this.gitDirectory = new File(rootGitDirectory, projectName);
|
||||
this.userName = userName;
|
||||
this.userEmail = userEmail;
|
||||
this.commitMessage = commitMessage;
|
||||
this.when = when;
|
||||
}
|
||||
|
||||
public GitDirectoryContents(
|
||||
List<RawFile> files, File rootGitDirectory, String projectName, Snapshot snapshot) {
|
||||
this(
|
||||
files,
|
||||
rootGitDirectory,
|
||||
projectName,
|
||||
snapshot.getUserName(),
|
||||
snapshot.getUserEmail(),
|
||||
snapshot.getComment(),
|
||||
snapshot.getCreatedAt());
|
||||
}
|
||||
|
||||
public void write() throws IOException {
|
||||
Util.deleteInDirectoryApartFrom(gitDirectory, ".git");
|
||||
for (RawFile fileNode : files) {
|
||||
fileNode.writeToDisk(gitDirectory);
|
||||
}
|
||||
}
|
||||
|
||||
public File getDirectory() {
|
||||
return gitDirectory;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return userName;
|
||||
}
|
||||
|
||||
public String getUserEmail() {
|
||||
return userEmail;
|
||||
}
|
||||
|
||||
public String getCommitMessage() {
|
||||
return commitMessage;
|
||||
}
|
||||
|
||||
public Date getWhen() {
|
||||
return when;
|
||||
}
|
||||
}
|
@@ -0,0 +1,19 @@
|
||||
package uk.ac.ic.wlgitbridge.data.filestore;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/*
|
||||
* Created by Winston on 16/11/14.
|
||||
*/
|
||||
public class RawDirectory {
|
||||
|
||||
private final Map<String, RawFile> fileTable;
|
||||
|
||||
public RawDirectory(Map<String, RawFile> fileTable) {
|
||||
this.fileTable = fileTable;
|
||||
}
|
||||
|
||||
public Map<String, RawFile> getFileTable() {
|
||||
return fileTable;
|
||||
}
|
||||
}
|
@@ -0,0 +1,43 @@
|
||||
package uk.ac.ic.wlgitbridge.data.filestore;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Arrays;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 16/11/14.
|
||||
*/
|
||||
public abstract class RawFile {
|
||||
|
||||
public abstract String getPath();
|
||||
|
||||
public abstract byte[] getContents();
|
||||
|
||||
public abstract long size();
|
||||
|
||||
public final void writeToDisk(File directory) throws IOException {
|
||||
writeToDiskWithName(directory, getPath());
|
||||
}
|
||||
|
||||
public final void writeToDiskWithName(File directory, String name) throws IOException {
|
||||
File file = new File(directory, name);
|
||||
file.getParentFile().mkdirs();
|
||||
file.createNewFile();
|
||||
OutputStream out = new FileOutputStream(file);
|
||||
out.write(getContents());
|
||||
out.close();
|
||||
Log.debug("Wrote file: {}", file.getAbsolutePath());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (!(obj instanceof RawFile)) {
|
||||
return false;
|
||||
}
|
||||
RawFile that = (RawFile) obj;
|
||||
return getPath().equals(that.getPath()) && Arrays.equals(getContents(), that.getContents());
|
||||
}
|
||||
}
|
@@ -0,0 +1,30 @@
|
||||
package uk.ac.ic.wlgitbridge.data.filestore;
|
||||
|
||||
/*
|
||||
* Created by Winston on 16/11/14.
|
||||
*/
|
||||
public class RepositoryFile extends RawFile {
|
||||
|
||||
private final String path;
|
||||
private final byte[] contents;
|
||||
|
||||
public RepositoryFile(String path, byte[] contents) {
|
||||
this.path = path;
|
||||
this.contents = contents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getContents() {
|
||||
return contents;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long size() {
|
||||
return contents.length;
|
||||
}
|
||||
}
|
@@ -0,0 +1,75 @@
|
||||
package uk.ac.ic.wlgitbridge.data.model;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotAttachment;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotData;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getforversion.SnapshotFile;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.SnapshotInfo;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.getsavedvers.WLUser;
|
||||
|
||||
/*
|
||||
* Created by Winston on 03/11/14.
|
||||
*/
|
||||
public class Snapshot implements Comparable<Snapshot> {
|
||||
|
||||
private final int versionID;
|
||||
private final String comment;
|
||||
private final String userName;
|
||||
private final String userEmail;
|
||||
private final Date createdAt;
|
||||
|
||||
private final List<SnapshotFile> srcs;
|
||||
private final List<SnapshotAttachment> atts;
|
||||
|
||||
public Snapshot(SnapshotInfo info, SnapshotData data) {
|
||||
versionID = info.getVersionId();
|
||||
comment = info.getComment();
|
||||
WLUser user = info.getUser();
|
||||
userName = user.getName();
|
||||
userEmail = user.getEmail();
|
||||
createdAt = new DateTime(info.getCreatedAt()).toDate();
|
||||
|
||||
srcs = data.getSrcs();
|
||||
atts = data.getAtts();
|
||||
}
|
||||
|
||||
public int getVersionID() {
|
||||
return versionID;
|
||||
}
|
||||
|
||||
public String getComment() {
|
||||
return comment;
|
||||
}
|
||||
|
||||
public String getUserName() {
|
||||
return userName;
|
||||
}
|
||||
|
||||
public String getUserEmail() {
|
||||
return userEmail;
|
||||
}
|
||||
|
||||
public List<SnapshotFile> getSrcs() {
|
||||
return srcs;
|
||||
}
|
||||
|
||||
public List<SnapshotAttachment> getAtts() {
|
||||
return atts;
|
||||
}
|
||||
|
||||
public Date getCreatedAt() {
|
||||
return createdAt;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(Snapshot snapshot) {
|
||||
return Integer.compare(versionID, snapshot.versionID);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.valueOf(versionID);
|
||||
}
|
||||
}
|
@@ -0,0 +1,31 @@
|
||||
package uk.ac.ic.wlgitbridge.git.exception;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class FileLimitExceededException extends GitUserException {
|
||||
|
||||
private final long numFiles;
|
||||
|
||||
private final long maxFiles;
|
||||
|
||||
public FileLimitExceededException(long numFiles, long maxFiles) {
|
||||
this.numFiles = numFiles;
|
||||
this.maxFiles = maxFiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return "too many files";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDescriptionLines() {
|
||||
return Arrays.asList(
|
||||
"repository contains "
|
||||
+ numFiles
|
||||
+ " files, which exceeds the limit of "
|
||||
+ maxFiles
|
||||
+ " files");
|
||||
}
|
||||
}
|
@@ -0,0 +1,13 @@
|
||||
package uk.ac.ic.wlgitbridge.git.exception;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public abstract class GitUserException extends Exception {
|
||||
|
||||
public abstract String getMessage();
|
||||
|
||||
public abstract List<String> getDescriptionLines();
|
||||
}
|
@@ -0,0 +1,20 @@
|
||||
package uk.ac.ic.wlgitbridge.git.exception;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class InvalidGitRepository extends GitUserException {
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return "invalid git repo";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDescriptionLines() {
|
||||
return Arrays.asList(
|
||||
"Your Git repository contains a reference we cannot resolve.",
|
||||
"If your project contains a Git submodule,",
|
||||
"please remove it and try again.");
|
||||
}
|
||||
}
|
@@ -0,0 +1,34 @@
|
||||
package uk.ac.ic.wlgitbridge.git.exception;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
public class SizeLimitExceededException extends GitUserException {
|
||||
|
||||
private final Optional<String> path;
|
||||
|
||||
private final long actualSize;
|
||||
|
||||
private final long maxSize;
|
||||
|
||||
public SizeLimitExceededException(Optional<String> path, long actualSize, long maxSize) {
|
||||
this.path = path;
|
||||
this.actualSize = actualSize;
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return "file too big";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDescriptionLines() {
|
||||
String filename = path.isPresent() ? "File '" + path.get() + "' is" : "There's a file";
|
||||
return Arrays.asList(
|
||||
filename + " too large to push to " + Util.getServiceName() + " via git",
|
||||
"the recommended maximum file size is 50 MiB");
|
||||
}
|
||||
}
|
@@ -0,0 +1,8 @@
|
||||
package uk.ac.ic.wlgitbridge.git.exception;
|
||||
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.JSONSource;
|
||||
|
||||
/*
|
||||
* Created by winston on 20/08/2016.
|
||||
*/
|
||||
public abstract class SnapshotAPIException extends GitUserException implements JSONSource {}
|
@@ -0,0 +1,68 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.util.Optional;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.transport.ReceivePack;
|
||||
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
|
||||
import uk.ac.ic.wlgitbridge.bridge.Bridge;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.git.handler.hook.WriteLatexPutHook;
|
||||
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Created by Winston on 02/11/14.
|
||||
*/
|
||||
/*
|
||||
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
|
||||
* user Git requests.
|
||||
*
|
||||
* This class just puts a {@link WriteLatexPutHook} into the {@link ReceivePack}
|
||||
* that it returns.
|
||||
*/
|
||||
public class WLReceivePackFactory implements ReceivePackFactory<HttpServletRequest> {
|
||||
|
||||
private final RepoStore repoStore;
|
||||
|
||||
private final Bridge bridge;
|
||||
|
||||
public WLReceivePackFactory(RepoStore repoStore, Bridge bridge) {
|
||||
this.repoStore = repoStore;
|
||||
this.bridge = bridge;
|
||||
}
|
||||
|
||||
/*
|
||||
* Puts a {@link WriteLatexPutHook} into the returned {@link ReceivePack}.
|
||||
*
|
||||
* The {@link WriteLatexPutHook} needs our hostname, which we get from the
|
||||
* original {@link HttpServletRequest}, used to provide a postback URL to
|
||||
* the {@link SnapshotApi}. We also give it the oauth2 that we injected in
|
||||
* the {@link Oauth2Filter}, and the {@link Bridge}.
|
||||
*
|
||||
* At this point, the repository will have been synced to the latest on
|
||||
* Overleaf, but it's possible that an update happens on Overleaf while our
|
||||
* put hook is running. In this case, we fail, and the user tries again,
|
||||
* triggering another sync, and so on.
|
||||
* @param httpServletRequest the original request
|
||||
* @param repository the JGit {@link Repository} provided by
|
||||
* {@link WLRepositoryResolver}
|
||||
* @return a correctly hooked {@link ReceivePack}
|
||||
*/
|
||||
@Override
|
||||
public ReceivePack create(HttpServletRequest httpServletRequest, Repository repository) {
|
||||
Log.debug("[{}] Creating receive-pack", repository.getWorkTree().getName());
|
||||
Optional<Credential> oauth2 =
|
||||
Optional.ofNullable(
|
||||
(Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY));
|
||||
ReceivePack receivePack = new ReceivePack(repository);
|
||||
String hostname = Util.getPostbackURL();
|
||||
if (hostname == null) {
|
||||
hostname = httpServletRequest.getLocalName();
|
||||
}
|
||||
receivePack.setPreReceiveHook(new WriteLatexPutHook(repoStore, bridge, hostname, oauth2));
|
||||
return receivePack;
|
||||
}
|
||||
}
|
@@ -0,0 +1,112 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.eclipse.jgit.errors.RepositoryNotFoundException;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
|
||||
import org.eclipse.jgit.transport.resolver.RepositoryResolver;
|
||||
import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
|
||||
import uk.ac.ic.wlgitbridge.bridge.Bridge;
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.server.Oauth2Filter;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.base.ForbiddenException;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Created by Winston on 02/11/14.
|
||||
*/
|
||||
/*
|
||||
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
|
||||
* user Git requests.
|
||||
*
|
||||
* This class is used by all Git requests to resolve a project name to a
|
||||
* JGit {@link Repository}, or fail by throwing an exception.
|
||||
*
|
||||
* It has a single method, {@link #open(HttpServletRequest, String)}, which
|
||||
* calls into the {@link Bridge} to synchronise the project with Overleaf, i.e.
|
||||
* bringing it onto disk and applying commits to it until it is up-to-date with
|
||||
* Overleaf.
|
||||
*/
|
||||
public class WLRepositoryResolver implements RepositoryResolver<HttpServletRequest> {
|
||||
|
||||
private final Bridge bridge;
|
||||
|
||||
public WLRepositoryResolver(Bridge bridge) {
|
||||
this.bridge = bridge;
|
||||
}
|
||||
|
||||
/*
|
||||
* Calls into the Bridge to resolve a project name to a JGit
|
||||
* {@link Repository}, or throw an exception.
|
||||
*
|
||||
* On success, the repository will have been brought onto disk and updated
|
||||
* to the latest (synced).
|
||||
*
|
||||
* In the case of clones and fetches, upload packs are created from the
|
||||
* returned JGit {@link Repository} by the {@link WLUploadPackFactory}.
|
||||
*
|
||||
* The project lock is acquired for this process so it can't be swapped out.
|
||||
*
|
||||
* However, it can still be swapped out between this and a Git push. The
|
||||
* push would fail due to the project changed on Overleaf between the sync
|
||||
* and the actual push to Overleaf (performed by the
|
||||
* {@link WLReceivePackFactory} and {@link WriteLatexPutHook}. In this case,
|
||||
* the user will have to try again (which prompts another update, etc. until
|
||||
* this no longer happens).
|
||||
* @param httpServletRequest The HttpServletRequest as required by the
|
||||
* interface. We injected the oauth2 creds into it with
|
||||
* {@link Oauth2Filter}, which was set up by the {@link GitBridgeServer}.
|
||||
* @param name The name of the project
|
||||
* @return the JGit {@link Repository}.
|
||||
* @throws RepositoryNotFoundException If the project does not exist
|
||||
* @throws ServiceNotAuthorizedException If the user did not auth when
|
||||
* required to
|
||||
* @throws ServiceMayNotContinueException If any other general user
|
||||
* exception occurs that must be propogated back to the user, e.g.
|
||||
* internal errors (IOException, etc), too large file, and so on.
|
||||
*/
|
||||
@Override
|
||||
public Repository open(HttpServletRequest httpServletRequest, String name)
|
||||
throws RepositoryNotFoundException,
|
||||
ServiceNotAuthorizedException,
|
||||
ServiceMayNotContinueException {
|
||||
Log.debug("[{}] Request to open git repo", name);
|
||||
Optional<Credential> oauth2 =
|
||||
Optional.ofNullable(
|
||||
(Credential) httpServletRequest.getAttribute(Oauth2Filter.ATTRIBUTE_KEY));
|
||||
String projName = Util.removeAllSuffixes(name, "/", ".git");
|
||||
try {
|
||||
return bridge.getUpdatedRepo(oauth2, projName).getJGitRepository();
|
||||
} catch (RepositoryNotFoundException e) {
|
||||
Log.warn("Repository not found: " + name);
|
||||
throw e;
|
||||
/*
|
||||
} catch (ServiceNotAuthorizedException e) {
|
||||
cannot occur
|
||||
} catch (ServiceNotEnabledException e) {
|
||||
cannot occur
|
||||
*/
|
||||
} catch (ServiceMayNotContinueException e) {
|
||||
/* Such as FailedConnectionException */
|
||||
throw e;
|
||||
} catch (CannotAcquireLockException e) {
|
||||
throw new ServiceMayNotContinueException(e.getMessage());
|
||||
} catch (RuntimeException e) {
|
||||
Log.warn("Runtime exception when trying to open repo: " + projName, e);
|
||||
throw new ServiceMayNotContinueException(e);
|
||||
} catch (ForbiddenException e) {
|
||||
throw new ServiceNotAuthorizedException();
|
||||
} catch (GitUserException e) {
|
||||
throw new ServiceMayNotContinueException(
|
||||
e.getMessage() + "\n" + String.join("\n", e.getDescriptionLines()), e);
|
||||
} catch (IOException e) {
|
||||
Log.warn("IOException when trying to open repo: " + projName, e);
|
||||
throw new ServiceMayNotContinueException("Internal server error.");
|
||||
}
|
||||
}
|
||||
}
|
@@ -0,0 +1,34 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.transport.UploadPack;
|
||||
import org.eclipse.jgit.transport.resolver.UploadPackFactory;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 02/11/14.
|
||||
*/
|
||||
/*
|
||||
* One of the "big three" interfaces created by {@link WLGitServlet} to handle
|
||||
* user Git requests.
|
||||
*
|
||||
* The actual class doesn't do much, and most of the work is done when the
|
||||
* project name is being resolved by the {@link WLRepositoryResolver}.
|
||||
*/
|
||||
public class WLUploadPackFactory implements UploadPackFactory<HttpServletRequest> {
|
||||
|
||||
/*
|
||||
* This does nothing special. Synchronising the project with Overleaf will
|
||||
* have been performed by {@link WLRepositoryResolver}.
|
||||
* @param __ Not used, required by the {@link UploadPackFactory} interface
|
||||
* @param repository The JGit repository provided by the
|
||||
* {@link WLRepositoryResolver}
|
||||
* @return the {@link UploadPack}, used by JGit to serve the request
|
||||
*/
|
||||
@Override
|
||||
public UploadPack create(HttpServletRequest __, Repository repository) {
|
||||
Log.debug("[{}] Creating upload-pack", repository.getWorkTree().getName());
|
||||
return new UploadPack(repository);
|
||||
}
|
||||
}
|
@@ -0,0 +1,143 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler.hook;
|
||||
|
||||
import com.google.api.client.auth.oauth2.Credential;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.Optional;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.transport.PreReceiveHook;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand.Result;
|
||||
import org.eclipse.jgit.transport.ReceivePack;
|
||||
import uk.ac.ic.wlgitbridge.bridge.Bridge;
|
||||
import uk.ac.ic.wlgitbridge.bridge.repo.RepoStore;
|
||||
import uk.ac.ic.wlgitbridge.data.CannotAcquireLockException;
|
||||
import uk.ac.ic.wlgitbridge.data.filestore.RawDirectory;
|
||||
import uk.ac.ic.wlgitbridge.git.exception.GitUserException;
|
||||
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.ForcedPushException;
|
||||
import uk.ac.ic.wlgitbridge.git.handler.hook.exception.WrongBranchException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.InternalErrorException;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.OutOfDateException;
|
||||
import uk.ac.ic.wlgitbridge.util.Log;
|
||||
|
||||
/*
|
||||
* Created by Winston on 03/11/14.
|
||||
*/
|
||||
/*
|
||||
* Created by {@link WLReceivePackFactory} to update the {@link Bridge} for a
|
||||
* user's Git push request, or fail with an error. The hook is able to approve
|
||||
* or reject a request.
|
||||
*/
|
||||
public class WriteLatexPutHook implements PreReceiveHook {
|
||||
|
||||
private final RepoStore repoStore;
|
||||
|
||||
private final Bridge bridge;
|
||||
private final String hostname;
|
||||
private final Optional<Credential> oauth2;
|
||||
|
||||
/*
|
||||
* The constructor to use, which provides the hook with the {@link Bridge},
|
||||
* the hostname (used to construct a URL to give to Overleaf to postback),
|
||||
* and the oauth2 (used to authenticate with the Snapshot API).
|
||||
* @param repoStore
|
||||
* @param bridge the {@link Bridge}
|
||||
* @param hostname the hostname used for postback from the Snapshot API
|
||||
* @param oauth2 used to authenticate with the snapshot API, or null
|
||||
*/
|
||||
public WriteLatexPutHook(
|
||||
RepoStore repoStore, Bridge bridge, String hostname, Optional<Credential> oauth2) {
|
||||
this.repoStore = repoStore;
|
||||
this.bridge = bridge;
|
||||
this.hostname = hostname;
|
||||
this.oauth2 = oauth2;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onPreReceive(ReceivePack receivePack, Collection<ReceiveCommand> receiveCommands) {
|
||||
Log.debug(
|
||||
"-> Handling {} commands in {}",
|
||||
receiveCommands.size(),
|
||||
receivePack.getRepository().getDirectory().getAbsolutePath());
|
||||
for (ReceiveCommand receiveCommand : receiveCommands) {
|
||||
try {
|
||||
handleReceiveCommand(oauth2, receivePack.getRepository(), receiveCommand);
|
||||
} catch (IOException e) {
|
||||
Log.error("IOException on pre receive", e);
|
||||
receivePack.sendError(e.getMessage());
|
||||
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, e.getMessage());
|
||||
} catch (OutOfDateException e) {
|
||||
Log.error("OutOfDateException on pre receive", e);
|
||||
receiveCommand.setResult(Result.REJECTED_NONFASTFORWARD);
|
||||
} catch (GitUserException e) {
|
||||
Log.error("GitUserException on pre receive", e);
|
||||
handleSnapshotPostException(receivePack, receiveCommand, e);
|
||||
} catch (CannotAcquireLockException e) {
|
||||
Log.info("CannotAcquireLockException on pre receive");
|
||||
receivePack.sendError(e.getMessage());
|
||||
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, e.getMessage());
|
||||
} catch (Throwable t) {
|
||||
Log.error("Throwable on pre receive", t);
|
||||
handleSnapshotPostException(receivePack, receiveCommand, new InternalErrorException());
|
||||
}
|
||||
}
|
||||
Log.debug(
|
||||
"-> Handled {} commands in {}",
|
||||
receiveCommands.size(),
|
||||
receivePack.getRepository().getDirectory().getAbsolutePath());
|
||||
}
|
||||
|
||||
private void handleSnapshotPostException(
|
||||
ReceivePack receivePack, ReceiveCommand receiveCommand, GitUserException e) {
|
||||
String message = e.getMessage();
|
||||
receivePack.sendError(message);
|
||||
StringBuilder msg = new StringBuilder();
|
||||
for (Iterator<String> it = e.getDescriptionLines().iterator(); it.hasNext(); ) {
|
||||
String line = it.next();
|
||||
msg.append("hint: ");
|
||||
msg.append(line);
|
||||
if (it.hasNext()) {
|
||||
msg.append('\n');
|
||||
}
|
||||
}
|
||||
receivePack.sendMessage("");
|
||||
receivePack.sendMessage(msg.toString());
|
||||
receiveCommand.setResult(Result.REJECTED_OTHER_REASON, message);
|
||||
}
|
||||
|
||||
private void handleReceiveCommand(
|
||||
Optional<Credential> oauth2, Repository repository, ReceiveCommand receiveCommand)
|
||||
throws IOException, GitUserException, CannotAcquireLockException {
|
||||
checkBranch(receiveCommand);
|
||||
checkForcedPush(receiveCommand);
|
||||
bridge.push(
|
||||
oauth2,
|
||||
repository.getWorkTree().getName(),
|
||||
getPushedDirectoryContents(repository, receiveCommand),
|
||||
getOldDirectoryContents(repository),
|
||||
hostname);
|
||||
}
|
||||
|
||||
private void checkBranch(ReceiveCommand receiveCommand) throws WrongBranchException {
|
||||
if (!receiveCommand.getRefName().equals("refs/heads/master")) {
|
||||
throw new WrongBranchException();
|
||||
}
|
||||
}
|
||||
|
||||
private void checkForcedPush(ReceiveCommand receiveCommand) throws ForcedPushException {
|
||||
if (receiveCommand.getType() == ReceiveCommand.Type.UPDATE_NONFASTFORWARD) {
|
||||
throw new ForcedPushException();
|
||||
}
|
||||
}
|
||||
|
||||
private RawDirectory getPushedDirectoryContents(
|
||||
Repository repository, ReceiveCommand receiveCommand) throws IOException, GitUserException {
|
||||
return repoStore.useJGitRepo(repository, receiveCommand.getNewId()).getDirectory();
|
||||
}
|
||||
|
||||
private RawDirectory getOldDirectoryContents(Repository repository)
|
||||
throws IOException, GitUserException {
|
||||
return repoStore.useJGitRepo(repository, repository.resolve("HEAD")).getDirectory();
|
||||
}
|
||||
}
|
@@ -0,0 +1,33 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
|
||||
|
||||
import com.google.gson.JsonElement;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
|
||||
import uk.ac.ic.wlgitbridge.util.Util;
|
||||
|
||||
/*
|
||||
* Created by Winston on 16/11/14.
|
||||
*/
|
||||
public class ForcedPushException extends SnapshotPostException {
|
||||
|
||||
private static final String[] DESCRIPTION_LINES = {
|
||||
"You can't git push --force to a " + Util.getServiceName() + " project.",
|
||||
"Try to put your changes on top of the current head.",
|
||||
"If everything else fails, delete and reclone your repository, "
|
||||
+ "make your changes, then push again."
|
||||
};
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return "forced push prohibited";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDescriptionLines() {
|
||||
return Arrays.asList(DESCRIPTION_LINES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fromJSON(JsonElement json) {}
|
||||
}
|
@@ -0,0 +1,29 @@
|
||||
package uk.ac.ic.wlgitbridge.git.handler.hook.exception;
|
||||
|
||||
import com.google.gson.JsonElement;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import uk.ac.ic.wlgitbridge.snapshot.push.exception.SnapshotPostException;
|
||||
|
||||
/*
|
||||
* Created by Winston on 19/12/14.
|
||||
*/
|
||||
public class WrongBranchException extends SnapshotPostException {
|
||||
|
||||
private static final String[] DESCRIPTION_LINES = {
|
||||
"You can't push any new branches.", "Please use the master branch."
|
||||
};
|
||||
|
||||
@Override
|
||||
public String getMessage() {
|
||||
return "wrong branch";
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getDescriptionLines() {
|
||||
return Arrays.asList(DESCRIPTION_LINES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fromJSON(JsonElement json) {}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user