diff --git a/.github/workflows/_build-backend.yml b/.github/workflows/_build-backend.yml new file mode 100644 index 00000000..b9b14ca8 --- /dev/null +++ b/.github/workflows/_build-backend.yml @@ -0,0 +1,97 @@ +name: Build and deploy backend + +on: + workflow_call: + inputs: + environment: + required: true + type: string # dev, stg, prd + ref: + required: true + type: string # development, staging, main + version_tag: + required: true + type: string # e.g. "development" or "v2.1.3" + +jobs: + build-backend-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }} + PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Setup env + run: | + echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV + echo "BRANCH_NAME=${{ inputs.version_tag }}" >> $GITHUB_ENV + echo "VERSION=${{ inputs.version_tag }}" >> $GITHUB_ENV + - name: Log in to the container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push docker image + run: | + make all + - id: output + name: Output image tag + run: | + export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION} + echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT + + deploy-backend: + needs: build-backend-and-push + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Clone deploy repo + run: | + git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy + - name: Generate kubernetes manifests + env: + OPENAI_BASE_URL: ${{ inputs.environment == 'dev' && secrets.OPENAI_BASE_URL_DEV || inputs.environment == 'stg' && secrets.OPENAI_BASE_URL_STG || secrets.OPENAI_BASE_URL_PRD }} + OPENAI_API_KEY: ${{ inputs.environment == 'dev' && secrets.OPENAI_API_KEY_DEV || inputs.environment == 'stg' && secrets.OPENAI_API_KEY_STG || secrets.OPENAI_API_KEY_PRD }} + INFERENCE_BASE_URL: ${{ inputs.environment == 'dev' && secrets.INFERENCE_BASE_URL_DEV || inputs.environment == 'stg' && secrets.INFERENCE_BASE_URL_STG || secrets.INFERENCE_BASE_URL_PRD }} + INFERENCE_API_KEY: ${{ inputs.environment == 'dev' && secrets.INFERENCE_API_KEY_DEV || inputs.environment == 'stg' && secrets.INFERENCE_API_KEY_STG || secrets.INFERENCE_API_KEY_PRD }} + MCP_BASIC_KEY: ${{ inputs.environment == 'dev' && secrets.MCP_BASIC_KEY_DEV || inputs.environment == 'stg' && secrets.MCP_BASIC_KEY_STG || secrets.MCP_BASIC_KEY_PRD }} + MCP_PAPERSCORE_KEY: ${{ inputs.environment == 'dev' && secrets.MCP_PAPERSCORE_KEY_DEV || inputs.environment == 'stg' && secrets.MCP_PAPERSCORE_KEY_STG || secrets.MCP_PAPERSCORE_KEY_PRD }} + XTRAMCP_OPENAI_BASE_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENAI_BASE_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENAI_BASE_URL_STG || secrets.XTRAMCP_OPENAI_BASE_URL_PRD }} + XTRAMCP_OPENAI_API_KEY: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENAI_API_KEY_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENAI_API_KEY_STG || secrets.XTRAMCP_OPENAI_API_KEY_PRD }} + XTRAMCP_OPENREVIEW_BASE_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_BASE_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_BASE_URL_STG || secrets.XTRAMCP_OPENREVIEW_BASE_URL_PRD }} + XTRAMCP_OPENREVIEW_USERNAME: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_USERNAME_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_USERNAME_STG || secrets.XTRAMCP_OPENREVIEW_USERNAME_PRD }} + XTRAMCP_OPENREVIEW_PASSWORD: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_PASSWORD_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_PASSWORD_STG || secrets.XTRAMCP_OPENREVIEW_PASSWORD_PRD }} + XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_STG || secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_PRD }} + XTRAMCP_DOI_EMAIL_ADDRESS: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_DOI_EMAIL_ADDRESS_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_DOI_EMAIL_ADDRESS_STG || secrets.XTRAMCP_DOI_EMAIL_ADDRESS_PRD }} + XTRAMCP_ACL_METADATA_DB_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_ACL_METADATA_DB_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_ACL_METADATA_DB_URL_STG || secrets.XTRAMCP_ACL_METADATA_DB_URL_PRD }} + XTRAMCP_ARXIV_METADATA_DB_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_ARXIV_METADATA_DB_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_ARXIV_METADATA_DB_URL_STG || secrets.XTRAMCP_ARXIV_METADATA_DB_URL_PRD }} + XTRAMCP_MONGO_URI: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_MONGO_URI_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_MONGO_URI_STG || secrets.XTRAMCP_MONGO_URI_PRD }} + MONGO_URI: ${{ inputs.environment == 'dev' && secrets.MONGO_URI_DEV || inputs.environment == 'stg' && secrets.MONGO_URI_STG || secrets.MONGO_URI_PRD }} + GHCR_DOCKER_CONFIG: ${{ inputs.environment == 'dev' && secrets.GHCR_DOCKER_CONFIG_DEV || inputs.environment == 'stg' && secrets.GHCR_DOCKER_CONFIG_STG || secrets.GHCR_DOCKER_CONFIG_PRD }} + CLOUDFLARE_TUNNEL_TOKEN: ${{ inputs.environment == 'dev' && secrets.CLOUDFLARE_TUNNEL_TOKEN_DEV || inputs.environment == 'stg' && secrets.CLOUDFLARE_TUNNEL_TOKEN_STG || secrets.CLOUDFLARE_TUNNEL_TOKEN_PRD }} + run: | + export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }} + mkdir -p ../deploy/${{ inputs.environment }} + ./hack/${{ inputs.environment }}.sh > ../deploy/${{ inputs.environment }}/paperdebugger.yaml + - name: Push changes to deploy repo + run: | + export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }} + cd ../deploy + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git add ${{ inputs.environment }}/paperdebugger.yaml + git diff --staged --quiet || git commit -m "chore: update paperdebugger ${{ inputs.environment }}, revision ${IMAGE_TAG}" + git push diff --git a/.github/workflows/_build-ext.yml b/.github/workflows/_build-ext.yml new file mode 100644 index 00000000..6bf91858 --- /dev/null +++ b/.github/workflows/_build-ext.yml @@ -0,0 +1,70 @@ +name: Build and publish Chrome extension + +on: + workflow_call: + inputs: + environment: + required: true + type: string # stg, prd + ref: + required: true + type: string # staging, main + extension_id: + required: true + type: string + +jobs: + build-chrome-extension: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: lts/* + + - name: Build + run: | + export GRAFANA_API_KEY=${{ secrets.GRAFANA_API_KEY }} + cd webapp/_webapp + npm install + npm run build:${{ inputs.environment }}:chrome + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: paperdebugger-chrome-extension-${{ inputs.environment }} + path: webapp/_webapp/dist + + push-to-chrome-web-store: + runs-on: ubuntu-latest + needs: build-chrome-extension + steps: + - name: Install zip + run: | + sudo apt-get update && sudo apt-get install -y zip + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: paperdebugger-chrome-extension-${{ inputs.environment }} + path: dist + + - name: Zip extension files + run: | + zip -r dist.zip dist/* + + - name: Upload to Chrome Web Store (upload only) + uses: mobilefirstllc/cws-publish@latest + with: + action: 'upload' + client_id: ${{ secrets.CHROME_EXT_CLIENT_ID }} + client_secret: ${{ secrets.CHROME_EXT_CLIENT_SECRET }} + refresh_token: ${{ secrets.CHROME_EXT_REFRESH_TOKEN }} + extension_id: ${{ inputs.extension_id }} + zip_file: 'dist.zip' diff --git a/.github/workflows/build-backend-dev.yml b/.github/workflows/build-backend-dev.yml index b80802e8..b1a8005b 100644 --- a/.github/workflows/build-backend-dev.yml +++ b/.github/workflows/build-backend-dev.yml @@ -5,83 +5,10 @@ on: - development jobs: - build-backend-and-push: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }} - PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: development - - name: Setup env - run: | - echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV - echo "BRANCH_NAME=development" >> $GITHUB_ENV - echo "VERSION=development" >> $GITHUB_ENV - - name: Log in to the container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push docker image - run: | - make all - - id: output - name: Output image tag - run: | - export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION} - echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT - echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT - - deploy-backend: - needs: build-backend-and-push - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: development - - name: Clone deploy repo - run: | - git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy - - name: Generate kubernetes manifests - env: - OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL_DEV }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_DEV }} - INFERENCE_BASE_URL: ${{ secrets.INFERENCE_BASE_URL_DEV }} - INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY_DEV }} - MCP_BASIC_KEY: ${{ secrets.MCP_BASIC_KEY_DEV }} - MCP_PAPERSCORE_KEY: ${{ secrets.MCP_PAPERSCORE_KEY_DEV }} - XTRAMCP_OPENAI_BASE_URL: ${{ secrets.XTRAMCP_OPENAI_BASE_URL_DEV }} - XTRAMCP_OPENAI_API_KEY: ${{ secrets.XTRAMCP_OPENAI_API_KEY_DEV }} - XTRAMCP_OPENREVIEW_BASE_URL: ${{ secrets.XTRAMCP_OPENREVIEW_BASE_URL_DEV }} - XTRAMCP_OPENREVIEW_USERNAME: ${{ secrets.XTRAMCP_OPENREVIEW_USERNAME_DEV }} - XTRAMCP_OPENREVIEW_PASSWORD: ${{ secrets.XTRAMCP_OPENREVIEW_PASSWORD_DEV }} - XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_DEV }} - XTRAMCP_DOI_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_DOI_EMAIL_ADDRESS_DEV }} - XTRAMCP_ARXIV_METADATA_DB_URL: ${{ secrets.XTRAMCP_ARXIV_METADATA_DB_URL_DEV }} - XTRAMCP_MONGO_URI: ${{ secrets.XTRAMCP_MONGO_URI_DEV }} - MONGO_URI: ${{ secrets.MONGO_URI_DEV }} - GHCR_DOCKER_CONFIG: ${{ secrets.GHCR_DOCKER_CONFIG_DEV }} - CLOUDFLARE_TUNNEL_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_DEV }} - run: | - export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }} - mkdir -p ../deploy/dev - ./hack/dev.sh > ../deploy/dev/paperdebugger.yaml - - name: Push changes to deploy repo - run: | - export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }} - cd ../deploy - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git add dev/paperdebugger.yaml - git diff --staged --quiet || git commit -m "chore: update paperdebugger dev, revision ${IMAGE_TAG}" - git push + build-and-deploy: + uses: ./.github/workflows/_build-backend.yml + secrets: inherit + with: + environment: dev + ref: development + version_tag: development diff --git a/.github/workflows/build-backend-prd.yml b/.github/workflows/build-backend-prd.yml index 412e8763..9531d89d 100644 --- a/.github/workflows/build-backend-prd.yml +++ b/.github/workflows/build-backend-prd.yml @@ -1,90 +1,13 @@ name: build backend for prd on: - push: - tags: - - 'v*' repository_dispatch: types: [prd] jobs: - build-backend-and-push: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }} - PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - name: Setup env - run: | - echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV - echo "BRANCH_NAME=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV - echo "VERSION=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV - - name: Log in to the container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push docker image - run: | - make all - - id: output - name: Output image tag - run: | - export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION} - echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT - echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT - - deploy-backend: - needs: build-backend-and-push - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - name: Clone deploy repo - run: | - git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy - - name: Generate kubernetes manifests - env: - OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL_PRD }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_PRD }} - INFERENCE_BASE_URL: ${{ secrets.INFERENCE_BASE_URL_PRD }} - INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY_PRD }} - MCP_BASIC_KEY: ${{ secrets.MCP_BASIC_KEY_PRD }} - MCP_PAPERSCORE_KEY: ${{ secrets.MCP_PAPERSCORE_KEY_PRD }} - XTRAMCP_OPENAI_BASE_URL: ${{ secrets.XTRAMCP_OPENAI_BASE_URL_PRD }} - XTRAMCP_OPENAI_API_KEY: ${{ secrets.XTRAMCP_OPENAI_API_KEY_PRD }} - XTRAMCP_OPENREVIEW_BASE_URL: ${{ secrets.XTRAMCP_OPENREVIEW_BASE_URL_PRD }} - XTRAMCP_OPENREVIEW_USERNAME: ${{ secrets.XTRAMCP_OPENREVIEW_USERNAME_PRD }} - XTRAMCP_OPENREVIEW_PASSWORD: ${{ secrets.XTRAMCP_OPENREVIEW_PASSWORD_PRD }} - XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_PRD }} - XTRAMCP_DOI_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_DOI_EMAIL_ADDRESS_PRD }} - XTRAMCP_ACL_METADATA_DB_URL: ${{ secrets.XTRAMCP_ACL_METADATA_DB_URL_PRD }} - XTRAMCP_ARXIV_METADATA_DB_URL: ${{ secrets.XTRAMCP_ARXIV_METADATA_DB_URL_PRD }} - XTRAMCP_MONGO_URI: ${{ secrets.XTRAMCP_MONGO_URI_PRD }} - MONGO_URI: ${{ secrets.MONGO_URI_PRD }} - GHCR_DOCKER_CONFIG: ${{ secrets.GHCR_DOCKER_CONFIG_PRD }} - CLOUDFLARE_TUNNEL_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_PRD }} - run: | - export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }} - mkdir -p ../deploy/prd - ./hack/prd.sh > ../deploy/prd/paperdebugger.yaml - - name: Push changes to deploy repo - run: | - export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }} - cd ../deploy - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git add prd/paperdebugger.yaml - git diff --staged --quiet || git commit -m "chore: update paperdebugger prd, revision ${IMAGE_TAG}" - git push + build-and-deploy: + uses: ./.github/workflows/_build-backend.yml + secrets: inherit + with: + environment: prd + ref: main + version_tag: ${{ github.event.client_payload.tag }} diff --git a/.github/workflows/build-backend-stg.yml b/.github/workflows/build-backend-stg.yml index bcc54942..54bfffb0 100644 --- a/.github/workflows/build-backend-stg.yml +++ b/.github/workflows/build-backend-stg.yml @@ -1,90 +1,13 @@ name: build backend for stg on: - push: - tags: - - 'v*' repository_dispatch: types: [stg] jobs: - build-backend-and-push: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }} - PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: staging - - name: Setup env - run: | - echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV - echo "BRANCH_NAME=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV - echo "VERSION=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV - - name: Log in to the container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push docker image - run: | - make all - - id: output - name: Output image tag - run: | - export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION} - echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT - echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT - - deploy-backend: - needs: build-backend-and-push - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: staging - - name: Clone deploy repo - run: | - git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy - - name: Generate kubernetes manifests - env: - OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL_STG }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_STG }} - INFERENCE_BASE_URL: ${{ secrets.INFERENCE_BASE_URL_STG }} - INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY_STG }} - MCP_BASIC_KEY: ${{ secrets.MCP_BASIC_KEY_STG }} - MCP_PAPERSCORE_KEY: ${{ secrets.MCP_PAPERSCORE_KEY_STG }} - XTRAMCP_OPENAI_BASE_URL: ${{ secrets.XTRAMCP_OPENAI_BASE_URL_STG }} - XTRAMCP_OPENAI_API_KEY: ${{ secrets.XTRAMCP_OPENAI_API_KEY_STG }} - XTRAMCP_OPENREVIEW_BASE_URL: ${{ secrets.XTRAMCP_OPENREVIEW_BASE_URL_STG }} - XTRAMCP_OPENREVIEW_USERNAME: ${{ secrets.XTRAMCP_OPENREVIEW_USERNAME_STG }} - XTRAMCP_OPENREVIEW_PASSWORD: ${{ secrets.XTRAMCP_OPENREVIEW_PASSWORD_STG }} - XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_STG }} - XTRAMCP_DOI_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_DOI_EMAIL_ADDRESS_STG }} - XTRAMCP_ACL_METADATA_DB_URL: ${{ secrets.XTRAMCP_ACL_METADATA_DB_URL_STG }} - XTRAMCP_ARXIV_METADATA_DB_URL: ${{ secrets.XTRAMCP_ARXIV_METADATA_DB_URL_STG }} - XTRAMCP_MONGO_URI: ${{ secrets.XTRAMCP_MONGO_URI_STG }} - MONGO_URI: ${{ secrets.MONGO_URI_STG }} - GHCR_DOCKER_CONFIG: ${{ secrets.GHCR_DOCKER_CONFIG_STG }} - CLOUDFLARE_TUNNEL_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_STG }} - run: | - export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }} - mkdir -p ../deploy/stg - ./hack/stg.sh > ../deploy/stg/paperdebugger.yaml - - name: Push changes to deploy repo - run: | - export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }} - cd ../deploy - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git add stg/paperdebugger.yaml - git diff --staged --quiet || git commit -m "chore: update paperdebugger stg, revision ${IMAGE_TAG}" - git push + build-and-deploy: + uses: ./.github/workflows/_build-backend.yml + secrets: inherit + with: + environment: stg + ref: staging + version_tag: ${{ github.event.client_payload.tag }} diff --git a/.github/workflows/build-ext-dev.yml b/.github/workflows/build-ext-dev.yml deleted file mode 100644 index b71e031b..00000000 --- a/.github/workflows/build-ext-dev.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: build chrome extension for dev (unsupported) - -on: - repository_dispatch: - types: [dev] - -jobs: - build-chrome-extension: - runs-on: ubuntu-latest - steps: - - name: 🚫 Dev extension build/release is not supported - run: | - echo "::notice::Dev environment is not supported for Chrome extension build/release." - cat <<'EOF' - Development environment is not supported to build a Chrome extension and push to the Chrome Web Store. - - The reason is that we only have two versions of the third-party extension: - 1. Production - 2. Staging - - If you want to release your new Chrome extension, please use the staging environment. - EOF diff --git a/.github/workflows/build-ext-prd.yml b/.github/workflows/build-ext-prd.yml index b8ca7984..bd5272de 100644 --- a/.github/workflows/build-ext-prd.yml +++ b/.github/workflows/build-ext-prd.yml @@ -1,60 +1,13 @@ name: build chrome extension for prd - on: repository_dispatch: types: [prd] jobs: - build-chrome-extension: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # 获取完整历史 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version: lts/* - - - name: Build - run: | - export GRAFANA_API_KEY=${{ secrets.GRAFANA_API_KEY }} - cd webapp/_webapp - npm install - npm run build:prd:chrome - - - name: 👉 Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: paperdebugger-chrome-extension-prd - path: webapp/_webapp/dist - - push-to-chrome-web-store: - runs-on: ubuntu-latest - needs: build-chrome-extension - steps: - - name: Install zip - run: | - sudo apt-get update && sudo apt-get install -y zip - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: paperdebugger-chrome-extension-prd - path: dist - - - name: Zip extension files - run: | - zip -r dist.zip dist/* - - - name: Upload to Chrome Web Store (upload only) - uses: mobilefirstllc/cws-publish@latest - with: - action: 'upload' # one of: upload, publish, testers - client_id: ${{ secrets.CHROME_EXT_CLIENT_ID }} - client_secret: ${{ secrets.CHROME_EXT_CLIENT_SECRET }} - refresh_token: ${{ secrets.CHROME_EXT_REFRESH_TOKEN }} - extension_id: 'dfkedikhakpapbfcnbpmfhpklndgiaog' - zip_file: 'dist.zip' + build-and-publish: + uses: ./.github/workflows/_build-ext.yml + secrets: inherit + with: + environment: prd + ref: main + extension_id: dfkedikhakpapbfcnbpmfhpklndgiaog diff --git a/.github/workflows/build-ext-stg.yml b/.github/workflows/build-ext-stg.yml index d281d12a..1656a420 100644 --- a/.github/workflows/build-ext-stg.yml +++ b/.github/workflows/build-ext-stg.yml @@ -1,61 +1,13 @@ name: build chrome extension for stg - on: repository_dispatch: types: [stg] jobs: - build-chrome-extension: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: staging - fetch-depth: 0 # 获取完整历史 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version: lts/* - - - name: Build - run: | - export GRAFANA_API_KEY=${{ secrets.GRAFANA_API_KEY }} - cd webapp/_webapp - npm install - npm run build:stg:chrome - - - name: 👉 Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: paperdebugger-chrome-extension-stg - path: webapp/_webapp/dist - - push-to-chrome-web-store: - runs-on: ubuntu-latest - needs: build-chrome-extension - steps: - - name: Install zip - run: | - sudo apt-get update && sudo apt-get install -y zip - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: paperdebugger-chrome-extension-stg - path: dist - - - name: Zip extension files - run: | - zip -r dist.zip dist/* - - - name: Upload to Chrome Web Store (upload only) - uses: mobilefirstllc/cws-publish@latest - with: - action: 'upload' # one of: upload, publish, testers - client_id: ${{ secrets.CHROME_EXT_CLIENT_ID }} - client_secret: ${{ secrets.CHROME_EXT_CLIENT_SECRET }} - refresh_token: ${{ secrets.CHROME_EXT_REFRESH_TOKEN }} - extension_id: 'gjkgbnnlfophcfffoinfgdnhjnfankbn' - zip_file: 'dist.zip' + build-and-publish: + uses: ./.github/workflows/_build-ext.yml + secrets: inherit + with: + environment: stg + ref: staging + extension_id: gjkgbnnlfophcfffoinfgdnhjnfankbn diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1b4f89f5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,152 @@ +# Contributing to PaperDebugger + +## Getting Started + +**Prerequisites:** Go 1.24+, MongoDB, Node.js + +```bash +make deps # install dev tooling (linters, protoc, buf, wire, frontend deps) +cp .env.example .env # configure environment variables +docker run -d --name mongodb -p 27017:27017 mongo:latest # start MongoDB +make build && ./dist/pd.exe # build and run +``` + +See [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md) for detailed setup instructions. + +## Branch Structure & Naming + +| Branch | Purpose | Version Bump | Protection | +|--------|---------|-------------|------------| +| `main` | Production — every commit must be deployable | Minor (`2.1.x` → `2.2.0`) | Fully protected | +| `staging` | Pre-production checkpoint, always aligned with or following `main` | Patch (`2.1.9` → `2.1.10`) | PR required, review optional | +| `development` | Sandbox for rapid iteration and prod-env simulation | None | No restrictions | +| `feat/*`, `fix/*` | Working branches for features and bug fixes | — | — | + +Branch names are kept simple — use `feat/*` for new functionality and `fix/*` for bug fixes. The semantic distinction (docs, chore, refactor, etc.) is carried by the PR title instead (see [Pull Request Guidelines](#pull-request-guidelines)). + +## Merge Workflow + +### Standard Flow (substantial changes) + +``` +feat/* or fix/* → staging → main + ↓ ↓ + integration production + testing release +``` + +1. Create `feat/*` or `fix/*` branch from `staging` +2. Develop and test locally +3. PR to `staging` — merge for integration testing on the staging endpoint +4. Once no defects are found, PR from `staging` to `main` + +### Fast-track Flow (small, isolated changes) + +``` +feat/* or fix/* → main +``` + +For small, well-tested changes (typo fixes, minor refactors) that don't need integration testing. PR review is still required. + +> **Note:** Fast-track merges to `main` will cause `staging` to drift behind. This is fine — `staging` does not need to be in sync at all times. Sync `staging` with `main` before starting a new round of integration testing (e.g., merge `main` into `staging`). + +### When to Use Which + +| Use `staging` | Go direct to `main` | +|---------------|---------------------| +| Changes touch multiple components | Small, isolated changes | +| Needs integration testing with recent changes | Confident in local testing | +| Uncertain about production impact | Low-risk (typos, minor refactors) | +| Want to verify on a prod-like environment first | — | + +**Important:** Keep `staging` clean. It is the last gate before production — take care not to leave it in a broken state. + +## Branch Protection + +**`main`** +- Pull request required (no direct pushes) +- At least 1 approving review required +- All status checks must pass +- Linear history enforced (no merge commits) +- Force pushes and deletions blocked + +**`staging`** +- Pull request required +- Review optional, but **strongly recommended** for complex changes +- All status checks must pass +- Force pushes allowed (for rebasing) + +**`development`** +- No restrictions — free to push / merge without PR review +- Useful for simulating production environment behavior + +## Pull Request Guidelines + +PR titles **must** follow [Conventional Commits](https://www.conventionalcommits.org/) format (enforced by CI). While branch names use simple `feat/*` or `fix/*` prefixes, the PR title carries the semantic meaning: + +``` +feat: add tab completion support +fix: resolve token expiration bug +chore: update dependencies +docs: improve setup instructions +refactor: simplify chat service +test: add citation parsing tests +ci: add staging deploy workflow +``` + +**Merge strategy:** +- PR to `staging` — **merge commit** (retain full commit history for clarity and debugging during integration testing) +- PR to `main` — **squash and merge** (one atomic, deployable commit per PR to keep history clean) + +**Two-layer review process:** +- PR to `staging` — first review layer, catches mistakes early (review optional but recommended) +- PR to `main` — second review layer, always requires approval + +## Version Numbering + +We follow semantic versioning (`MAJOR.MINOR.PATCH`): + +| Component | When | How | +|-----------|------|-----| +| **Major** (`X.0.0`) | Breaking changes | Manual | +| **Minor** (`0.X.0`) | Merge to `main` | Auto-increment | +| **Patch** (`0.0.X`) | Merge to `staging` | Auto-increment | + +Example: `feat/new-feature` → staging (`2.1.9` → `2.1.10`) → main (`2.1.10` → `2.2.0`) + +Minor and patch versions are bumped automatically via CI. For a **major version bump** (breaking changes), manually create a tag: + +```bash +git tag v3.0.0 +git push origin v3.0.0 +``` + +Subsequent automated bumps will increment from the new tag. + +## Code Quality + +Run these before submitting a PR: + +```bash +make fmt # format Go, proto, and frontend code +make lint # lint Go (golangci-lint), proto (buf lint), and frontend (eslint) +make test # run all tests with coverage (requires MongoDB on localhost:27017) +``` + +All `make` commands and their details can be inspected in the [Makefile](Makefile). + +**Code stability guarantee:** Every commit on `main` must: +- Compile without errors +- Pass all tests +- Be deployable to production +- Have no known breaking bugs + +## Proto / API Changes + +When modifying `.proto` files in `proto/`: + +```bash +make gen # regenerate Go + gRPC + gateway + TypeScript bindings +``` + +Commit the generated files in `pkg/gen/` alongside your proto changes. diff --git a/README.md b/README.md index 1fcbada2..26088813 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Stay connected with the PaperDebugger community! Join our [Discord](https://disc - [Custom Endpoint Configuration](#custom-endpoint-configuration) - [Architecture Overview](#architecture-overview) - [Self-Host Development Setup](#self-host-development-setup) +- [Contributing](#contributing) ## Features @@ -103,6 +104,10 @@ The PaperDebugger backend is built with modern technologies: Please refer to [DEVELOPMENT.md](docs/DEVELOPMENT.md). +## Contributing + +Please refer to [CONTRIBUTING.md](CONTRIBUTING.md) for branch structure, merge workflow, PR guidelines, and code quality requirements. + ## Citation ``` @misc{hou2025paperdebugger, diff --git a/hack/values-dev.yaml b/hack/values-dev.yaml index 5f95b780..20fd0374 100644 --- a/hack/values-dev.yaml +++ b/hack/values-dev.yaml @@ -1,4 +1,9 @@ namespace: paperdebugger-dev +nodeSelector: + kubernetes.io/hostname: xtras3 +cloudflareNodeSelector: + kubernetes.io/hostname: nuc + mongo: in_cluster: false diff --git a/hack/values-prd.yaml b/hack/values-prd.yaml index d6b8402e..e508e664 100644 --- a/hack/values-prd.yaml +++ b/hack/values-prd.yaml @@ -1,4 +1,9 @@ namespace: paperdebugger +nodeSelector: + kubernetes.io/hostname: xtras3 +cloudflareNodeSelector: + kubernetes.io/hostname: nuc + mongo: in_cluster: false diff --git a/hack/values-stg.yaml b/hack/values-stg.yaml index bad7f201..4813ce19 100644 --- a/hack/values-stg.yaml +++ b/hack/values-stg.yaml @@ -1,4 +1,9 @@ namespace: paperdebugger-stg +nodeSelector: + kubernetes.io/hostname: xtras3 +cloudflareNodeSelector: + kubernetes.io/hostname: nuc + mongo: in_cluster: false diff --git a/helm-chart/templates/cloudflare.yaml b/helm-chart/templates/cloudflare.yaml index ab89262c..25371b3c 100644 --- a/helm-chart/templates/cloudflare.yaml +++ b/helm-chart/templates/cloudflare.yaml @@ -23,6 +23,10 @@ spec: labels: pod: cloudflared spec: + {{- with .Values.cloudflareNodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} securityContext: sysctls: # Allows ICMP traffic (ping, traceroute) to resources behind cloudflared. diff --git a/helm-chart/templates/mongo.yaml b/helm-chart/templates/mongo.yaml index 47e787d7..6871e8fc 100644 --- a/helm-chart/templates/mongo.yaml +++ b/helm-chart/templates/mongo.yaml @@ -14,6 +14,10 @@ spec: labels: app: mongo spec: + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: mongo image: {{ .Values.mongo.image }} diff --git a/helm-chart/templates/paperdebugger-mcp.yaml b/helm-chart/templates/paperdebugger-mcp.yaml index 8896fe31..3a0ada45 100644 --- a/helm-chart/templates/paperdebugger-mcp.yaml +++ b/helm-chart/templates/paperdebugger-mcp.yaml @@ -13,6 +13,10 @@ spec: labels: app: paperdebugger-mcp-server spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} imagePullSecrets: - name: ghcr-secret containers: diff --git a/helm-chart/templates/paperdebugger-xtramcp-server.yaml b/helm-chart/templates/paperdebugger-xtramcp-server.yaml index 4d663dc6..072f41a7 100644 --- a/helm-chart/templates/paperdebugger-xtramcp-server.yaml +++ b/helm-chart/templates/paperdebugger-xtramcp-server.yaml @@ -13,6 +13,10 @@ spec: labels: app: paperdebugger-xtramcp-server spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} imagePullSecrets: - name: ghcr-secret containers: diff --git a/helm-chart/templates/paperdebugger.yaml b/helm-chart/templates/paperdebugger.yaml index 982e1f1a..b7b4649d 100644 --- a/helm-chart/templates/paperdebugger.yaml +++ b/helm-chart/templates/paperdebugger.yaml @@ -13,6 +13,10 @@ spec: labels: app: paperdebugger spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} imagePullSecrets: - name: ghcr-secret initContainers: @@ -72,6 +76,8 @@ data: INFERENCE_BASE_URL: "{{ .Values.inference_base_url }}" INFERENCE_API_KEY: "{{ .Values.inference_api_key }}" JWT_SIGNING_KEY: "{{ .Values.jwt_signing_key }}" - {{ if not .Values.mongo.in_cluster }} + {{- if .Values.mongo.in_cluster }} + PD_MONGO_URI: "mongodb://mongo.{{ .Values.namespace }}.svc.cluster.local:27017/?replicaSet=in-cluster" + {{- else }} PD_MONGO_URI: "{{ .Values.mongo.uri }}" - {{ end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 86546e8e..b89d4051 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -16,6 +16,9 @@ paperdebuggerMcpServer: paperdebuggerXtraMcpServer: image: ghcr.io/paperdebugger/xtragpt-mcp-server:sha-510bc96 +nodeSelector: {} +cloudflareNodeSelector: {} + mongo: in_cluster: true uri: "" # if in_cluster is false, use the external mongo instead