local-deep-research/.github/workflows/publish.yml

name: Publish to PyPI

# SECURITY: Only triggered via repository_dispatch from release.yml
# (after security gate passes). We intentionally do NOT support
# workflow_dispatch because that would bypass security checks.
# To re-publish, trigger a new release through release.yml.
on:
  repository_dispatch:
    types: [publish-pypi]

permissions: {}  # Minimal top-level for OSSF Scorecard Token-Permissions

jobs:
  # Build frontend if package.json exists (isolated, no secrets)
  build-frontend:
    runs-on: ubuntu-latest
    outputs:
      has-frontend: ${{ steps.check.outputs.has-frontend }}
    container:
      image: node:20-alpine@sha256:bcd88137d802e2482c9df3cdec71e0431857ebbbdba6973776b5593214056d86 # node:20-alpine
      # Note: Network is needed for npm ci to work, but no secrets are available
      options: --user 1001
    permissions:
      contents: read

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
      with:
        egress-policy: audit

    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      with:
        persist-credentials: false

    - name: Check for frontend assets
      id: check
      run: |
        if [ -f "package.json" ]; then
          echo "has-frontend=true" >> "$GITHUB_OUTPUT"
          echo "Found package.json - will build frontend"
        else
          echo "has-frontend=false" >> "$GITHUB_OUTPUT"
          echo "ERROR: No package.json found - frontend build is required for PyPI releases"
          exit 1
        fi

    - name: Build frontend
      if: steps.check.outputs.has-frontend == 'true'
      run: |
        echo "=== Frontend Build Process Starting ==="
        echo "Current directory: $(pwd)"
        echo "Node version: $(node --version)"
        echo "NPM version: $(npm --version)"

        # Install dependencies from root package.json
        echo ""
        echo "📦 Installing dependencies..."
        # Use npm ci for reproducible builds from lockfile
        npm ci --ignore-scripts --no-audit --no-fund
        echo "✅ Dependencies installed"

        # Show pre-build directory structure
        echo ""
        echo "📁 Pre-build directory structure:"
        echo "Contents of src/local_deep_research/web/static/:"
        find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "Directory not found"

        # Build with Vite (outputs to src/local_deep_research/web/static/dist)
        echo ""
        echo "🔨 Running Vite build..."
        npm run build
        echo "✅ Vite build completed"

        # Show post-build directory structure
        echo ""
        echo "📁 Post-build directory structure:"
        echo "Contents of src/local_deep_research/web/static/:"
        find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null
        echo ""
        echo "Contents of src/local_deep_research/web/static/dist/:"
        find src/local_deep_research/web/static/dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null
        echo ""
        echo "Looking for manifest.json:"
        find src/local_deep_research/web/static/dist -name "manifest.json" -type f 2>/dev/null || echo "No manifest.json found"

        # Verify the build completed successfully
        if [ ! -f "src/local_deep_research/web/static/dist/.vite/manifest.json" ]; then
          echo "❌ ERROR: Build failed - manifest.json not created at expected location"
          echo "Expected location: src/local_deep_research/web/static/dist/.vite/manifest.json"
          echo "Actual dist contents:"
          find src/local_deep_research/web/static/dist -type f | head -20
          exit 1
        fi

        echo "✅ Found manifest.json at: src/local_deep_research/web/static/dist/.vite/manifest.json"
        echo "Manifest contents (first 10 lines):"
        head -10 src/local_deep_research/web/static/dist/.vite/manifest.json

        # Create build marker
        echo ""
        echo "📝 Creating build marker..."
        echo "{\"status\":\"complete\",\"built\":\"$(date -Iseconds)\"}" > src/local_deep_research/web/static/.frontend-built
        echo "✅ Build marker created"

    - name: Upload frontend artifacts
      if: steps.check.outputs.has-frontend == 'true'
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: frontend-assets
        path: src/local_deep_research/web/static/
        retention-days: 1
        include-hidden-files: true  # Ensure .vite directory is included

  # Build Python package (isolated, no PyPI access)
  build-package:
    needs: build-frontend
    runs-on: ubuntu-latest
    container:
      image: python:3.12-slim@sha256:971f04b358cf483ec445a8d388fb55267451f080d90fb136c8e69684a02a9604 # python:3.12-slim
    permissions:
      contents: read

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
      with:
        egress-policy: audit

    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      with:
        persist-credentials: false

    - name: Install build dependencies
      run: |
        apt-get update && apt-get install -y libsqlcipher-dev build-essential
        pip install pdm==2.26.2

    - name: Download frontend artifacts
      if: needs.build-frontend.outputs.has-frontend == 'true'
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      continue-on-error: true
      with:
        name: frontend-assets
        path: src/local_deep_research/web/static/

    - name: Verify frontend build artifacts
      run: |
        echo "=== Frontend Artifacts Verification ==="
        echo "Current directory: $(pwd)"
        echo ""
        echo "📁 Checking downloaded artifacts structure:"
        echo "Contents of src/:"
        find src/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/ directory not found"
        echo ""
        echo "Contents of src/local_deep_research/:"
        find src/local_deep_research/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/ not found"
        echo ""
        echo "Contents of src/local_deep_research/web/:"
        find src/local_deep_research/web/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/web/ not found"
        echo ""
        echo "Contents of src/local_deep_research/web/static/:"
        find src/local_deep_research/web/static/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null || echo "src/local_deep_research/web/static/ not found"

        # Check if dist directory exists
        echo ""
        echo "🔍 Checking for dist directory..."
        if [ ! -d "src/local_deep_research/web/static/dist" ]; then
          echo "❌ ERROR: Frontend dist directory not found!"
          echo "Expected location: src/local_deep_research/web/static/dist"
          echo "Actual structure of static directory:"
          find src/local_deep_research/web/static -type d 2>/dev/null | head -20
          echo "The frontend build artifacts are missing."
          exit 1
        fi
        echo "✅ Found dist directory at: src/local_deep_research/web/static/dist"

        # Show dist directory structure
        echo ""
        echo "📁 Dist directory structure:"
        echo "Contents of dist/:"
        find src/local_deep_research/web/static/dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null
        echo ""
        echo "All files in dist (recursive):"
        find src/local_deep_research/web/static/dist -type f | head -20

        # Check for critical files
        echo ""
        echo "🔍 Checking for manifest.json..."
        # Check both possible locations for manifest.json
        MANIFEST_PATH=""
        if [ -f "src/local_deep_research/web/static/dist/.vite/manifest.json" ]; then
          MANIFEST_PATH="src/local_deep_research/web/static/dist/.vite/manifest.json"
        elif [ -f "src/local_deep_research/web/static/dist/manifest.json" ]; then
          MANIFEST_PATH="src/local_deep_research/web/static/dist/manifest.json"
        fi

        if [ -z "$MANIFEST_PATH" ]; then
          echo "❌ ERROR: Vite manifest.json not found!"
          echo "Checked locations:"
          echo "  - src/local_deep_research/web/static/dist/.vite/manifest.json"
          echo "  - src/local_deep_research/web/static/dist/manifest.json"
          echo "Looking for any manifest.json files:"
          find src/local_deep_research/web/static -name "manifest.json" -type f 2>/dev/null || echo "None found"
          echo "Checking if .vite directory exists:"
          find src/local_deep_research/web/static/dist/ -maxdepth 1 -name ".*" -type d || echo "No hidden directories found"
          echo ""
          echo "This is likely due to artifact transfer losing the hidden .vite directory."
          echo "The include-hidden-files option should fix this."
          echo "The frontend build appears to be incomplete - cannot continue."
          exit 1
        fi
        echo "✅ Found manifest.json at: $MANIFEST_PATH"
        echo "Manifest size: $(wc -c < "$MANIFEST_PATH") bytes"

        # Check for JS files
        echo ""
        echo "🔍 Checking for JavaScript files..."
        JS_COUNT=$(find src/local_deep_research/web/static/dist -name "*.js" -type f | wc -l)
        if [ "$JS_COUNT" -eq 0 ]; then
          echo "❌ ERROR: No JavaScript files found!"
          echo "Expected JS files in dist/"
          echo "The frontend build appears to be incomplete."
          exit 1
        fi
        echo "✅ Found $JS_COUNT JavaScript file(s)"
        echo "JS files:"
        find src/local_deep_research/web/static/dist -name "*.js" -type f | head -5

        # Check for CSS files
        echo ""
        echo "🔍 Checking for CSS files..."
        CSS_COUNT=$(find src/local_deep_research/web/static/dist -name "*.css" -type f | wc -l)
        if [ "$CSS_COUNT" -eq 0 ]; then
          echo "❌ ERROR: No CSS files found!"
          echo "Expected CSS files in dist/"
          echo "The frontend build appears to be incomplete."
          exit 1
        fi
        echo "✅ Found $CSS_COUNT CSS file(s)"
        echo "CSS files:"
        find src/local_deep_research/web/static/dist -name "*.css" -type f | head -5

        # Check for build marker
        echo ""
        echo "🔍 Checking for build marker..."
        if [ ! -f "src/local_deep_research/web/static/.frontend-built" ]; then
          echo "⚠️  WARNING: Frontend build marker not found (non-critical)"
        else
          echo "✅ Found build marker"
          echo "Build marker contents:"
          cat src/local_deep_research/web/static/.frontend-built
        fi

        echo ""
        echo "=== ✅ Frontend build artifacts verified successfully ==="
        echo "Summary:"
        echo "  - Dist directory: ✓"
        echo "  - Manifest.json: ✓"
        echo "  - JS files: $JS_COUNT file(s)"
        echo "  - CSS files: $CSS_COUNT file(s)"

    - name: Build Python package
      run: |
        echo "=== Python Package Build ==="
        echo "Current directory: $(pwd)"
        echo ""
        echo "📦 Building Python package with PDM..."
        pdm build
        echo ""
        echo "✅ Package build completed"
        echo ""
        echo "📁 Package contents:"
        find dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null
        echo ""
        echo "Package sizes:"
        du -h dist/*

    - name: Upload package
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: python-dist
        path: dist/
        retention-days: 1

  # Publish (ONLY job with PyPI access)
  publish:
    needs: build-package
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write
      attestations: write  # Required for generating attestations

    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
      with:
        egress-policy: audit

    - name: Download package
      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      with:
        name: python-dist
        path: dist/

    - name: List packages
      run: |
        echo "=== Downloaded Package Artifacts ==="
        echo "📦 Packages in dist/:"
        ls -lh dist/
        echo ""
        echo "Package details:"
        for file in dist/*; do
          echo "  - $(basename "$file"): $(du -h "$file" | cut -f1)"
        done

    - name: Verify package contents
      run: |
        echo "=== Package Content Verification ==="
        echo "Verifying package contains frontend assets..."
        echo ""

        # Install wheel to inspect package
        echo "📦 Installing wheel for package inspection..."
        pip install wheel==0.46.2

        # Find the wheel file
        WHEEL_FILE=$(find dist -name "*.whl" -type f 2>/dev/null | head -1)
        TAR_FILE=$(find dist -name "*.tar.gz" -type f 2>/dev/null | head -1)

        if [ -z "$WHEEL_FILE" ]; then
          echo "❌ ERROR: No wheel file found in dist/"
          echo "Found files:"
          find dist/ -maxdepth 1 -type f -exec ls -la {} + 2>/dev/null
          exit 1
        fi

        echo "✅ Found wheel file: $WHEEL_FILE"
        echo "Wheel size: $(du -h "$WHEEL_FILE" | cut -f1)"
        echo ""

        if [ -n "$TAR_FILE" ]; then
          echo "✅ Found source distribution: $TAR_FILE"
          echo "Source dist size: $(du -h "$TAR_FILE" | cut -f1)"
        fi
        echo ""

        echo "🔍 Inspecting wheel contents..."
        # Extract and check for frontend files
        python -m zipfile -l "$WHEEL_FILE" > wheel_contents.txt

        TOTAL_FILES=$(wc -l < wheel_contents.txt)
        echo "Total files in wheel: $TOTAL_FILES"
        echo ""

        # Count different file types
        echo "📊 File type breakdown:"
        echo "  Python files: $(grep -c '\.py' wheel_contents.txt || echo 0)"
        echo "  JavaScript files: $(grep -c '\.js' wheel_contents.txt || echo 0)"
        echo "  CSS files: $(grep -c '\.css' wheel_contents.txt || echo 0)"
        echo "  JSON files: $(grep -c '\.json' wheel_contents.txt || echo 0)"
        echo "  HTML files: $(grep -c '\.html' wheel_contents.txt || echo 0)"
        echo ""

        # Check for critical frontend directories/files
        echo "🔍 Checking for frontend assets..."

        echo "Looking for dist directory..."
        if ! grep -q "local_deep_research/web/static/dist/" wheel_contents.txt; then
          echo "❌ ERROR: Frontend dist directory not found in package!"
          echo "Package is missing frontend build artifacts."
          echo "Static files found in package:"
          grep "static" wheel_contents.txt | head -10 || echo "No static files found"
          exit 1
        fi
        echo "✅ Found dist directory"

        echo ""
        echo "Looking for JavaScript files..."
        JS_IN_WHEEL=$(grep -c "local_deep_research/web/static/dist.*\.js" wheel_contents.txt || echo 0)
        if [ "$JS_IN_WHEEL" -eq 0 ]; then
          echo "❌ ERROR: Frontend JS files not found in package!"
          echo "JavaScript files in package:"
          grep "\.js" wheel_contents.txt | head -10 || echo "No JS files found"
          exit 1
        fi
        echo "✅ Found $JS_IN_WHEEL JavaScript file(s) in dist"

        echo ""
        echo "Looking for CSS files..."
        CSS_IN_WHEEL=$(grep -c "local_deep_research/web/static/dist.*\.css" wheel_contents.txt || echo 0)
        if [ "$CSS_IN_WHEEL" -eq 0 ]; then
          echo "❌ ERROR: Frontend CSS files not found in package!"
          echo "CSS files in package:"
          grep "\.css" wheel_contents.txt | head -10 || echo "No CSS files found"
          exit 1
        fi
        echo "✅ Found $CSS_IN_WHEEL CSS file(s) in dist"

        echo ""
        echo "Looking for manifest.json..."
        if ! grep -q "local_deep_research/web/static/dist/.vite/manifest.json" wheel_contents.txt; then
          echo "⚠️  WARNING: manifest.json not found in package (may be okay)"
        else
          echo "✅ Found manifest.json"
        fi

        echo ""
        echo "=== ✅ Package verification successful ==="
        echo "Summary:"
        echo "  - Wheel file: ✓"
        echo "  - Frontend dist: ✓"
        echo "  - JS files: $JS_IN_WHEEL file(s)"
        echo "  - CSS files: $CSS_IN_WHEEL file(s)"
        echo ""
        echo "Sample frontend assets in package:"
        grep "local_deep_research/web/static/dist/" wheel_contents.txt | head -10

    - name: Generate attestations
      uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
      with:
        subject-path: 'dist/*'

    - name: Pre-publish summary
      env:
        EVENT_NAME: ${{ github.event_name }}
        DISPATCH_TAG: ${{ github.event.client_payload.tag }}
        DISPATCH_PRERELEASE: ${{ github.event.client_payload.prerelease }}
        RELEASE_TAG: ${{ github.event.release.tag_name }}
        RELEASE_PRERELEASE: ${{ github.event.release.prerelease }}
      run: |
        echo "=== 🚀 Ready to Publish ==="
        echo ""
        echo "📦 Package Information:"
        # Handle both release event and repository_dispatch
        if [ "$EVENT_NAME" = "repository_dispatch" ]; then
          echo "  - Trigger: repository_dispatch"
          echo "  - Tag: $DISPATCH_TAG"
          echo "  - Prerelease: $DISPATCH_PRERELEASE"
        else
          echo "  - Trigger: release"
          echo "  - Tag: $RELEASE_TAG"
          echo "  - Prerelease: $RELEASE_PRERELEASE"
        fi
        echo ""
        echo "📁 Package files to publish:"
        ls -lh dist/
        echo ""
        echo "✅ All checks passed - proceeding with publication"

    - name: Publish to Test PyPI
      if: (github.event_name == 'release' && github.event.release.prerelease == true) || (github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease == true)
      uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
      with:
        repository-url: https://test.pypi.org/legacy/
        skip-existing: true  # Don't fail if version already exists

    - name: Publish to PyPI
      if: (github.event_name == 'release' && github.event.release.prerelease != true) || (github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease != true)
      uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
      with:
        skip-existing: true  # Don't fail if version already exists

  # Verify the published package is installable (skipped for prereleases which go to Test PyPI)
  verify-publish:
    needs: publish
    if: ${{ !(github.event_name == 'release' && github.event.release.prerelease == true) && !(github.event_name == 'repository_dispatch' && github.event.client_payload.prerelease == true) }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@a5ad31d6a139d249332a2605b85202e8c0b78450 # v2.19.1
      with:
        egress-policy: audit

    - name: Determine published version
      id: version
      env:
        EVENT_NAME: ${{ github.event_name }}
        PAYLOAD_TAG: ${{ github.event.client_payload.tag }}
        RELEASE_TAG: ${{ github.event.release.tag_name }}
      run: |
        if [ "$EVENT_NAME" = "repository_dispatch" ]; then
          TAG="$PAYLOAD_TAG"
        else
          TAG="$RELEASE_TAG"
        fi
        # Strip leading 'v' if present
        VERSION="${TAG#v}"
        echo "version=$VERSION" >> "$GITHUB_OUTPUT"
        echo "Published version: $VERSION"

    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libsqlcipher-dev

    - name: Wait for PyPI index to update
      run: |
        echo "Waiting 60 seconds for PyPI index propagation..."
        sleep 60

    - name: Verify package installation
      run: |
        python3 -m venv /tmp/verify-env
        . /tmp/verify-env/bin/activate

        VERSION="${{ steps.version.outputs.version }}"
        echo "Installing local-deep-research==${VERSION}..."

        # Retry up to 5 times with increasing delay for PyPI propagation
        for i in 1 2 3 4 5; do
          if pip install "local-deep-research==${VERSION}" 2>&1; then
            echo "Installation successful on attempt $i"
            break
          fi
          if [ "$i" -lt 5 ]; then
            echo "Attempt $i failed, waiting $((i * 30))s..."
            sleep $((i * 30))
          else
            echo "::error::Failed to install local-deep-research==${VERSION} after 5 attempts"
            exit 1
          fi
        done

        # Verify import works
        python3 -c "import local_deep_research; print(f'Successfully imported v{local_deep_research.__version__}')"