Skip to content

Commit

Permalink
ci/hotfix: legacy workflow (Lightning-AI#16721)
Browse files Browse the repository at this point in the history
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
Borda and pre-commit-ci[bot] authored Feb 13, 2023
1 parent 2e002d4 commit c7ffd41
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 15 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@

## Deployment

| workflow file | action |
| ---------------------------------------- | ---------------------------------------------------------------------------------- |
| .github/workflows/release-pypi.yml | Publish a release to PyPI. |
| .github/workflows/release-docker.yml | Build Docker images from dockers/\*/Dockerfile and publish them on hub.docker.com. |
| .github/workflows/legacy-checkpoints.yml | App on request generate legacy checkpoints and upload them to AWS S3. |
| workflow file | action |
| ------------------------------------------ | ---------------------------------------------------------------------------------- |
| .github/workflows/release-pypi.yml | Publish a release to PyPI. |
| .github/workflows/release-docker.yml | Build Docker images from dockers/\*/Dockerfile and publish them on hub.docker.com. |
| .github/workflows/\_legacy-checkpoints.yml | App on request generate legacy checkpoints and upload them to AWS S3. |

## Bots

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ on:
default: false
required: false
type: boolean
secrets: inherit

env:
legacy_dir: tests/legacy

defaults:
run:
Expand All @@ -46,9 +48,6 @@ jobs:
runs-on: ubuntu-20.04
outputs:
pl-version: ${{ steps.decide-version.outputs.pl-version }}
defaults:
run:
working-directory: tests/legacy
steps:
- uses: actions/checkout@v3

Expand All @@ -65,7 +64,6 @@ jobs:
aws-region: us-east-1

- name: Install PL from source
working-directory: ./
env:
PACKAGE_NAME: pytorch
FREEZE_REQUIREMENTS: 1
Expand All @@ -81,8 +79,15 @@ jobs:
pip list
if: inputs.pl_version != ''

- name: Adjust tests -> PL
if: ${{ matrix.pkg-name != 'lightning' }}
run: |
pip install -q -r .actions/requirements.txt
python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
--source_import="lightning.fabric,lightning.pytorch" \
--target_import="lightning_fabric,pytorch_lightning"
- name: Pull legacy checkpoints
working-directory: ./
run: bash .actions/pull_legacy_checkpoints.sh

- name: Decide PL version to create a PR with
Expand All @@ -91,29 +96,32 @@ jobs:
python -c "import pytorch_lightning as pl; print(f'pl-version={pl.__version__}')" >> $GITHUB_OUTPUT || echo pl-version='' >> $GITHUB_OUTPUT
- name: Generate checkpoints
working-directory: ${{ env.legacy_dir }}
run: |
bash generate_checkpoints.sh ${{ inputs.pl_version }}
- name: Upload checkpoints to GitHub Actions artifact
uses: actions/upload-artifact@v3
with:
name: checkpoints-${{ github.sha }}
path: tests/legacy/checkpoints/
path: ${{ env.legacy_dir }}/checkpoints/

- name: Upload checkpoints to S3 (dryrun)
working-directory: ${{ env.legacy_dir }}
run: |
aws s3 sync --dryrun checkpoints/ s3://pl-public-data/legacy/checkpoints/
zip -r checkpoints.zip checkpoints
aws s3 cp --dryrun checkpoints.zip s3://pl-public-data/legacy/ --acl public-read
- name: Upload checkpoints to S3
working-directory: ${{ env.legacy_dir }}
run: |
aws s3 sync checkpoints/ s3://pl-public-data/legacy/checkpoints/
zip -r checkpoints.zip checkpoints
aws s3 cp checkpoints.zip s3://pl-public-data/legacy/ --acl public-read
if: inputs.push_to_s3

adding-ckpt-test:
add-ckpt-test:
runs-on: ubuntu-20.04
if: inputs.create_pr
needs: create-legacy-ckpts
Expand All @@ -123,7 +131,7 @@ jobs:
ref: master

- name: Append a new version to legacy checkpoint list
run: echo ${{ needs.create-legacy-ckpts.outputs.pl-version }} >> tests/legacy/back-compatible-versions.txt
run: echo ${{ needs.create-legacy-ckpts.outputs.pl-version }} >> ${{ env.legacy_dir }}/back-compatible-versions.txt

- name: Create Pull Request
uses: peter-evans/create-pull-request@v4
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/ci-checkpoints.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: Validate legacy ckpts

on:
push:
branches: [master, "release/*"]
pull_request:
branches: [master, "release/*"]
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
paths:
- ".github/workflows/ci-checkpoints.yml"
- ".github/workflows/_legacy-checkpoints.yml.yml"

jobs:

try-legacy-checkpoints:
uses: ./.github/workflows/_legacy-checkpoints.yml
with:
push_to_s3: false
create_pr: false
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/release-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ jobs:

legacy-checkpoints:
needs: [build-packages]
uses: ./.github/workflows/legacy-checkpoints.yml
uses: ./.github/workflows/_legacy-checkpoints.yml
with:
push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
Expand Down

0 comments on commit c7ffd41

Please sign in to comment.