diff --git a/.github/workflows/main-deploy-dev.yml b/.github/workflows/main-deploy-dev.yml index 4ef4872f..4c291fb7 100644 --- a/.github/workflows/main-deploy-dev.yml +++ b/.github/workflows/main-deploy-dev.yml @@ -12,7 +12,7 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/.github/workflows/main-deploy-docs.yml b/.github/workflows/main-deploy-docs.yml index a6c7caf7..ba755e1c 100644 --- a/.github/workflows/main-deploy-docs.yml +++ b/.github/workflows/main-deploy-docs.yml @@ -12,7 +12,7 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/.github/workflows/main-deploy.yml b/.github/workflows/main-deploy.yml index 8bec1c95..7b86ae03 100644 --- a/.github/workflows/main-deploy.yml +++ b/.github/workflows/main-deploy.yml @@ -12,7 +12,7 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/.github/workflows/main-publish.yml b/.github/workflows/main-publish.yml new file mode 100644 index 00000000..1c4e9c79 --- /dev/null +++ b/.github/workflows/main-publish.yml @@ -0,0 +1,41 @@ +# PyPi auto-publication support + +name: publish + +# Controls when the action will run. +on: + + # Publish on all tags + push: + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.11 + - name: Install Python dependencies for publish + run: python -m pip install dcicutils==8.7.0.1b2 + - name: Update the gitinfo.json file with latest relevant git info + run: | + echo "{\"repo\": \"https://github.com/${{ github.repository }}\", \"branch\": \"${GITHUB_REF##*/}\", \"commit\": \"${GITHUB_SHA}\"}" > chalicelib_fourfront/gitinfo.json + - name: Publish + env: + PYPI_USER: ${{ secrets.PYPI_USER }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + make configure + make publish-for-ga diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8301dead..59e9896d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,18 @@ foursight Change Log ---------- +4.4.0 +===== +* Added update of a gitinfo.json file in GitHub Actions (.github/workflows/main-publish.yml). +* Update foursight-core with fix to Portal Reindex page (to not show initial deploy), + and straighten out blue/green staging/data dichotomy on Reindex and Redeploy pages. + +4.3.0 +===== +* Fix wfr_checks.md5run_status for bug where it was missing the first item in the result + set because it was calling any() on a generator before iterating through it, which is + destructive of the generator, i.e. causing to to move one item forwared. + 4.2.2 ===== @@ -31,7 +43,6 @@ Change Log `PR 558: refactor doppelganger check `_ - 4.1.4 ===== diff --git a/Makefile b/Makefile index ef30f9a9..76b79247 100644 --- a/Makefile +++ b/Makefile @@ -33,4 +33,5 @@ publish: publish-for-ga: # New Python based publish script in dcicutils (2023-04-25). + pip install dcicutils==8.7.0.1b2 poetry run publish-to-pypi --noconfirm diff --git a/chalicelib_fourfront/checks/wfr_checks.py b/chalicelib_fourfront/checks/wfr_checks.py index 4cbde4d8..f3830ac5 100644 --- a/chalicelib_fourfront/checks/wfr_checks.py +++ b/chalicelib_fourfront/checks/wfr_checks.py @@ -135,9 +135,11 @@ def md5run_status(connection, **kwargs): query += '&limit=' + limit # The search res = ff_utils.search_metadata(query, key=my_auth, is_generator=True) - if not any(res): - check.summary = 'All Good!' - return check + # 2024-02-08: Note that any() is destructive of the (res) generator; + # so this code was causing us to skip past the first result in the generator. + # if not any(res): + # check.summary = 'All Good!' + # return check # if there are files, make sure they are not on s3 no_s3_file = [] running = [] @@ -147,10 +149,19 @@ def md5run_status(connection, **kwargs): not_switched_status_to_wait = [] # multiple failed runs problems = [] - my_s3_util = s3Utils(env=connection.ff_env) - raw_bucket = my_s3_util.raw_file_bucket - out_bucket = my_s3_util.outfile_bucket + # 2024-02-08: Per above comment WRT any() on a generator, only do + # this S3 access only if we have at least one result in the loop below. + # Be nicer to have a generator wrapper to allow easily + # checking for any results without skipping past the first item. + # my_s3_util = s3Utils(env=connection.ff_env) + # raw_bucket = my_s3_util.raw_file_bucket + # out_bucket = my_s3_util.outfile_bucket + my_s3_util = None for a_file in res: + if not my_s3_util: + my_s3_util = s3Utils(env=connection.ff_env) + raw_bucket = my_s3_util.raw_file_bucket + out_bucket = my_s3_util.outfile_bucket # lambda has a time limit (300sec), kill before it is reached so we get some results now = datetime.utcnow() if (now-start).seconds > lambda_limit: @@ -190,6 +201,13 @@ def md5run_status(connection, **kwargs): not_switched_status.append(file_id) else: not_switched_status_to_wait.append(file_id) + else: + # We get here at the end of the result set iteration for loop, regardless of where or not + # there were results; if my_s3_util is not set at this point it means there were no results. + if not my_s3_util: + check.summary = 'All Good!' + return check + summary = '' if running: summary += 'Some files are running md5run\n' diff --git a/chalicelib_fourfront/gitinfo.json b/chalicelib_fourfront/gitinfo.json new file mode 100644 index 00000000..0ea71f63 --- /dev/null +++ b/chalicelib_fourfront/gitinfo.json @@ -0,0 +1 @@ +{"Note": "Populated by GitHub Actions with git info; please do not edit."} diff --git a/poetry.lock b/poetry.lock index af2ded1e..bd394c18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -569,13 +569,13 @@ files = [ [[package]] name = "foursight-core" -version = "5.2.0" +version = "5.3.0" description = "Serverless Chalice Application for Monitoring" optional = false python-versions = ">=3.8,<3.12" files = [ - {file = "foursight_core-5.2.0-py3-none-any.whl", hash = "sha256:1c0735b3d67267bc784abf923fa68d8eb7fccf8b91c2e500f131a7a09617d502"}, - {file = "foursight_core-5.2.0.tar.gz", hash = "sha256:79c4bfd3e910f7896d3072231109f05d2b427a0f6f7213c78af6cc651feef8e9"}, + {file = "foursight_core-5.3.0-py3-none-any.whl", hash = "sha256:1ae66657263b3b89726a71edb0d86ce784a5cc784cfad3f668a0de0c00f54db9"}, + {file = "foursight_core-5.3.0.tar.gz", hash = "sha256:50873d9de3346dfc0cc67b6a562197cc8532dfa03ac14e81609662f88b25a5e6"}, ] [package.dependencies] @@ -584,7 +584,7 @@ botocore = ">=1.31.62,<2.0.0" click = ">=7.1.2,<8.0.0" cron-descriptor = ">=1.2.31,<2.0.0" cryptography = "39.0.2" -dcicutils = ">=8.0.0,<9.0.0" +dcicutils = "8.0.0" elasticsearch = "7.13.4" elasticsearch-dsl = ">=7.0.0,<8.0.0" geocoder = "1.38.1" @@ -1755,19 +1755,19 @@ tomlkit = ">=0.11.0,<0.12.0" [[package]] name = "tibanna-ff" -version = "3.1.1" +version = "3.2.0.2b1" description = "Tibanna runs portable pipelines (in CWL/WDL) on the AWS Cloud." optional = false python-versions = ">=3.8,<3.12" files = [ - {file = "tibanna_ff-3.1.1-py3-none-any.whl", hash = "sha256:fac37cc31318dcc9f0fa6ce9dd9b1ce6347452c704f499939d054d51cab167b4"}, - {file = "tibanna_ff-3.1.1.tar.gz", hash = "sha256:7801dface3e60ef6b867b8cda875238cf511bce3938d2f52f28ccb77a83c6e17"}, + {file = "tibanna_ff-3.2.0.2b1-py3-none-any.whl", hash = "sha256:83837cb857c1bf62644f526a42b6eccf06b4935f37007469396370a5e585a74d"}, + {file = "tibanna_ff-3.2.0.2b1.tar.gz", hash = "sha256:ebd2233485cd20c8d69864fe340bb464bbb08ed85d0aa7fa9276f707a6850d77"}, ] [package.dependencies] boto3 = ">=1.28.56,<2.0.0" botocore = ">=1.31.56,<2.0.0" -dcicutils = "8.0.0" +dcicutils = ">=8.0.0,<9.0.0" requests = "2.27.1" tibanna = "5.1.0" tomlkit = ">=0.11.0,<0.12.0" @@ -1987,4 +1987,4 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "122c7ed73eaa88e4f076aa94d22a62cbf97d33d8385431b917c772a0cdbc6c55" +content-hash = "93c7c3088f12145d3d4c5946fe56bedbddf5868379ff6e7dd0f8a15699f49e73" diff --git a/pyproject.toml b/pyproject.toml index 2f2698e2..063139b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "foursight" -version = "4.2.2" +version = "4.4.0" description = "Serverless Chalice Application for Monitoring" authors = ["4DN-DCIC Team "] license = "MIT" @@ -24,10 +24,10 @@ elasticsearch = "7.13.4" elasticsearch-dsl = "^7.0.0" gitpython = "^3.1.2" pytz = "^2020.1" -tibanna-ff = "^3.1.1" +tibanna-ff = "^3.2.0" ## adding foursight-core # use below for deployment -foursight-core = "5.2.0" +foursight-core = "^5.3.0" # use below for tests but not for deployment # foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="master" } # Need pytest-redis 3.0.2 or higher for pytest 7.4.2 (or higher).