Merge pull request #557 from 4dn-dcic/dmichaels-20240103

Support for gitinfo.json

.github/workflows/main-deploy-dev.yml

@@ -12,7 +12,7 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/main-deploy-docs.yml

@@ -12,7 +12,7 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/main-deploy.yml

@@ -12,7 +12,7 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/main-publish.yml

@@ -0,0 +1,41 @@
+# PyPi auto-publication support
+
+name: publish 
+
+# Controls when the action will run. 
+on:
+
+  # Publish on all tags
+  push:
+    tags:
+    - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-22.04
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+      - name: Install Python dependencies for publish
+        run: python -m pip install dcicutils==8.7.0.1b2
+      - name: Update the gitinfo.json file with latest relevant git info
+        run: |
+          echo "{\"repo\": \"https://github.com/${{ github.repository }}\", \"branch\": \"${GITHUB_REF##*/}\", \"commit\": \"${GITHUB_SHA}\"}" > chalicelib_fourfront/gitinfo.json
+      - name: Publish
+        env:
+          PYPI_USER: ${{ secrets.PYPI_USER }}
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: | 
+          make configure
+          make publish-for-ga

CHANGELOG.rst

@@ -7,6 +7,18 @@ foursight
 Change Log
 ----------
 
+4.4.0
+=====
+* Added update of a gitinfo.json file in GitHub Actions (.github/workflows/main-publish.yml).
+* Update foursight-core with fix to Portal Reindex page (to not show initial deploy),
+  and straighten out blue/green staging/data dichotomy on Reindex and Redeploy pages. 
+
+4.3.0
+=====
+* Fix wfr_checks.md5run_status for bug where it was missing the first item in the result
+  set because it was calling any() on a generator before iterating through it, which is
+  destructive of the generator, i.e. causing to to move one item forwared.
+
 4.2.2
 =====
 
@@ -31,7 +43,6 @@ Change Log
 
 `PR 558: refactor doppelganger check <https://github.com/4dn-dcic/foursight/pull/558>`_
 
-
 4.1.4
 =====
 

Makefile

@@ -33,4 +33,5 @@ publish:
 
 publish-for-ga:
 	# New Python based publish script in dcicutils (2023-04-25).
+	pip install dcicutils==8.7.0.1b2
 	poetry run publish-to-pypi --noconfirm

chalicelib_fourfront/checks/wfr_checks.py

@@ -135,9 +135,11 @@ def md5run_status(connection, **kwargs):
     query += '&limit=' + limit
     # The search
     res = ff_utils.search_metadata(query, key=my_auth, is_generator=True)
-    if not any(res):
-        check.summary = 'All Good!'
-        return check
+    # 2024-02-08: Note that any() is destructive of the (res) generator;
+    # so this code was causing us to skip past the first result in the generator.
+    # if not any(res):
+    #    check.summary = 'All Good!'
+    #    return check
     # if there are files, make sure they are not on s3
     no_s3_file = []
     running = []
@@ -147,10 +149,19 @@ def md5run_status(connection, **kwargs):
     not_switched_status_to_wait = []
     # multiple failed runs
     problems = []
-    my_s3_util = s3Utils(env=connection.ff_env)
-    raw_bucket = my_s3_util.raw_file_bucket
-    out_bucket = my_s3_util.outfile_bucket
+    # 2024-02-08: Per above comment WRT any() on a generator, only do
+    # this S3 access only if we have at least one result in the loop below.
+    # Be nicer to have a generator wrapper to allow easily
+    # checking for any results without skipping past the first item.
+    # my_s3_util = s3Utils(env=connection.ff_env)
+    # raw_bucket = my_s3_util.raw_file_bucket
+    # out_bucket = my_s3_util.outfile_bucket
+    my_s3_util = None
     for a_file in res:
+        if not my_s3_util:
+            my_s3_util = s3Utils(env=connection.ff_env)
+            raw_bucket = my_s3_util.raw_file_bucket
+            out_bucket = my_s3_util.outfile_bucket
         # lambda has a time limit (300sec), kill before it is reached so we get some results
         now = datetime.utcnow()
         if (now-start).seconds > lambda_limit:
@@ -190,6 +201,13 @@ def md5run_status(connection, **kwargs):
                 not_switched_status.append(file_id)
             else:
                 not_switched_status_to_wait.append(file_id)
+    else:
+        # We get here at the end of the result set iteration for loop, regardless of where or not
+        # there were results; if my_s3_util is not set at this point it means there were no results.
+        if not my_s3_util:
+            check.summary = 'All Good!'
+            return check
+
     summary = ''
     if running:
         summary += 'Some files are running md5run\n'

chalicelib_fourfront/gitinfo.json

@@ -0,0 +1 @@
+{"Note": "Populated by GitHub Actions with git info; please do not edit."}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "foursight"
-version = "4.2.2"
+version = "4.4.0"
 description = "Serverless Chalice Application for Monitoring"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -24,10 +24,10 @@ elasticsearch = "7.13.4"
 elasticsearch-dsl = "^7.0.0"
 gitpython = "^3.1.2"
 pytz = "^2020.1"
-tibanna-ff = "^3.1.1"
+tibanna-ff = "^3.2.0"
 ## adding foursight-core
 # use below for deployment
-foursight-core = "5.2.0"
+foursight-core = "^5.3.0"
 # use below for tests but not for deployment
 # foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="master" }
 # Need pytest-redis 3.0.2 or higher for pytest 7.4.2 (or higher).