Merge pull request #470 from epam/develop

Develop

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-# [1.16.0] - 2025-01-10
+# [1.16.0] - 2025-01-14
 - Added support for the AppSync resource
 - Added the possibility to generate `s3_bucket` meta for static website hosting without public access
 - Added the possibility to manage CloudWatch logging for API Gateway resources
@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed duplication lambda function and lambda layer records in the output file after updating the resources
 - Fixed issue related to lambda function updating in case of changing lambda's alias name
 - Fixed issue related to tests generation for lambda function with runtime Python
+- Fixed duplication of the API Gateway of type `web_socket_api_gateway` during deployment
 - Fix `tag_resources` and `untag_resources` to handle exceptions properly
 - Update `apply_tags`, `remove_tags`, and `update_tags` to return success status
 - Fix `clean` command for `output` folder to correctly resolve ARN for Lambda and properly process and remove the `outputs` folder if Lambda is part of the deployment
@@ -35,12 +36,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add resource tags for `ec2 launch-template` and  for versions in update and deploy operations
 - Add support tags for `OAS V3` as part of the `oas_v3.json` file, using the `x-syndicate-openapi-tags` key
 - Remove the directory 'target' that contains temporary files after assembling lambda artifacts
+- Unused library `colorama` removed from the requirements
 - Correct the return code of all the commands if they failed or were aborted
 - Java plugin version updated to 1.15.0 with changes:
   - Added the verification of Lambda resources existence to prevent deploy without Lambda resources
   - Added support of the syndicate `--errors_allowed` flag to skip build process interruption in case of errors
 - Java template updated to use the Syndicate Java plugin version 1.15.0 for the new lambda generation
 - Java examples updated to use the Syndicate Java plugin version 1.15.0
+- Fix bug when assembling multiple lambdas with different runtimes with `--force_bundle` flag
 
 # [1.15.0] - 2024-10-28
 - Added `--skip_tests` option to `build`, `test` and `assemble_java_mvn` commands to not run tests during or after 

requirements.txt

@@ -2,7 +2,6 @@ click==7.1.2
 botocore==1.29.80
 boto3==1.26.80
 tqdm==4.65.2
-colorama==0.4.5
 requests==2.31.0
 tabulate==0.9.0
 troposphere==4.1.0

setup.py

@@ -35,7 +35,6 @@
         'botocore==1.29.80',
         'boto3==1.26.80',
         'tqdm==4.65.2',
-        'colorama==0.4.5',
         'pyyaml==6.0.1',
         'requests-aws-sign==0.1.6',
         'requests==2.31.0',

syndicate/core/build/artifact_processor.py

@@ -53,25 +53,15 @@
 _LOG = get_logger(__name__)
 
 
-def assemble_artifacts(bundle_name, project_path, runtime, force_upload=None,
-                       errors_allowed=False, skip_tests=False):
+def assemble_artifacts(bundle_name, project_path, runtime,
+                       errors_allowed=False, skip_tests=False, **kwargs):
     if runtime not in SUPPORTED_RUNTIMES:
         raise AssertionError(
             f'Runtime {runtime} is not supported. '
             f'Currently available runtimes:{SUPPORTED_RUNTIMES}')
 
     bundle_dir = resolve_bundle_directory(bundle_name=bundle_name)
 
-    if force_upload is True:
-        _LOG.warning(f"Force upload is True, going to check if bundle"
-                     f" directory already exists.")
-        normalized_bundle_dir = os.path.normpath(bundle_dir)
-        if os.path.exists(normalized_bundle_dir):
-            _LOG.warning(f"Bundle with name: {bundle_name} already exists by"
-                         f" path `{normalized_bundle_dir}`, going to remove"
-                         f" this bundle locally.")
-            shutil.rmtree(normalized_bundle_dir)
-
     os.makedirs(bundle_dir, exist_ok=True)
     _LOG.debug(f'Target directory: {bundle_dir}')
 

syndicate/core/build/bundle_processor.py

@@ -15,6 +15,7 @@
 """
 import json
 import os
+import shutil
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import PurePath
 from botocore.exceptions import ClientError
@@ -333,3 +334,16 @@ def _put_package_to_s3(path, path_to_package):
                             path).as_posix()
     CONN.s3().upload_single_file(path_to_package, key_compound,
                                  CONFIG.deploy_target_bucket)
+
+
+def remove_bundle_dir_locally(bundle_name: str):
+    bundle_dir = resolve_bundle_directory(bundle_name=bundle_name)
+    normalized_bundle_dir = os.path.normpath(bundle_dir)
+    if os.path.exists(normalized_bundle_dir):
+        _LOG.warning(f'Going to remove bundle folder '
+                     f'`{normalized_bundle_dir}` locally.')
+        try:
+            shutil.rmtree(normalized_bundle_dir)
+        except Exception as e:
+            _LOG.error(f'Cannot delete folder {normalized_bundle_dir}')
+            raise e

syndicate/core/handlers.py

@@ -29,10 +29,9 @@
 from syndicate.core.build.artifact_processor import RUNTIME_NODEJS, \
     assemble_artifacts, RUNTIME_JAVA, RUNTIME_PYTHON, RUNTIME_SWAGGER_UI, \
     RUNTIME_DOTNET, RUNTIME_APPSYNC
-from syndicate.core.build.bundle_processor import (create_bundles_bucket,
-                                                   load_bundle,
-                                                   upload_bundle_to_s3,
-                                                   if_bundle_exist)
+from syndicate.core.build.bundle_processor import create_bundles_bucket, \
+    load_bundle, upload_bundle_to_s3, if_bundle_exist, \
+    remove_bundle_dir_locally
 from syndicate.core.build.deployment_processor import \
     create_deployment_resources, remove_deployment_resources, \
     update_deployment_resources
@@ -717,10 +716,14 @@ def assemble_java_mvn(bundle_name, project_path, force_upload, skip_tests,
     :return:
     """
     USER_LOG.info(f'Command compile java project path: {project_path}')
+    if force_upload:
+        _LOG.info(f'Force upload is enabled, going to check if bundle '
+                  f'directory already exists locally.')
+        remove_bundle_dir_locally(bundle_name)
+
     assemble_artifacts(bundle_name=bundle_name,
                        project_path=project_path,
                        runtime=RUNTIME_JAVA,
-                       force_upload=force_upload,
                        skip_tests=skip_tests,
                        errors_allowed=errors_allowed)
     USER_LOG.info('Java artifacts were prepared successfully.')
@@ -767,10 +770,14 @@ def assemble_python(bundle_name, project_path, force_upload, errors_allowed,
     :return:
     """
     USER_LOG.info(f'Command assemble python: project_path: {project_path} ')
+    if force_upload:
+        _LOG.info(f'Force upload is enabled, going to check if bundle '
+                  f'directory already exists locally.')
+        remove_bundle_dir_locally(bundle_name)
+
     assemble_artifacts(bundle_name=bundle_name,
                        project_path=project_path,
                        runtime=RUNTIME_PYTHON,
-                       force_upload=force_upload,
                        errors_allowed=errors_allowed)
     USER_LOG.info('Python artifacts were prepared successfully.')
     return OK_RETURN_CODE
@@ -812,11 +819,14 @@ def assemble_node(bundle_name, project_path, force_upload,
     :return:
     """
     USER_LOG.info(f'Command assemble node: project_path: {project_path} ')
+    if force_upload:
+        _LOG.info(f'Force upload is enabled, going to check if bundle '
+                  f'directory already exists locally.')
+        remove_bundle_dir_locally(bundle_name)
+
     assemble_artifacts(bundle_name=bundle_name,
                        project_path=project_path,
-                       runtime=RUNTIME_NODEJS,
-                       force_upload=force_upload
-                       )
+                       runtime=RUNTIME_NODEJS)
     USER_LOG.info('NodeJS artifacts were prepared successfully.')
     return OK_RETURN_CODE
 
@@ -857,11 +867,14 @@ def assemble_dotnet(bundle_name, project_path, force_upload,
     :return:
     """
     USER_LOG.info(f'Command assemble dotnet: project_path: {project_path} ')
+    if force_upload:
+        _LOG.info(f'Force upload is enabled, going to check if bundle '
+                  f'directory already exists locally.')
+        remove_bundle_dir_locally(bundle_name)
+
     assemble_artifacts(bundle_name=bundle_name,
                        project_path=project_path,
-                       runtime=RUNTIME_DOTNET,
-                       force_upload=force_upload
-                       )
+                       runtime=RUNTIME_DOTNET)
     USER_LOG.info('DotNet artifacts were prepared successfully.')
     return OK_RETURN_CODE
 
@@ -972,6 +985,11 @@ def assemble(ctx, bundle_name, force_upload, errors_allowed, skip_tests=False):
     :param skip_tests: allows to skip tests
     :return:
     """
+    if force_upload:
+        _LOG.info(f'Force upload is enabled, going to check if bundle '
+                  f'directory already exists locally.')
+        remove_bundle_dir_locally(bundle_name)
+
     USER_LOG.info(f'Building artifacts, bundle: {bundle_name}')
     from syndicate.core import PROJECT_STATE
     # Updates stale lambda state aggregation
@@ -984,7 +1002,7 @@ def assemble(ctx, bundle_name, force_upload, errors_allowed, skip_tests=False):
             if func:
                 return_code = ctx.invoke(func, bundle_name=bundle_name,
                                          project_path=value,
-                                         force_upload=force_upload,
+                                         force_upload=False, # because we have already deleted the bundle folder
                                          errors_allowed=errors_allowed,
                                          skip_tests=skip_tests)
                 if return_code != OK_RETURN_CODE:

syndicate/core/resources/api_gateway_resource.py

@@ -1144,6 +1144,10 @@ def _create_web_socket_api_from_meta(self, name: str, meta: dict):
         stage_name = meta.get('deploy_stage')
         resources = meta.get('resources') or {}
         route_selection_expression = meta.get('route_selection_expression')
+        api_gw_describe = self.describe_v2_api_gateway(name, meta)
+        if api_gw_describe:
+            _LOG.info(f'Api gateway with name \'{name}\' exists. Returning')
+            return api_gw_describe
         api_id = self.apigw_v2.create_web_socket_api(
             name=name, route_selection_expression=route_selection_expression,
             tags=meta.get('tags'))

syndicate/core/resources/cloud_watch_alarm_resource.py

@@ -119,3 +119,5 @@ def _remove_alarms(self, arn, config):
                 _LOG.warn(f'Alarm {alarm_name} was found despite the '
                           f'`ResourceNotFoundException` error.')
                 raise e
+
+        return {arn: config}

syndicate/core/resources/dynamo_db_resource.py

@@ -392,6 +392,8 @@ def _remove_tables_from_meta(self, arn, config):
         removed_tables, errors = self.dynamodb_conn.remove_tables_by_names(
             [db_name],
             log_not_found_error=False)
+        if errors:
+            raise Exception('; '.join(errors))
         _LOG.info(f'Dynamo DB tables {str(removed_tables)} were removed')
 
         alarm_args = []
@@ -407,3 +409,5 @@ def _remove_tables_from_meta(self, arn, config):
                 self.cw_alarm_conn.remove_alarms(alarm_args)
         except Exception as e:
             raise e
+
+        return {arn: config}

tests/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2025-01-15
+### Added
+- Added appsync tests for build, deploy, update and clean commands
+
 ## 2024-12-05
 ### Changed
 - Changed the way to pass lambda alias to checkers - use read_sdct_conf() function from util file instead pass it with lambda name

tests/smoke/README.md

@@ -241,6 +241,38 @@ not matter and only the presence of a particular key will be checked.
           }
         }
         ```
+- `appsync_modification` - Checks appsync resource configuration. 
+  - parameters:
+    - `resources` (dict) [REQUIRED] - Appsync configuration to check. Should include `resolvers` and `data_sources`.
+`functions` parameter is optional.
+        structure:
+        ```json5
+          {
+              "resources": {
+                "appsync_name": {
+                  "data_sources": [
+                    {
+                      "name": "table",
+                      "type": "AMAZON_DYNAMODB"
+                    }
+                  ],
+                  "resolvers": [
+                    {
+                      "type_name": "Post",
+                      "field_name": "id",
+                      "data_source_name": "table"
+                    }
+                  ],
+                  "functions": [
+                    {
+                      "name": "get_user",
+                      "data_source_name": "table"
+                    }
+                  ]
+                }
+              }
+          }
+        ```
 
 ### Temporary checks conditions
 - Use tags from `tests/smoke/sdct-auto-test/.syndicate-config/syndicate.yml` unless change them in happy_path_config.json: