fix(cts): split requests and e2e tests

generators/src/main/java/com/algolia/codegen/cts/manager/PythonCTSManager.java

@@ -17,6 +17,7 @@ public PythonCTSManager(String client) {
   public void addTestsSupportingFiles(List<SupportingFile> supportingFiles) {
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/requests", "__init__.py"));
+    supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/requests_e2e", "__init__.py"));
     supportingFiles.add(new SupportingFile("tests/__init__.mustache", "tests/output/python/tests/client", "__init__.py"));
   }
 

generators/src/main/java/com/algolia/codegen/cts/tests/TestsRequest.java

@@ -15,6 +15,7 @@
 public class TestsRequest extends TestsGenerator {
 
   private final boolean withSnippets;
+  private List<SupportingFile> supportingFiles;
 
   public TestsRequest(String language, String client, boolean withSnippets) {
     super(language, client);
@@ -64,13 +65,39 @@ public void addSupportingFiles(List<SupportingFile> supportingFiles, String outp
     if (!available()) {
       return;
     }
+    this.supportingFiles = supportingFiles;
+
     supportingFiles.add(
       new SupportingFile(
         "tests/requests/requests.mustache",
         "tests/output/" + language + "/" + outputFolder + "/requests",
         Helpers.createClientName(client, language) + extension
       )
     );
+    if (new File("templates/" + language + "/tests/requests/requests_e2e.mustache").exists()) {
+      supportingFiles.add(
+        new SupportingFile(
+          "tests/requests/requests_e2e.mustache",
+          "tests/output/" + language + "/" + outputFolder + "/requests_e2e",
+          Helpers.createClientName(client, language) + extension
+        )
+      );
+    }
+  }
+
+  private String escapeBody(String body) {
+    if (body == null) {
+      return null;
+    }
+
+    switch (language) {
+      case "go": // jsonassert expect % to be formatted, we need to escape them
+        return body.replace("%", "%%");
+      case "dart": // Same thing but for $
+        return body.replace("$", "\\$");
+      default:
+        return body;
+    }
   }
 
   @Override
@@ -82,6 +109,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
     }
 
     List<Object> blocks = new ArrayList<>();
+    List<Object> blocksE2E = new ArrayList<>();
     ParametersWithDataType paramsType = new ParametersWithDataType(models, language, client);
 
     bundle.put("e2eApiKey", client.equals("monitoring") ? "MONITORING_API_KEY" : "ALGOLIA_ADMIN_KEY");
@@ -138,15 +166,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
               req.request.body = "{}";
             }
 
-            // For golang, jsonassert expect % to be formatted, we need to escape them
-            if (language.equals("go") && req.request.body != null) {
-              req.request.body = req.request.body.replace("%", "%%");
-            }
-
-            // For dart, same thing but for $
-            if (language.equals("dart") && req.request.body != null) {
-              req.request.body = req.request.body.replace("$", "\\$");
-            }
+            req.request.body = escapeBody(req.request.body);
 
             // In a case of a `GET` or `DELETE` request, we want to assert if the body
             // is correctly parsed (absent from the payload)
@@ -156,7 +176,7 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
           }
 
           if (req.response != null) {
-            bundle.put("hasE2E", true);
+            req.response.body = escapeBody(req.response.body);
             test.put("response", req.response);
           }
 
@@ -219,7 +239,21 @@ public void run(Map<String, CodegenModel> models, Map<String, CodegenOperation>
       }
 
       blocks.add(testObj);
+
+      // extract e2e
+      List<Map<String, Object>> e2e = tests.stream().filter(t -> t.get("response") != null).toList();
+      if (e2e.size() > 0) {
+        Map<String, Object> e2eObj = new HashMap<>();
+        e2eObj.put("tests", e2e);
+        e2eObj.put("operationId", operationId);
+        blocksE2E.add(e2eObj);
+      }
     }
     bundle.put("blocksRequests", blocks);
+    if (!blocksE2E.isEmpty()) {
+      bundle.put("blocksRequestsE2E", blocksE2E);
+    } else {
+      supportingFiles.removeIf(f -> f.getTemplateFile().equals("tests/requests/requests_e2e.mustache"));
+    }
   }
 }

scripts/ci/githubActions/createMatrix.ts

@@ -69,15 +69,15 @@ async function createClientMatrix(baseBranch: string): Promise<void> {
     const testsRootFolder = `tests/output/${language}`;
     const testsOutputBase = `${testsRootFolder}/${getTestOutputFolder(language)}`;
     // We delete tests to ensure the CI only run tests against what changed.
-    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests`;
+    const testsToDelete = `${testsOutputBase}/client ${testsOutputBase}/requests ${testsOutputBase}/requests_e2e`;
 
     // We only store tests of clients that ran during this job, the rest stay as is
     let testsToStore = matrix[language].toRun
       .map((client) => {
         const clientName = createClientName(client, language);
         const extension = getTestExtension(language);
 
-        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension}`;
+        return `${testsOutputBase}/client/${clientName}${extension} ${testsOutputBase}/requests/${clientName}${extension} ${testsOutputBase}/requests_e2e/${clientName}${extension}`;
       })
       .join(' ');
 

scripts/cts/runCts.ts

@@ -36,9 +36,12 @@ async function runCtsOne(language: string): Promise<void> {
       break;
     case 'php':
       await runComposerInstall();
-      await run(`php ./clients/algoliasearch-client-php/vendor/bin/phpunit ${cwd}`, {
-        language,
-      });
+      await run(
+        `php ./clients/algoliasearch-client-php/vendor/bin/phpunit --testdox --fail-on-warning ${cwd}`,
+        {
+          language,
+        },
+      );
       break;
     case 'python':
       await run('poetry lock --no-update && poetry install --sync && poetry run pytest -vv', {

scripts/husky/pre-commit.mjs

@@ -22,6 +22,7 @@ export function getPatterns() {
   for (const [language, { tests }] of Object.entries(clientConfig)) {
     entries.unshift(`tests/output/${language}/${tests.outputFolder}/client/**`);
     entries.unshift(`tests/output/${language}/${tests.outputFolder}/requests/**`);
+    entries.unshift(`tests/output/${language}/${tests.outputFolder}/requests_e2e/**`);
   }
   return entries;
 }

templates/csharp/tests/requests/requests.mustache

@@ -12,31 +12,13 @@ using Action = Algolia.Search.Models.Search.Action;
 
 public class {{client}}RequestTests
 {
-private readonly {{client}} _client{{#hasE2E}}, _e2eClient{{/hasE2E}};
+private readonly {{client}} _client;
   private readonly EchoHttpRequester _echo;
 
   public {{client}}RequestTests()
   {
     _echo = new EchoHttpRequester();
     _client = new {{client}}(new {{clientPrefix}}Config("appId", "apiKey"{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}), _echo);
-
-    {{#hasE2E}}
-      DotEnv.Load(options: new DotEnvOptions(ignoreExceptions: true, probeForEnv: true, probeLevelsToSearch: 8, envFilePaths: new[] { ".env" }));
-
-      var e2EAppId = Environment.GetEnvironmentVariable("ALGOLIA_APPLICATION_ID");
-      if (e2EAppId == null)
-      {
-        throw new Exception("please provide an `ALGOLIA_APPLICATION_ID` env var for e2e tests");
-      }
-
-      var e2EApiKey = Environment.GetEnvironmentVariable("{{e2eApiKey}}");
-      if (e2EApiKey == null)
-      {
-        throw new Exception("please provide an `{{e2eApiKey}}` env var for e2e tests");
-      }
-
-      _e2eClient = new {{client}}(new {{clientPrefix}}Config(e2EAppId, e2EApiKey{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}));
-    {{/hasE2E}}
   }
 
   [Fact]
@@ -99,32 +81,6 @@ private readonly {{client}} _client{{#hasE2E}}, _e2eClient{{/hasE2E}};
         }
     {{/headers}}
     {{/request}}
-
-
-    {{#response}}
-      // e2e
-      try {
-        var resp = await _e2eClient.{{#lambda.pascalcase}}{{method}}{{/lambda.pascalcase}}Async{{#isGeneric}}<Hit>{{/isGeneric}}({{#parametersWithDataType}}{{> tests/generateParams}}{{^-last}},{{/-last}}{{/parametersWithDataType}}{{#hasRequestOptions}}, new RequestOptions(){
-          {{#requestOptions.queryParameters}}
-            QueryParameters = new Dictionary<string, object>(){ {{#parametersWithDataType}} {"{{{key}}}", {{> tests/requests/requestOptionsParams}} } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
-          {{/requestOptions.queryParameters}}
-          {{#requestOptions.headers}}
-            Headers = new Dictionary<string, string>(){ {{#parametersWithDataType}} {"{{{key}}}", "{{{value}}}" } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
-          {{/requestOptions.headers}}
-        }{{/hasRequestOptions}});
-      {{#statusCode}}
-          // Check status code {{statusCode}}
-          Assert.NotNull(resp);
-      {{/statusCode}}
-
-      {{#body}}
-          JsonAssert.EqualOverrideDefault("{{#lambda.escapeQuotes}}{{{.}}}{{/lambda.escapeQuotes}}", JsonSerializer.Serialize(resp, JsonConfig.Options), new JsonDiffConfig(true));
-      {{/body}}
-      } catch (Exception e)
-      {
-        Assert.Fail("An exception was thrown: " + e.Message);
-      }
-    {{/response}}
   }
   {{/tests}}
   {{/blocksRequests}}

templates/csharp/tests/requests/requests_e2e.mustache

@@ -0,0 +1,73 @@
+// {{generationBanner}}
+using Algolia.Search.Http;
+using Algolia.Search.Clients;
+using Algolia.Search.Models.{{clientPrefix}};
+using Algolia.Search.Serializer;
+using Algolia.Search.Tests.Utils;
+using Xunit;
+using System.Text.Json;
+using Quibble.Xunit;
+using dotenv.net;
+using Action = Algolia.Search.Models.Search.Action;
+
+public class {{client}}RequestTestsE2E
+{
+private readonly {{client}} _client;
+
+  public {{client}}RequestTestsE2E()
+  {
+    DotEnv.Load(options: new DotEnvOptions(ignoreExceptions: true, probeForEnv: true, probeLevelsToSearch: 8, envFilePaths: new[] { ".env" }));
+
+    var appId = Environment.GetEnvironmentVariable("ALGOLIA_APPLICATION_ID");
+    if (appId == null)
+    {
+      throw new Exception("please provide an `ALGOLIA_APPLICATION_ID` env var for e2e tests");
+    }
+
+    var apiKey = Environment.GetEnvironmentVariable("{{e2eApiKey}}");
+    if (apiKey == null)
+    {
+      throw new Exception("please provide an `{{e2eApiKey}}` env var for e2e tests");
+    }
+
+    _client = new {{client}}(new {{clientPrefix}}Config(appId, apiKey{{#hasRegionalHost}},"{{defaultRegion}}"{{/hasRegionalHost}}));
+  }
+
+  [Fact]
+  public void Dispose()
+  {
+
+  }
+
+  {{#blocksRequestsE2E}}
+  {{#tests}}
+  [Fact(DisplayName = "{{{testName}}}")]
+  public async Task {{#lambda.pascalcase}}{{method}}Test{{testIndex}}{{/lambda.pascalcase}}()
+  {
+      try {
+        var resp = await _client.{{#lambda.pascalcase}}{{method}}{{/lambda.pascalcase}}Async{{#isGeneric}}<Hit>{{/isGeneric}}({{#parametersWithDataType}}{{> tests/generateParams}}{{^-last}},{{/-last}}{{/parametersWithDataType}}{{#hasRequestOptions}}, new RequestOptions(){
+          {{#requestOptions.queryParameters}}
+            QueryParameters = new Dictionary<string, object>(){ {{#parametersWithDataType}} {"{{{key}}}", {{> tests/requests/requestOptionsParams}} } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
+          {{/requestOptions.queryParameters}}
+          {{#requestOptions.headers}}
+            Headers = new Dictionary<string, string>(){ {{#parametersWithDataType}} {"{{{key}}}", "{{{value}}}" } {{^-last}},{{/-last}}{{/parametersWithDataType}} },
+          {{/requestOptions.headers}}
+        }{{/hasRequestOptions}});
+      {{#response}}
+      {{#statusCode}}
+          // Check status code {{statusCode}}
+          Assert.NotNull(resp);
+      {{/statusCode}}
+
+      {{#body}}
+          JsonAssert.EqualOverrideDefault("{{#lambda.escapeQuotes}}{{{.}}}{{/lambda.escapeQuotes}}", JsonSerializer.Serialize(resp, JsonConfig.Options), new JsonDiffConfig(true));
+      {{/body}}
+      } catch (Exception e)
+      {
+        Assert.Fail("An exception was thrown: " + e.Message);
+      }
+    {{/response}}
+  }
+  {{/tests}}
+  {{/blocksRequestsE2E}}
+}

templates/go/tests/requests/requests.mustache

@@ -3,21 +3,16 @@ package requests
 
 import (
   "encoding/json"
-  "net/url"
-  "os"
-  "testing"
-  "time"
-  "strings"
+	"testing"
+	"time"
 
-  "github.com/kinbiko/jsonassert"
-  "github.com/stretchr/testify/require"
+	"github.com/kinbiko/jsonassert"
+	"github.com/stretchr/testify/require"
 
-  "github.com/joho/godotenv"
+	"gotests/tests"
 
-  "gotests/tests"
-
-  "github.com/algolia/algoliasearch-client-go/v4/algolia/{{clientImport}}"
-  "github.com/algolia/algoliasearch-client-go/v4/algolia/transport"
+	"github.com/algolia/algoliasearch-client-go/v4/algolia/{{clientImport}}"
+	"github.com/algolia/algoliasearch-client-go/v4/algolia/transport"
 )
 
 func create{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t *testing.T) (*{{clientPrefix}}.APIClient, *tests.EchoRequester) {
@@ -38,24 +33,6 @@ func create{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t *t
   return client, echo
 }
 
-{{#hasE2E}}
-func createE2E{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t *testing.T) *{{clientPrefix}}.APIClient {
-  t.Helper()
-
-  appID := os.Getenv("ALGOLIA_APPLICATION_ID")
-  if appID == "" && os.Getenv("CI") != "true" {
-    err := godotenv.Load("../../../../.env")
-    require.NoError(t, err)
-    appID = os.Getenv("ALGOLIA_APPLICATION_ID")
-  }
-  apiKey := os.Getenv("{{e2eApiKey}}")
-  client, err := {{clientPrefix}}.NewClient(appID, apiKey, {{#hasRegionalHost}}{{clientPrefix}}.{{#lambda.uppercase}}{{defaultRegion}}{{/lambda.uppercase}},{{/hasRegionalHost}})
-  require.NoError(t, err)
-
-  return client
-}
-{{/hasE2E}}
-
 {{#blocksRequests}}
 func Test{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}_{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}(t *testing.T) {
     client, echo := create{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t)
@@ -102,37 +79,6 @@ func Test{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}_{{#lambda.ti
         }
       {{/queryParameters}}
       {{/request}}
-      {{#response}}
-      clientE2E := createE2E{{#lambda.titlecase}}{{clientPrefix}}{{/lambda.titlecase}}Client(t)
-      res, err := clientE2E.{{#lambda.titlecase}}{{method}}{{/lambda.titlecase}}({{#hasOperationParams}}client.NewApi{{#lambda.titlecase}}{{method}}{{/lambda.titlecase}}Request(
-        {{#parametersWithDataType}}{{#required}}{{> tests/generateParams}},{{/required}}{{/parametersWithDataType}}
-      ){{#parametersWithDataType}}{{^required}}.With{{#lambda.pascalcase}}{{{key}}}{{/lambda.pascalcase}}({{> tests/generateParams}}){{/required}}{{/parametersWithDataType}}{{/hasOperationParams}}{{#requestOptions}}{{#hasOperationParams}},{{/hasOperationParams}}
-      {{#queryParameters.parametersWithDataType}}{{clientPrefix}}.QueryParamOption("{{{key}}}", {{> tests/generateInnerParams}}),{{/queryParameters.parametersWithDataType}}{{#headers.parametersWithDataType}}{{clientPrefix}}.HeaderParamOption("{{{key}}}", {{> tests/generateInnerParams}}),{{/headers.parametersWithDataType}}
-      {{/requestOptions}})
-      require.NoError(t, err)
-      _ = res
-
-      {{#body}}
-      rawBody, err := json.Marshal(res)
-      require.NoError(t, err)
-
-      var rawBodyMap any
-      err = json.Unmarshal(rawBody, &rawBodyMap)
-      require.NoError(t, err)
-
-      expectedBodyRaw := `{{{.}}}`
-      var expectedBody any
-      err = json.Unmarshal([]byte(expectedBodyRaw), &expectedBody)
-      require.NoError(t, err)
-
-      unionBody := tests.Union(expectedBody, rawBodyMap)
-      unionBodyRaw, err := json.Marshal(unionBody)
-      require.NoError(t, err)
-
-      jaE2E := jsonassert.New(t)
-      jaE2E.Assertf(expectedBodyRaw, strings.ReplaceAll(string(unionBodyRaw), "%", "%%"))
-      {{/body}}
-      {{/response}}
     })
 {{/tests}}
 }