Refactor chapel-py to use the C Python Stable API

doc/rst/conf.py

@@ -30,11 +30,13 @@
 # to prevent failures if chapel-py is not built/installed, check if its installed
 # if installed, add the path and generate the rst file
 # if not installed, just create the file so that the build doesn't fail
-chapel_py_dir = os.path.abspath(
-    "../../third-party/chpl-venv/install/chpl-frontend-py-deps-py"
-    + str(sys.version_info.major)
-    + str(sys.version_info.minor)
-)
+old_sys_path = sys.path.copy()
+sys.path.insert(0, os.path.abspath('../../util/chplenv'))
+import chpl_home_utils
+chapel_py_dir = chpl_home_utils.get_chpldeps(True)
+del chpl_home_utils
+sys.path = old_sys_path
+
 include_chapel_py_docs = False
 chapel_py_api_template = os.path.abspath("./tools/chapel-py/chapel-py-api-template.rst")
 chapel_py_api_rst = os.path.abspath("./tools/chapel-py/chapel-py-api.rst")

third-party/chpl-venv/Makefile.include

@@ -32,5 +32,5 @@ CHPL_VENV_VIRTUALENV_DIR_DEPS2_OK=$(CHPL_VENV_BUILD)/build-venv/ok2
 CHPL_VENV_VIRTUALENV_BIN=$(CHPL_VENV_VIRTUALENV_DIR)/bin
 CHPL_VENV_INSTALL=$(CHPL_VENV_DIR)/install
 CHPL_VENV_CHPLDEPS=$(CHPL_VENV_INSTALL)/chpldeps
-CHPL_VENV_CHPL_FRONTEND_PY_DEPS=$(CHPL_VENV_INSTALL)/chpl-frontend-py-deps-py$(shell $(PYTHON) -c 'import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))')
+CHPL_VENV_CHPL_FRONTEND_PY_DEPS=$(shell $(PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/chpl_home_utils.py --chpldeps-chapel-py)
 CHPL_VENV_CHPLDEPS_MAIN=$(CHPL_VENV_CHPLDEPS)/__main__.py

third-party/chpl-venv/chapel-py-requirements.txt

@@ -4,3 +4,5 @@ lsprotocol==2023.0.1
 pygls==1.3.1
 typeguard==4.3.0
 ConfigArgParse==1.7
+# needed for <python3.11
+exceptiongroup==1.2.2

tools/chapel-py/setup.py

@@ -84,6 +84,15 @@
 
 os.environ["CC"] = host_cc
 os.environ["CXX"] = host_cxx
+
+# We use a stable API for Python 3.10+
+# The limiting factor here is PyUnicode_AsUTF8AndSize, which was introduced in 3.10
+# If we are trying to build for an older version, we need to use the non-stable API
+use_stable_api = False
+if sys.version_info[:2] >= (3, 10):
+    use_stable_api = True
+    CXXFLAGS += ["-DCHAPEL_PY_USE_STABLE_API"]
+
 setup(
     name="chapel",
     version="0.1",
@@ -96,6 +105,7 @@
             depends=glob.glob("src/**/*.h", recursive=True),
             extra_compile_args=CXXFLAGS,
             extra_link_args=LDFLAGS,
+            py_limited_api=use_stable_api,
         )
     ],
 )

tools/chapel-py/src/PythonWrapper.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2023-2025 Hewlett Packard Enterprise Development LP
+ * Other additional copyright holders may be indicated within.
+ *
+ * The entirety of this work is licensed under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CHAPEL_PY_PYTHON_WRAPPER_H
+#define CHAPEL_PY_PYTHON_WRAPPER_H
+
+#ifdef CHAPEL_PY_USE_STABLE_API
+// use a stable API version of python 3.10
+#define Py_LIMITED_API 0x030A0000
+#endif
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#endif

tools/chapel-py/src/chapel.cpp

@@ -17,8 +17,7 @@
  * limitations under the License.
  */
 
-#define PY_SSIZE_T_CLEAN
-#include "Python.h"
+#include "PythonWrapper.h"
 #include "chpl/framework/check-build.h"
 #include "chpl/framework/Context.h"
 #include "chpl/parsing/parsing-queries.h"
@@ -44,23 +43,19 @@ extern "C" {
 PyMODINIT_FUNC PyInit_core(void) {
   PyObject* chapelModule = nullptr;
 
-  setupAstIterType();
-  setupAstCallIterType();
-  setupGeneratedTypes();
+  if(!setupAstIterType()) return nullptr;
+  if(!setupAstCallIterType()) return nullptr;
 
-#define READY_TYPE(NAME) if (PyType_Ready(&NAME##Type) < 0) return nullptr;
-#define GENERATED_TYPE(ROOT, ROOT_TYPE, NAME, TYPE, TAG, FLAGS) READY_TYPE(NAME)
-#include "generated-types-list.h"
-#undef GENERATED_TYPE
-  READY_TYPE(AstIter)
-  READY_TYPE(AstCallIter)
+  // these must be called before setupGeneratedTypes, since they are base classes
+  if (AstNodeObject::ready() < 0) return nullptr;
+  if (ChapelTypeObject::ready() < 0) return nullptr;
+  if (ParamObject::ready() < 0) return nullptr;
+
+  if (!setupGeneratedTypes()) return nullptr;
 
   if (ContextObject::ready() < 0) return nullptr;
   if (LocationObject::ready() < 0) return nullptr;
   if (ScopeObject::ready() < 0) return nullptr;
-  if (AstNodeObject::ready() < 0) return nullptr;
-  if (ChapelTypeObject::ready() < 0) return nullptr;
-  if (ParamObject::ready() < 0) return nullptr;
   if (ErrorObject::ready() < 0) return nullptr;
   if (ErrorManagerObject::ready() < 0) return nullptr;
   if (ResolvedExpressionObject::ready() < 0) return nullptr;
@@ -70,7 +65,7 @@ PyMODINIT_FUNC PyInit_core(void) {
   chapelModule = PyModule_Create(&ChapelModule);
   if (!chapelModule) return nullptr;
 
-#define ADD_TYPE(NAME) if (PyModule_AddObject(chapelModule, #NAME, (PyObject*) &NAME##Type) < 0) return nullptr;
+#define ADD_TYPE(NAME) if (PyModule_AddObject(chapelModule, #NAME, (PyObject*) NAME##Type) < 0) return nullptr;
 #define GENERATED_TYPE(ROOT, ROOT_TYPE, NAME, TYPE, TAG, FLAGS) ADD_TYPE(NAME)
 #include "generated-types-list.h"
 #undef GENERATED_TYPE

tools/chapel-py/src/core-types-gen.cpp

@@ -44,7 +44,12 @@ using namespace uast;
  */
 #define DEFINE_INIT_FOR(NAME, TAG)\
   int NAME##Object_init(NAME##Object* self, PyObject* args, PyObject* kwargs) { \
-    return parentTypeFor(TAG)->tp_init((PyObject*) self, args, kwargs); \
+    auto func = ((int(*)(PyObject*, PyObject*, PyObject*)) PyType_GetSlot(parentTypeFor(TAG), Py_tp_init)); \
+    if (!func) { \
+      PyErr_SetString(PyExc_TypeError, "Cannot initialize Chapel AST node"); \
+      return -1; \
+    } \
+    return func((PyObject*) self, args, kwargs); \
   } \
 
 /* Use the X-macros pattern to invoke DEFINE_INIT_FOR for each AST node type. */
@@ -92,7 +97,9 @@ struct InvokeHelper<void(Args...)> {
   template <typename F>
   static PyObject* invoke(ContextObject* contextObject, F&& fn) {
     fn();
-    Py_RETURN_NONE;
+    // In python3.12, Py_None is immortal and this is not needed
+    Py_INCREF(Py_None);
+    return Py_None;
   }
 };
 
@@ -128,7 +135,7 @@ struct InvokeHelper<void(Args...)> {
     if (!node) return nullptr; \
     \
     auto argList = Py_BuildValue("(O)", (PyObject*) self); \
-    auto astCallIterObjectPy = PyObject_CallObject((PyObject *) &AstCallIterType, argList); \
+    auto astCallIterObjectPy = PyObject_CallObject((PyObject *) AstCallIterType, argList); \
     Py_XDECREF(argList); \
     \
     auto astCalliterObject = (AstCallIterObject*) astCallIterObjectPy; \
@@ -153,27 +160,36 @@ ACTUAL_ITERATOR(FnCall);
    macro defines what a type object for an AST node (abstract or not) should
    look like. */
 
-#define DEFINE_PY_TYPE_FOR(NAME)\
-  PyTypeObject NAME##Type = { \
-    PyVarObject_HEAD_INIT(NULL, 0) \
-  }; \
+#define DEFINE_PY_TYPE_FOR(NAME) PyTypeObject* NAME##Type = NULL;
 
 /* Now, invoke DEFINE_PY_TYPE_FOR for each AST node to get our type objects. */
 #define GENERATED_TYPE(ROOT, ROOT_TYPE, NAME, TYPE, TAG, FLAGS) DEFINE_PY_TYPE_FOR(NAME)
 #include "generated-types-list.h"
 
 #define INITIALIZE_PY_TYPE_FOR(NAME, TYPE, TAG, FLAGS)\
-  TYPE.tp_name = "chapel."#NAME; \
-  TYPE.tp_basicsize = sizeof(NAME##Object); \
-  TYPE.tp_itemsize = 0; \
-  TYPE.tp_flags = FLAGS; \
-  TYPE.tp_doc = PyDoc_STR("A Chapel " #NAME " AST node"); \
-  TYPE.tp_methods = (PyMethodDef*) PerTypeMethods<NAME##Object>::methods; \
-  TYPE.tp_base = parentTypeFor(TAG); \
-  TYPE.tp_init = (initproc) NAME##Object_init; \
-  TYPE.tp_new = PyType_GenericNew; \
-
-void setupGeneratedTypes() {
+  do { \
+    PyType_Slot slots[] = { \
+      {Py_tp_doc, (void*) PyDoc_STR("A Chapel " #NAME " AST node")}, \
+      {Py_tp_methods, (void*) PerTypeMethods<NAME##Object>::methods}, \
+      {Py_tp_init, (void*) NAME##Object_init}, \
+      {Py_tp_new, (void*) PyType_GenericNew}, \
+      {0, nullptr} \
+    }; \
+    PyType_Spec spec = { \
+      /*name*/ "chapel."#NAME, \
+      /*basicsize*/ sizeof(NAME##Object), \
+      /*itemsize*/ 0, \
+      /*flags*/ FLAGS, \
+      /*slots*/ slots \
+    }; \
+    auto parentType = parentTypeFor(TAG); \
+    auto bases = PyTuple_Pack(1, parentType); \
+    TYPE = (PyTypeObject*)PyType_FromSpecWithBases(&spec, bases); \
+    if (!TYPE || PyType_Ready(TYPE) < 0) return false; \
+  } while(0);
+
+bool setupGeneratedTypes() {
 #define GENERATED_TYPE(ROOT, ROOT_TYPE, NAME, TYPE, TAG, FLAGS) INITIALIZE_PY_TYPE_FOR(NAME, NAME##Type, TAG, FLAGS)
 #include "generated-types-list.h"
+return true;
 }

tools/chapel-py/src/core-types-gen.h

@@ -47,7 +47,7 @@
     ContextObject* context() const { return (ContextObject*) parent.contextObject; } \
   }; \
   \
-  extern PyTypeObject NAME##Type;
+  extern PyTypeObject* NAME##Type;
 
 /* Generate a Python object for reach AST node type. */
 #define GENERATED_TYPE(ROOT, ROOT_TYPE, NAME, TYPE, TAG, FLAGS) DECLARE_PY_OBJECT_FOR(ROOT, NAME)
@@ -95,6 +95,6 @@ struct PerTypeMethods {
   {#NAME, NODE##Object_##NAME, METH_NOARGS, DOCSTR},
 #include "method-tables.h"
 
-void setupGeneratedTypes();
+bool setupGeneratedTypes();
 
 #endif

tools/chapel-py/src/core-types.cpp

@@ -26,6 +26,7 @@
 #include "python-types.h"
 #include "error-tracker.h"
 #include "resolution.h"
+#include "python-type-helper.h"
 
 using namespace chpl;
 using namespace uast;
@@ -105,7 +106,7 @@ std::string generatePyiFile() {
     printedAnything = false; \
     generated.insert(NODE##Object::Name); \
     if (auto parentType = ParentTypeInfo<NODE##Object>::parentTypeObject()) { \
-      ss << "(" << parentType->tp_name << ")"; \
+      ss << "(" << getTypeName(parentType) << ")"; \
     } \
     ss << ":" << std::endl;
   #define METHOD(NODE, NAME, DOCSTR, TYPEFN, BODY) \
@@ -132,7 +133,7 @@ std::string generatePyiFile() {
     if(generated.find(NODE##Object::Name) == generated.end()) { \
       ss << "class " << NODE##Object::Name; \
       if (auto parentType = ParentTypeInfo<NODE##Object>::parentTypeObject()) { \
-        ss << "(" << parentTypeFor(TAG)->tp_name << ")"; \
+        ss << "(" << getTypeName(parentType) << ")"; \
       } \
       ss << ":" << std::endl; \
       ss << "    pass" << std::endl; \
@@ -155,7 +156,7 @@ PyObject* AstNodeObject::iter(AstNodeObject *self) {
 
 void ChapelTypeObject_dealloc(ChapelTypeObject* self) {
   Py_XDECREF(self->contextObject);
-  Py_TYPE(self)->tp_free((PyObject *) self);
+  call_tp_free(ChapelTypeObject::PythonType, (PyObject*) self);
 }
 
 PyObject* ChapelTypeObject::str(ChapelTypeObject* self) {
@@ -186,14 +187,14 @@ PyTypeObject* parentTypeFor(asttags::AstTag tag) {
 #define AST_BEGIN_SUBCLASSES(NAME)
 #define AST_END_SUBCLASSES(NAME) \
   if (tag > asttags::START_##NAME && tag < asttags::END_##NAME) { \
-    return &NAME##Type; \
+    return NAME##Type; \
   }
 #include "chpl/uast/uast-classes-list.h"
 #undef AST_NODE
 #undef AST_LEAF
 #undef AST_BEGIN_SUBCLASSES
 #undef AST_END_SUBCLASSES
-  return &AstNodeObject::PythonType;
+  return AstNodeObject::PythonType;
 }
 
 PyTypeObject* parentTypeFor(types::typetags::TypeTag tag) {
@@ -202,18 +203,18 @@ PyTypeObject* parentTypeFor(types::typetags::TypeTag tag) {
 #define TYPE_BEGIN_SUBCLASSES(NAME)
 #define TYPE_END_SUBCLASSES(NAME) \
   if (tag > types::typetags::START_##NAME && tag < types::typetags::END_##NAME) { \
-    return &NAME##Type; \
+    return NAME##Type; \
   }
 #include "chpl/types/type-classes-list.h"
 #undef TYPE_NODE
 #undef BUILTIN_TYPE_NODE
 #undef TYPE_BEGIN_SUBCLASSES
 #undef TYPE_END_SUBCLASSES
-  return &ChapelTypeObject::PythonType;
+  return ChapelTypeObject::PythonType;
 }
 
 PyTypeObject* parentTypeFor(chpl::types::paramtags::ParamTag tag) {
-  return &ParamObject::PythonType;
+  return ParamObject::PythonType;
 }
 
 PyObject* wrapGeneratedType(ContextObject* context, const AstNode* node) {
@@ -226,7 +227,7 @@ PyObject* wrapGeneratedType(ContextObject* context, const AstNode* node) {
   switch (node->tag()) {
 #define CAST_TO(NAME) \
     case asttags::NAME: \
-      toReturn = PyObject_CallObject((PyObject*) &NAME##Type, args); \
+      toReturn = PyObject_CallObject((PyObject*) NAME##Type, args); \
       ((NAME##Object*) toReturn)->parent.value_ = node->to##NAME(); \
       break;
 #define AST_NODE(NAME) CAST_TO(NAME)
@@ -256,7 +257,7 @@ PyObject* wrapGeneratedType(ContextObject* context, const types::Type* node) {
   switch (node->tag()) {
 #define CAST_TO(NAME) \
     case types::typetags::NAME: \
-      toReturn = PyObject_CallObject((PyObject*) &NAME##Type, args); \
+      toReturn = PyObject_CallObject((PyObject*) NAME##Type, args); \
       ((NAME##Object*) toReturn)->parent.value_ = (const types::Type*) node->to##NAME(); \
       break;
 #define TYPE_NODE(NAME) CAST_TO(NAME)
@@ -286,7 +287,7 @@ PyObject* wrapGeneratedType(ContextObject* context, const chpl::types::Param* no
   switch (node->tag()) {
 #define PARAM_NODE(NAME, TYPE) \
     case chpl::types::paramtags::NAME: \
-      toReturn = PyObject_CallObject((PyObject*) &NAME##Type, args); \
+      toReturn = PyObject_CallObject((PyObject*) NAME##Type, args); \
       ((NAME##Object*) toReturn)->parent.value_ = node->to##NAME(); \
       break;
 #include "chpl/types/param-classes-list.h"