Dyno: materialize iterators into arrays where necessary (#26549)

This PR implements saving iterators into arrays, closing
#26466.

Brad outlines a number of special cases in
#26466 (comment).
Fortunately, these special cases are all implemented in module code; as
a result, rather than making Dyno reason about the shapes / domains of
iterators, this PR simply took the route of enabling the resolution of
various code in `ChapelArray`. This included:

1. Implementing the `_iteratorRecord` type (which simply accepts any
`IterableType`). This is required to select the various `chpl__initCopy`
overloads for constructing arrays.
2. Enabling computing `_shape_` on iterators. This has two components:
actually computing the shape (which is implemented using a Dyno query),
and exposing a `_shape_` field through special casing (like `eltType` in
Dyno's `c_ptr`). Crucially, reflection was used to check if `_shape_`
was set; this meant ensuring that `name to field num` and other
primitives work correctly with "phantom" fields on iterable types.
3. Ensuring that types that are instantiations (including via their
parent types) always have an `instantiatedFrom`, since that field is
used in many places to quickly check if a type is instantiated. I
originally thought it would make more sense to stop relying on that
invariant, but have come to a decision that quick checks of
`instantiatedFrom` are most elegant.
4. Adjusting various places in the Resolver and InitResolver to support
invoking `init` on arrays, and running the arrays' constructors. This
was needed because module code calls `new _array`.
5. Fixing a bug in which parenless procedures returning values could not
be called on class types due to a mis-use of `toCompositeType` vs
`getCompositeType`.
6. Many more fixes, too numerous to be listed here with any value. Many
fixes correspond to individual commits; consider reading through the
commit list to see if anything jumps out.

## Future work
* Move `makeConst` etc to `Qualifier`
* Adjust `ArrayType` to lean on `_instance` more (@riftEmber)
* Consider using a global `QualifiedType()` to be able to return
references and avoid constructing QTs when not needed.

# Testing
- [x] dyno tests
- [x] paratest

frontend/include/chpl/framework/all-global-strings.h

@@ -34,6 +34,9 @@ X(atomic              , "atomic")
 X(bool_               , "bool")
 X(borrow              , "borrow")
 X(borrowed            , "borrowed")
+X(buildTuple          , "_build_tuple")
+X(buildTupleNoref     , "_build_tuple_noref")
+X(buildTupleAlwaysRef , "_build_tuple_always_allow_ref")
 X(by                  , "by")
 X(bytes               , "bytes")
 X(coforall            , "coforall")

frontend/include/chpl/resolution/can-pass.h

@@ -287,8 +287,11 @@ class KindProperties {
 
   bool valid() const { return isValid; }
 
-  /* Creates an corresponding kind that is const */
+  /* Creates a corresponding kind that is const */
   static types::QualifiedType::Kind makeConst(types::QualifiedType::Kind kind);
+
+  /* Creates a corresponding kind that is not a reference */
+  static types::QualifiedType::Kind removeRef(types::QualifiedType::Kind kind);
 };
 
 /**

frontend/include/chpl/resolution/resolution-queries.h

@@ -611,6 +611,9 @@ const types::QualifiedType&
 yieldTypeForIterator(ResolutionContext* rc,
                      const types::IteratorType* iter);
 
+const types::Type* shapeForIterator(Context* context,
+                                    const types::IteratorType* iter);
+
 /**
   Resolve a call to the special 'these' iterator method. For certain types,
   this circumvents the normal call resolution process, since iterating over

frontend/include/chpl/resolution/resolution-types.h

@@ -2013,6 +2013,11 @@ class CallResolutionResult {
     poiInfo_.mark(context);
   }
 
+  static bool update(CallResolutionResult& keep,
+                     CallResolutionResult& addin) {
+    return defaultUpdate(keep, addin);
+  }
+
   size_t hash() const {
     return chpl::hash(mostSpecific_, exprType_, yieldedType_, poiInfo_,
                       speciallyHandled_, rejectedPossibleIteratorCandidates_);
@@ -2284,6 +2289,7 @@ class AssociatedAction {
     ASSIGN,       // same type or different type assign
     COPY_INIT,    // init= from same type
     INIT_OTHER,   // init= from other type
+    CUSTOM_COPY_INIT, // chpl__copyInit for specialized behavior
     DEFAULT_INIT,
     DEINIT,
     ITERATE,      // aka "these"

frontend/include/chpl/types/ArrayType.h

@@ -69,14 +69,16 @@ class ArrayType final : public CompositeType {
   static const ArrayType* getGenericArrayType(Context* context);
 
   static const ArrayType* getArrayType(Context* context,
+                                       const QualifiedType& instance,
                                        const QualifiedType& domainType,
                                        const QualifiedType& eltType);
 
   const Type* substitute(Context* context,
                          const PlaceholderMap& subs) const override {
-    return getArrayType(context,
-                        domainType().substitute(context, subs),
-                        eltType().substitute(context, subs));
+    return getArrayTypeQuery(context,
+                             id_, name_,
+                             Type::substitute(context, instantiatedFrom_->toArrayType(), subs),
+                             resolution::substituteInMap(context, subs_, subs)).get();
   }
 
   QualifiedType domainType() const {

frontend/include/chpl/types/TupleType.h

@@ -128,7 +128,7 @@ class TupleType final : public CompositeType {
   }
 
   /** Return the type of the i'th element */
-  QualifiedType elementType(int i) const;
+  const QualifiedType& elementType(int i) const;
 
   /** Return true if this is a *star tuple* - that is, all of the
       element types are the same. */

frontend/lib/resolution/InitResolver.cpp

@@ -280,6 +280,62 @@ bool InitResolver::isFinalReceiverStateValid(void) {
   return ret;
 }
 
+static std::pair<const BasicClassType*, const BasicClassType*>
+extractBasicSubclassFromInstance(const QualifiedType& instanceQT) {
+  if (instanceQT.isUnknownOrErroneous()) return {nullptr, nullptr};
+
+  const BasicClassType* bct = nullptr;
+  if (auto ct = instanceQT.type()->toClassType()) {
+    bct = ct->basicClassType();
+  } else {
+    bct = instanceQT.type()->toBasicClassType();
+  }
+
+  if (!bct) return {nullptr, nullptr};
+
+  auto parentBct = bct->parentClassType();
+  if (!parentBct) return {nullptr, nullptr};
+
+  return {bct, parentBct};
+}
+
+template <size_t ... Is>
+struct ImplCreateNTuple {
+  template <size_t>
+  using wrap = QualifiedType;
+
+  using type = std::tuple<wrap<Is>...>;
+};
+
+// for the given N names, return N QualifiedTypes
+template <typename ...FieldNames, size_t ... Is>
+static auto helpExtractFields(Context* context, const BasicClassType* bct, std::index_sequence<Is...>, FieldNames... names) {
+  auto& rf = fieldsForTypeDecl(context, bct,
+                               DefaultsPolicy::IGNORE_DEFAULTS);
+  CHPL_ASSERT(rf.numFields() >= 0 && (size_t) rf.numFields() >= sizeof...(names));
+
+  typename ImplCreateNTuple<Is...>::type ret;
+  for (int i = 0; i < rf.numFields(); i++) {
+    ((rf.fieldName(i) == names ? (std::get<Is>(ret) = rf.fieldType(i), 0) : 0), ...);
+  }
+  return ret;
+}
+
+template <typename ...FieldNames>
+static auto extractFields(Context* context, const BasicClassType* bct,  FieldNames... names) {
+  return helpExtractFields(context, bct, std::make_index_sequence<sizeof...(FieldNames)>(), names...);
+}
+
+static std::tuple<QualifiedType, QualifiedType, QualifiedType>
+extractRectangularInfo(Context* context, const BasicClassType* bct) {
+  return extractFields(context, bct, "rank", "idxType", "strides");
+}
+
+static std::tuple<QualifiedType, QualifiedType>
+extractAssociativeInfo(Context* context, const BasicClassType* bct) {
+  return extractFields(context, bct, "idxType", "parSafe");
+}
+
 // Extract domain type information from _instance substitution
 static const DomainType* domainTypeFromSubsHelper(
     Context* context, const CompositeType::SubstitutionsMap& subs) {
@@ -289,65 +345,68 @@ static const DomainType* domainTypeFromSubsHelper(
   if (subs.size() != 1) return genericDomain;
 
   const QualifiedType instanceQt = subs.begin()->second;
-
-  if (auto instance = instanceQt.type()) {
-    if (auto instanceCt = instance->toClassType()) {
-      if (auto instanceBct = instanceCt->basicClassType()) {
-        // Get BaseRectangularDom parent subs for rectangular domain info
-        if (auto baseDom = instanceBct->parentClassType()) {
-          if (baseDom->id().symbolPath() == "ChapelDistribution.BaseRectangularDom") {
-            auto& rf = fieldsForTypeDecl(context, baseDom,
-                                         DefaultsPolicy::IGNORE_DEFAULTS);
-            CHPL_ASSERT(rf.numFields() == 3);
-            QualifiedType rank;
-            QualifiedType idxType;
-            QualifiedType strides;
-            for (int i = 0; i < rf.numFields(); i++) {
-              if (rf.fieldName(i) == "rank") {
-                rank = rf.fieldType(i);
-              } else if (rf.fieldName(i) == "idxType") {
-                idxType = rf.fieldType(i);
-              } else if (rf.fieldName(i) == "strides") {
-                strides = rf.fieldType(i);
-              }
-            }
-
-            return DomainType::getRectangularType(context, instanceQt, rank,
-                                                  idxType, strides);
-          } else if (baseDom->id().symbolPath() == "ChapelDistribution.BaseAssociativeDom") {
-            // Currently the relevant associative domain fields are defined
-            // on all the children of BaseAssociativeDom, so get information
-            // from there.
-            auto& rf = fieldsForTypeDecl(context, instanceBct,
-                                         DefaultsPolicy::IGNORE_DEFAULTS);
-            CHPL_ASSERT(rf.numFields() >= 2);
-            QualifiedType idxType;
-            QualifiedType parSafe;
-            for (int i = 0; i < rf.numFields(); i++) {
-              if (rf.fieldName(i) == "idxType") {
-                idxType = rf.fieldType(i);
-              } else if (rf.fieldName(i) == "parSafe") {
-                parSafe = rf.fieldType(i);
-              }
-            }
-
-            return DomainType::getAssociativeType(context, instanceQt, idxType,
-                                                  parSafe);
-          } else if (baseDom->id().symbolPath() == "ChapelDistribution.BaseSparseDom") {
-            // TODO: support sparse domains
-          } else {
-            // not a recognized domain type
-            return genericDomain;
-          }
-        }
-      }
-    }
+  auto [instanceBct, baseDom] = extractBasicSubclassFromInstance(instanceQt);
+  if (!instanceBct || !baseDom) return genericDomain;
+
+  if (baseDom->id().symbolPath() == "ChapelDistribution.BaseRectangularDom") {
+    auto [rank, idxType, strides] = extractRectangularInfo(context, baseDom);
+    return DomainType::getRectangularType(context, instanceQt, rank,
+                                          idxType, strides);
+  } else if (baseDom->id().symbolPath() == "ChapelDistribution.BaseAssociativeDom") {
+    // Currently the relevant associative domain fields are defined
+    // on all the children of BaseAssociativeDom, so get information
+    // from there.
+    auto [idxType, parSafe] = extractAssociativeInfo(context, instanceBct);
+    return DomainType::getAssociativeType(context, instanceQt, idxType,
+                                          parSafe);
+  } else if (baseDom->id().symbolPath() == "ChapelDistribution.BaseSparseDom") {
+    // TODO: support sparse domains
+  } else {
+    // not a recognized domain type
+    return genericDomain;
   }
 
   // If we reach here, we weren't able to resolve the domain type
   return genericDomain;
 }
 
+static const ArrayType* arrayTypeFromSubsHelper(
+    Context* context, const CompositeType::SubstitutionsMap& subs) {
+  auto genericArray = ArrayType::getGenericArrayType(context);
+
+  // Expect one substitution for _instance
+  if (subs.size() != 1) return genericArray;
+
+  const QualifiedType instanceQt = subs.begin()->second;
+  auto [instanceBct, baseArr] = extractBasicSubclassFromInstance(instanceQt);
+  if (!instanceBct || !baseArr) return genericArray;
+
+  if (baseArr->id().symbolPath() == "ChapelDistribution.BaseRectangularArr") {
+    // TODO: the ArrayType predates our '_instance-aware' code (developed
+    //       by Anna and currently at work in Domains). For now, just use the
+    //       "old" style containing a domain type and element type to instantiate
+    //       the array.
+    //
+    //       Anna is planning on tackling this in future work.
+
+    auto baseArrRect = baseArr->parentClassType();
+    CHPL_ASSERT(baseArrRect && baseArrRect->id().symbolPath() == "ChapelDistribution.BaseArrOverRectangularDom");
+
+    auto [rank, idxType, strides] = extractRectangularInfo(context, baseArrRect);
+    auto [eltType] = extractFields(context, baseArr, "eltType");
+
+    auto domain = DomainType::getRectangularType(context, instanceQt, rank,
+                                                 idxType, strides);
+    return ArrayType::getArrayType(context,
+                                   instanceQt,
+                                   QualifiedType(QualifiedType::TYPE, domain),
+                                   eltType);
+  }
+
+  // If we reach here, we weren't able to resolve the array type
+  return genericArray;
+}
+
 static const Type* ctFromSubs(Context* context,
                               const Type* receiverType,
                               const BasicClassType* superType,
@@ -383,6 +442,8 @@ static const Type* ctFromSubs(Context* context,
     ret = ClassType::get(context, basic, manager, dec);
   } else if (receiverType->isDomainType()) {
     ret = domainTypeFromSubsHelper(context, subs);
+  } else if (receiverType->isArrayType()) {
+    ret = arrayTypeFromSubsHelper(context, subs);
   } else {
     CHPL_ASSERT(false && "Not handled!");
   }
@@ -471,7 +532,8 @@ QualifiedType::Kind InitResolver::determineReceiverIntent(void) {
   if (initialRecvType_->isClassType()) {
     return QualifiedType::CONST_IN;
   } else if (initialRecvType_->isRecordType() ||
-             initialRecvType_->isDomainType()) {
+             initialRecvType_->isDomainType() ||
+             initialRecvType_->isArrayType()) {
     return QualifiedType::REF;
   } else {
     CHPL_ASSERT(false && "Not handled");