[Major][Version] Update to LLVM 19.1 and merge MLIR dialect (#203)

.github/workflows/config.yml

@@ -0,0 +1,48 @@
+# Copyright Allo authors. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Allo Test"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    container:
+      image: chhzh123/allo:llvm-19.x-py3.12
+
+    steps:
+    # https://github.com/actions/checkout/issues/363#issuecomment-1915075699
+    - name: Install git in container
+      run: |
+        apt-get install git
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.ref }}
+        fetch-depth: 0
+    - name: Build Allo
+      shell: bash
+      run: |
+        source activate allo
+        export LLVM_BUILD_DIR=/root/llvm-project/build
+        python3 -m pip install -v -e .
+    - name: Formatting Check
+      shell: bash
+      run: |
+        source activate allo
+        bash scripts/lint/task_lint.sh
+    - name: Testing
+      shell: bash
+      run: |
+        source activate allo
+        export PATH=/root/llvm-project/build/bin:${PATH}
+        python3 -m pytest tests -v
+        python3 -m pytest tutorials -v
+        python3 -m pytest examples/polybench -v

.github/workflows/sphinx_build.yml

@@ -23,40 +23,28 @@ jobs:
     # This job runs on Linux
     runs-on: ubuntu-latest
     container:
-      image: chhzh123/hcl-dialect:llvm-18.x-py3.12
+      image: chhzh123/allo:llvm-19.x-py3.12
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Install git in container
+      run: |
+        apt-get install git
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.ref }}
-    # - name: Patch LLVM Project
-    #   run: |
-    #     cp externals/llvm_patch /root/llvm-project
-    #     cd /root/llvm-project
-    #     git apply llvm_patch
-    - name: Build HCL-MLIR
+        fetch-depth: 0
+    - name: Build Allo
       shell: bash
       run: |
-        source activate hcl-dev
-        export SUBMOD_COMMIT=$(git rev-parse HEAD:externals/hcl_mlir)
-        cd /root/hcl-dialect
-        git remote update
-        git fetch
-        git checkout $SUBMOD_COMMIT
-        cd build
-        make -j4
-        cd tools/hcl/python_packages/hcl_core
-        python3 -m pip install -e .
+        source activate allo
+        export LLVM_BUILD_DIR=/root/llvm-project/build
+        python3 -m pip install -v -e .
     - name: Build docs
       shell: bash
       run: |
-        source activate hcl-dev
-        export ALLO_HOME=$(pwd)
-        export LLVM_BUILD_DIR=/root/llvm-project/build
-        export PATH=${LLVM_BUILD_DIR}/bin:${PATH}
-        export PYTHONPATH=$ALLO_HOME:${PYTHONPATH}
-        export PYTHONPATH=/root/hcl-dialect/build/tools/hcl/python_packages/hcl_core:${PYTHONPATH}
-        python3 -m pip install -r requirements.txt
+        source activate allo
+        export PATH=/root/llvm-project/build/bin:${PATH}
         cd docs
         python3 -m pip install -r requirements.txt
         make html
@@ -66,25 +54,10 @@ jobs:
       with:
         name: DocumentationHTML
         path: docs/build/html/
-
-  deploy:
-    # https://github.com/JamesIves/github-pages-deploy-action?tab=readme-ov-file#operating-system-support-
-    concurrency: ci-${{ github.ref }}
-    # Create a separate job that does not use the docker container
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-    - name: Checkout 🛎️
-      uses: actions/checkout@v4
-    - name: Download Artifacts 🔻 # The built project is downloaded into the 'DocumentationHTML' folder.
-      uses: actions/download-artifact@v4
-      with:
-        name: DocumentationHTML
-        path: DocumentationHTML
     - name: Deploy docs
       uses: JamesIves/github-pages-deploy-action@v4
       if: github.event_name == 'push'
       with:
         ACCESS_TOKEN: ${{ secrets.DEPLOY_ACCESS_TOKEN }}
         BRANCH: gh-pages
-        FOLDER: 'DocumentationHTML'
+        FOLDER: docs/build/html/

.gitignore

@@ -26,6 +26,52 @@ project_*
 project
 rapidjson
 
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+build
+.vscode
+
+# MLIR intermediate results
+*.opt
+
+# HLS
+*.log
+*.prj
+
+tmp
+*.pyc
+
 .vscode
 *.prj
 *.pdf

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "externals/hcl_mlir"]
-	path = externals/hcl_mlir
-	url = https://github.com/cornell-zhang/hcl-dialect.git
+[submodule "externals/llvm-project"]
+	path = externals/llvm-project
+	url = https://github.com/llvm/llvm-project.git

allo/__init__.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # pylint: disable=redefined-builtin
 
-from . import frontend, backend, ir, passes, library
+from . import frontend, backend, ir, passes, library, _mlir
 from .customize import customize
 from .backend.llvm import invoke_mlir_parser, LLVMModule
 from .backend.hls import HLSModule

allo/_mlir

@@ -0,0 +1 @@
+../mlir/build/tools/allo/_mlir

allo/backend/hls.py

@@ -7,13 +7,13 @@
 import io
 import subprocess
 import time
-from hcl_mlir.dialects import hcl as hcl_d
-from hcl_mlir.ir import (
+from .._mlir.dialects import allo as allo_d
+from .._mlir.ir import (
     Context,
     Location,
     Module,
 )
-from hcl_mlir.passmanager import PassManager
+from .._mlir.passmanager import PassManager
 
 from .vitis import (
     codegen_host,
@@ -165,7 +165,7 @@ def __init__(
         self.platform = platform
         self.ext_libs = [] if ext_libs is None else ext_libs
         with Context() as ctx, Location.unknown():
-            hcl_d.register_dialect(ctx)
+            allo_d.register_dialect(ctx)
             self.module = Module.parse(str(mod), ctx)
             self.func = find_func_in_module(self.module, top_func_name)
             if platform == "vitis_hls":
@@ -206,15 +206,15 @@ def __init__(
                 # used for lowering tensor.empty
                 "empty-tensor-to-alloc-tensor,"
                 # translate tensor dialect (virtual) to memref dialect (physical)
-                "one-shot-bufferize{allow-return-allocs bufferize-function-boundaries},"
+                "one-shot-bufferize{bufferize-function-boundaries},"
                 # common lowering passes
                 "func.func(convert-linalg-to-affine-loops)"
                 # DO NOT LOWER AFFINE DIALECT
                 ")"
             )
             pm.run(self.module.operation)
         buf = io.StringIO()
-        hcl_d.emit_vhls(self.module, buf)
+        allo_d.emit_vhls(self.module, buf)
         buf.seek(0)
         self.hls_code = buf.read()
         if project is not None:

allo/backend/llvm.py

@@ -5,20 +5,20 @@
 import os
 import ctypes
 import numpy as np
-from hcl_mlir.ir import (
+from .._mlir.ir import (
     Context,
     Location,
     Module,
     UnitAttr,
 )
-from hcl_mlir.dialects import hcl as hcl_d
-from hcl_mlir.passmanager import PassManager
-from hcl_mlir.execution_engine import ExecutionEngine
-from hcl_mlir.runtime import (
+from .._mlir.dialects import allo as allo_d
+from .._mlir.passmanager import PassManager
+from .._mlir.execution_engine import ExecutionEngine
+from .._mlir.runtime import (
     get_ranked_memref_descriptor,
     make_nd_memref_descriptor,
 )
-from hcl_mlir.exceptions import DTypeWarning
+from .._mlir.exceptions import DTypeWarning
 from ..ir.transform import find_func_in_module
 from ..passes import (
     _mlir_lower_pipeline,
@@ -46,7 +46,7 @@
 
 def invoke_mlir_parser(mod: str):
     with Context() as ctx, Location.unknown():
-        hcl_d.register_dialect(ctx)
+        allo_d.register_dialect(ctx)
         module = Module.parse(str(mod), ctx)
     return module
 
@@ -55,7 +55,7 @@ class LLVMModule:
     def __init__(self, mod, top_func_name, ext_libs=None):
         # Copy the module to avoid modifying the original one
         with Context() as ctx:
-            hcl_d.register_dialect(ctx)
+            allo_d.register_dialect(ctx)
             self.module = Module.parse(str(mod), ctx)
             self.top_func_name = top_func_name
             func = find_func_in_module(self.module, top_func_name)
@@ -68,17 +68,17 @@ def __init__(self, mod, top_func_name, ext_libs=None):
             if len(ext_libs) > 0:
                 call_ext_libs_in_ptr(self.module, ext_libs)
             # Remove .partition() annotation
-            hcl_d.remove_stride_map(self.module)
+            allo_d.remove_stride_map(self.module)
             # Resolve FixedType
-            hcl_d.lower_fixed_to_int(self.module)
-            hcl_d.lower_bit_ops(self.module)
+            allo_d.lower_fixed_to_int(self.module)
+            allo_d.lower_bit_ops(self.module)
             # Run through lowering passes
             pm = PassManager.parse(
                 "builtin.module("
                 # used for lowering tensor.empty
                 "empty-tensor-to-alloc-tensor,"
                 # translate tensor dialect (virtual) to memref dialect (physical)
-                "one-shot-bufferize{allow-return-allocs bufferize-function-boundaries},"
+                "one-shot-bufferize{bufferize-function-boundaries},"
                 # used for lowering memref.subview
                 "expand-strided-metadata,"
                 # common lowering passes
@@ -96,7 +96,9 @@ def __init__(self, mod, top_func_name, ext_libs=None):
             func.attributes["llvm.emit_c_interface"] = UnitAttr.get()
             func.attributes["top"] = UnitAttr.get()
             # Final lowering
-            hcl_d.lower_hcl_to_llvm(self.module, ctx)
+            allo_d.lower_allo_to_llvm(self.module, ctx)
+            pm = PassManager.parse("builtin.module(reconcile-unrealized-casts)")
+            pm.run(self.module.operation)
             # Add shared library
             if os.getenv("LLVM_BUILD_DIR") is not None:
                 shared_libs = [