[MLIR][SPIRV] Split (De-)serialization into 2 libs.

[ergawy]

Summary

This commit splits SPIR-V's serialization and deserialization code into
2 separate targets. The motiviation being that the serializer is used
more often the deserializer and therefore lumping them together
unnecessarily increases binary size for the most common case.

[antiagainst]

Awesome! Given we are here already; can we move them to the final place they should be? Specifically,

include/mlir/Target/
  SPIRV/
    Serialization.h (for containing the `serialize` entry point)
    Deserialization.h (for containing the `deserialize` entry point)
lib/Target/
  SPIRV/
    (All cpp files here)

[ergawy]

Re-organize files a bit differently.

[ergawy]

Fix header guards.

[ergawy]

Rebase.

[ergawy]

Hi,

Friendly reminder to land this :). I would like to add (de-)serialization support for OpSpecConstantOp and conflicts will be easier to handle if this patch is merged before I start. If you need to wait on this one for a bit more, then no problem I will work on my new patch and handle conflicts later.

[antiagainst]

Sorry about the delay! There were issues building with shared library. I fixed them. (You can diff the version here against the landed commit to find out what's gotten changed if curious.) Also need to wire up the BUILD changes for Bazel (https://github.com/google/llvm-bazel) for Google internal uses.

mlir/include/mlir/Target/SPIRV/Deserialization.h

 //===- Serialization.h - MLIR SPIR-V (De)serialization ----------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file declares the entry points for serialize and deserialize SPIR-V
-// binary modules.
+// This file declares the entry points for deserializing SPIR-V binary modules.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_SPIRV_SERIALIZATION_H_
-#define MLIR_DIALECT_SPIRV_SERIALIZATION_H_
+#ifndef MLIR_DIALECT_SPIRV_DESERIALIZATION_H_
+#define MLIR_DIALECT_SPIRV_DESERIALIZATION_H_
 
 #include "mlir/Support/LLVM.h"
 
 namespace mlir {
 struct LogicalResult;
 class MLIRContext;
 
 namespace spirv {
-class ModuleOp;
 class OwningSPIRVModuleRef;
 
-/// Serializes the given SPIR-V `module` and writes to `binary`. On failure,
-/// reports errors to the error handler registered with the MLIR context for
-/// `module`.
-LogicalResult serialize(ModuleOp module, SmallVectorImpl<uint32_t> &binary,
-                        bool emitDebugInfo = false);
-
 /// Deserializes the given SPIR-V `binary` module and creates a MLIR ModuleOp
 /// in the given `context`. Returns the ModuleOp on success; otherwise, reports
 /// errors to the error handler registered with `context` and returns a null
 /// module.
 OwningSPIRVModuleRef deserialize(ArrayRef<uint32_t> binary,
                                  MLIRContext *context);
 
 } // end namespace spirv
 } // end namespace mlir
 
-#endif // MLIR_DIALECT_SPIRV_SERIALIZATION_H_
+#endif // MLIR_DIALECT_SPIRV_DESERIALIZATION_H_

mlir/include/mlir/Target/SPIRV/Serialization.h

 //===- Serialization.h - MLIR SPIR-V (De)serialization ----------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file declares the entry points for serialize and deserialize SPIR-V
-// binary modules.
+// This file declares the entry point for serializing SPIR-V binary modules.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef MLIR_DIALECT_SPIRV_SERIALIZATION_H_
 #define MLIR_DIALECT_SPIRV_SERIALIZATION_H_
 
 #include "mlir/Support/LLVM.h"
 
 namespace mlir {
 struct LogicalResult;
 class MLIRContext;
 
 namespace spirv {
 class ModuleOp;
-class OwningSPIRVModuleRef;
 
 /// Serializes the given SPIR-V `module` and writes to `binary`. On failure,
 /// reports errors to the error handler registered with the MLIR context for
 /// `module`.
 LogicalResult serialize(ModuleOp module, SmallVectorImpl<uint32_t> &binary,
                         bool emitDebugInfo = false);
 
-/// Deserializes the given SPIR-V `binary` module and creates a MLIR ModuleOp
-/// in the given `context`. Returns the ModuleOp on success; otherwise, reports
-/// errors to the error handler registered with `context` and returns a null
-/// module.
-OwningSPIRVModuleRef deserialize(ArrayRef<uint32_t> binary,
-                                 MLIRContext *context);
-
 } // end namespace spirv
 } // end namespace mlir
 
 #endif // MLIR_DIALECT_SPIRV_SERIALIZATION_H_

mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "../PassDetail.h"
 #include "mlir/Conversion/GPUToVulkan/ConvertGPUToVulkanPass.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
-#include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/Module.h"
 #include "mlir/IR/StandardTypes.h"
+#include "mlir/Target/SPIRV/Serialization.h"
 
 using namespace mlir;
 
 static constexpr const char *kSPIRVBlobAttrName = "spirv_blob";
 static constexpr const char *kSPIRVEntryPointAttrName = "spirv_entry_point";
 static constexpr const char *kVulkanLaunch = "vulkanLaunch";
 
 namespace {

mlir/lib/Dialect/SPIRV/CMakeLists.txt

add_mlir_dialect_library(MLIRSPIRV
   MLIRIR
   MLIRParser
   MLIRSideEffectInterfaces
   MLIRSupport
   MLIRTransforms
   )
 
 add_subdirectory(Linking)
-add_subdirectory(Serialization)
 add_subdirectory(Transforms)

mlir/lib/Dialect/SPIRV/Serialization/CMakeLists.txt

-add_mlir_dialect_library(MLIRSPIRVSerialization
-  Deserializer.cpp
-  Serializer.cpp
-  SPIRVBinaryUtils.cpp
-  TranslateRegistration.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SPIRV
-
-  DEPENDS
-  MLIRSPIRVSerializationGen
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRSPIRV
-  MLIRSupport
-  MLIRTranslation
-  )

mlir/lib/Target/CMakeLists.txt

add_mlir_translation_library(MLIRTargetROCDLIR
 
   LINK_LIBS PUBLIC
   MLIRGPU
   MLIRIR
   MLIRLLVMIR
   MLIRROCDLIR
   MLIRTargetLLVMIRModuleTranslation
   )
+
+add_mlir_translation_library(MLIRSPIRVSerialization
+  SPIRV/Serialization.cpp
+  SPIRV/SPIRVBinaryUtils.cpp
+  SPIRV/TranslateRegistration.cpp
+
+  DEPENDS
+  MLIRSPIRVSerializationGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSPIRV
+  MLIRSupport
+  MLIRTranslation
+  )
+
+add_mlir_translation_library(MLIRSPIRVDeserialization
+  SPIRV/Deserialization.cpp
+  SPIRV/SPIRVBinaryUtils.cpp
+  SPIRV/TranslateRegistration.cpp
+
+  DEPENDS
+  MLIRSPIRVSerializationGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSPIRV
+  MLIRSupport
+  MLIRTranslation
+  )

mlir/lib/Target/SPIRV/Deserialization.cpp

 //===- Deserializer.cpp - MLIR SPIR-V Deserialization ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the SPIR-V binary to MLIR SPIR-V module deserialization.
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/SPIRV/Serialization.h"
+#include "mlir/Target/SPIRV/Deserialization.h"
 
 #include "mlir/Dialect/SPIRV/SPIRVAttributes.h"
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"

mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp

 //===- SPIRVBinaryUtils.cpp - MLIR SPIR-V Binary Module Utilities ---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines common utilities for SPIR-V binary module.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 
 using namespace mlir;
 
 void spirv::appendModuleHeader(SmallVectorImpl<uint32_t> &header,
                                spirv::Version version, uint32_t idBound) {
   uint32_t majorVersion = 1;
   uint32_t minorVersion = 0;
   switch (version) {
 #define MIN_VERSION_CASE(v)                                                    \
   case spirv::Version::V_1_##v:                                                \
     minorVersion = v;                                                          \
     break
 
     MIN_VERSION_CASE(0);
     MIN_VERSION_CASE(1);
     MIN_VERSION_CASE(2);
     MIN_VERSION_CASE(3);
     MIN_VERSION_CASE(4);
     MIN_VERSION_CASE(5);
 #undef MIN_VERSION_CASE
   }
 
   // See "2.3. Physical Layout of a SPIR-V Module and Instruction" in the SPIR-V
   // spec for the definition of the binary module header.
   //
   // The first five words of a SPIR-V module must be:
   // +-------------------------------------------------------------------------+
   // | Magic number                                                            |
   // +-------------------------------------------------------------------------+
   // | Version number (bytes: 0 | major number | minor number | 0)             |
   // +-------------------------------------------------------------------------+
   // | Generator magic number                                                  |
   // +-------------------------------------------------------------------------+
   // | Bound (all result <id>s in the module guaranteed to be less than it)    |
   // +-------------------------------------------------------------------------+
   // | 0 (reserved for instruction schema)                                     |
   // +-------------------------------------------------------------------------+
   header.push_back(spirv::kMagicNumber);
   header.push_back((majorVersion << 16) | (minorVersion << 8));
   header.push_back(kGeneratorNumber);
   header.push_back(idBound); // <id> bound
   header.push_back(0);       // Schema (reserved word)
 }
 
 /// Returns the word-count-prefixed opcode for an SPIR-V instruction.
 uint32_t spirv::getPrefixedOpcode(uint32_t wordCount, spirv::Opcode opcode) {
   assert(((wordCount >> 16) == 0) && "word count out of range!");
   return (wordCount << 16) | static_cast<uint32_t>(opcode);
 }
 
 LogicalResult spirv::encodeStringLiteralInto(SmallVectorImpl<uint32_t> &binary,
                                              StringRef literal) {
   // We need to encode the literal and the null termination.
   auto encodingSize = literal.size() / 4 + 1;
   auto bufferStartSize = binary.size();
   binary.resize(bufferStartSize + encodingSize, 0);
   std::memcpy(binary.data() + bufferStartSize, literal.data(), literal.size());
   return success();
 }

mlir/lib/Target/SPIRV/Serialization.cpp

 //===- Serializer.cpp - MLIR SPIR-V Serialization -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the MLIR SPIR-V module to SPIR-V binary serialization.
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/SPIRV/Serialization.h"
+#include "mlir/Target/SPIRV/Serialization.h"
 
 #include "mlir/Dialect/SPIRV/SPIRVAttributes.h"
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/RegionGraphTraits.h"
LogicalResult Serializer::emitDebugLine(SmallVectorImpl<uint32_t> &binary,
 
   auto fileLoc = loc.dyn_cast<FileLineColLoc>();
   if (fileLoc)
     encodeInstructionInto(binary, spirv::Opcode::OpLine,
                           {fileID, fileLoc.getLine(), fileLoc.getColumn()});
   return success();
 }
 
+namespace mlir {
 LogicalResult spirv::serialize(spirv::ModuleOp module,
                                SmallVectorImpl<uint32_t> &binary,
                                bool emitDebugInfo) {
   if (!module.vce_triple().hasValue())
     return module.emitError(
         "module must have 'vce_triple' attribute to be serializeable");
 
   Serializer serializer(module, emitDebugInfo);
 
   if (failed(serializer.serialize()))
     return failure();
 
   LLVM_DEBUG(serializer.printValueIDMap(llvm::dbgs()));
 
   serializer.collect(binary);
   return success();
 }
+} // namespace mlir

mlir/lib/Target/SPIRV/TranslateRegistration.cpp

 //===- TranslateRegistration.cpp - hooks to mlir-translate ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements a translation from SPIR-V binary module to MLIR SPIR-V
 // ModuleOp.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
-#include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/Module.h"
 #include "mlir/Parser.h"
 #include "mlir/Support/FileUtilities.h"
+#include "mlir/Target/SPIRV/Deserialization.h"
+#include "mlir/Target/SPIRV/Serialization.h"
 #include "mlir/Translation.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ToolOutputFile.h"
 
 using namespace mlir;

mlir/unittests/Dialect/SPIRV/CMakeLists.txt

-add_mlir_unittest(MLIRSPIRVTests
-  DeserializationTest.cpp
-  SerializationTest.cpp
-)
-target_link_libraries(MLIRSPIRVTests
-  PRIVATE
-  MLIRSPIRV
-  MLIRSPIRVSerialization)
+add_subdirectory(Serialization)
+add_subdirectory(Deserialization)

mlir/unittests/Dialect/SPIRV/Deserialization/CMakeLists.txt

+add_mlir_unittest(MLIRSPIRVDeserializationTests
+  DeserializationTest.cpp
+)
+target_link_libraries(MLIRSPIRVDeserializationTests
+  PRIVATE
+  MLIRSPIRV
+  MLIRSPIRVDeserialization)

mlir/unittests/Dialect/SPIRV/Deserialization/DeserializationTest.cpp

 //===- DeserializationTest.cpp - SPIR-V Deserialization Tests -------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // The purpose of this file is to provide negative deserialization tests.
 // For positive deserialization tests, please use serialization and
 // deserialization for roundtripping.
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Target/SPIRV/Deserialization.h"
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
-#include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/MLIRContext.h"
 #include "gmock/gmock.h"
 
 #include <memory>
 
 using namespace mlir;
 

mlir/unittests/Dialect/SPIRV/Serialization/CMakeLists.txt

+add_mlir_unittest(MLIRSPIRVSerializationTests
+  SerializationTest.cpp
+)
+target_link_libraries(MLIRSPIRVSerializationTests
+  PRIVATE
+  MLIRSPIRV
+  MLIRSPIRVSerialization)

mlir/unittests/Dialect/SPIRV/Serialization/SerializationTest.cpp

 //===- SerializationTest.cpp - SPIR-V Serialization Tests -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains corner case tests for the SPIR-V serializer that are not
 // covered by normal serialization and deserialization roundtripping.
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/SPIRV/Serialization.h"
+#include "mlir/Target/SPIRV/Serialization.h"
 #include "mlir/Dialect/SPIRV/SPIRVAttributes.h"
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVModule.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Location.h"