This is an archive of the discontinued LLVM Phabricator instance.

Differential D125432

[mlir][sparse] Moved _mlir_ciface_newSparseTensor closer to its macros
ClosedPublic

Authored by wrengr on May 11 2022, 4:41 PM.

Details

Reviewers
aartbik
bixia
penpornk
Commits
rGbfadd13df474: [mlir][sparse] Moved _mlir_ciface_newSparseTensor closer to its macros
Summary

This is a followup to D125431, to keep from confusing the machinery that generates diffs (since combining these two changes into one would obfuscate the changes actually made in the previous differential).

Depends On D125431

Diff Detail

Event Timeline

wrengr created this revision.May 11 2022, 4:41 PM
Herald added a project: Restricted Project. · View Herald TranscriptMay 11 2022, 4:41 PM
Herald added subscribers: sdasgup3, wenzhicui, Chia-hungDuan and 17 others. · View Herald Transcript
wrengr requested review of this revision.May 11 2022, 4:41 PM
Herald added a project: Restricted Project. · View Herald TranscriptMay 11 2022, 4:41 PM
Herald added subscribers: stephenneuendorffer, nicolasvasilache. · View Herald Transcript
Harbormaster completed remote builds in B164012: Diff 428814.May 11 2022, 7:23 PM
wrengr updated this revision to Diff 429031.May 12 2022, 11:26 AM

(rebasing for dependencies having rerun git-clang-format)

Harbormaster completed remote builds in B164166: Diff 429031.May 12 2022, 1:12 PM
aartbik accepted this revision.May 16 2022, 1:20 PM

Good to go after the required rebasing and merging.

This revision is now accepted and ready to land.May 16 2022, 1:20 PM
wrengr updated this revision to Diff 429897.May 16 2022, 4:46 PM

rebasing

Harbormaster completed remote builds in B164777: Diff 429897.May 16 2022, 5:05 PM
wrengr updated this revision to Diff 429903.May 16 2022, 5:05 PM

correcting a usage of tensor instead of coo

Harbormaster completed remote builds in B164780: Diff 429903.May 16 2022, 5:24 PM
Closed by commit rGbfadd13df474: [mlir][sparse] Moved _mlir_ciface_newSparseTensor closer to its macros (authored by wrengr). · Explain WhyMay 16 2022, 5:53 PM
This revision was automatically updated to reflect the committed changes.
wrengr added a commit: rGbfadd13df474: [mlir][sparse] Moved _mlir_ciface_newSparseTensor closer to its macros.

Revision Contents

PathSize
mlir/
lib/
ExecutionEngine/
270 lines

Diff 429903

mlir/lib/ExecutionEngine/SparseTensorUtils.cpp

Show First 20 Lines • Show All 1,404 Lines • ▼ Show 20 Lines if (action == Action::kToIterator) { \
coo->startIterator(); \ coo->startIterator(); \
} else { \ } else { \
assert(action == Action::kToCOO); \ assert(action == Action::kToCOO); \
} \ } \
return coo; \ return coo; \
} }
#define CASE_SECSAME(p, v, P, V) CASE(p, p, v, P, P, V) #define CASE_SECSAME(p, v, P, V) CASE(p, p, v, P, P, V)
// TODO(D125432): move `_mlir_ciface_newSparseTensor` closer to these
// macro definitions, but as a separate change so as not to muddy the diff. // Assume index_type is in fact uint64_t, so that _mlir_ciface_newSparseTensor
// can safely rewrite kIndex to kU64. We make this assertion to guarantee
// that this file cannot get out of sync with its header.
static_assert(std::is_same<index_type, uint64_t>::value,
"Expected index_type == uint64_t");
/// Constructs a new sparse tensor. This is the "swiss army knife"
/// method for materializing sparse tensors into the computation.
///
/// Action:
/// kEmpty = returns empty storage to fill later
/// kFromFile = returns storage, where ptr contains filename to read
/// kFromCOO = returns storage, where ptr contains coordinate scheme to assign
/// kEmptyCOO = returns empty coordinate scheme to fill and use with kFromCOO
/// kToCOO = returns coordinate scheme from storage in ptr to use with kFromCOO
/// kToIterator = returns iterator from storage in ptr (call getNext() to use)
void *
_mlir_ciface_newSparseTensor(StridedMemRefType<DimLevelType, 1> *aref, // NOLINT
StridedMemRefType<index_type, 1> *sref,
StridedMemRefType<index_type, 1> *pref,
OverheadType ptrTp, OverheadType indTp,
PrimaryType valTp, Action action, void *ptr) {
assert(aref && sref && pref);
assert(aref->strides[0] == 1 && sref->strides[0] == 1 &&
pref->strides[0] == 1);
assert(aref->sizes[0] == sref->sizes[0] && sref->sizes[0] == pref->sizes[0]);
const DimLevelType *sparsity = aref->data + aref->offset;
const index_type *shape = sref->data + sref->offset;
const index_type *perm = pref->data + pref->offset;
uint64_t rank = aref->sizes[0];
// Rewrite kIndex to kU64, to avoid introducing a bunch of new cases.
// This is safe because of the static_assert above.
if (ptrTp == OverheadType::kIndex)
ptrTp = OverheadType::kU64;
if (indTp == OverheadType::kIndex)
indTp = OverheadType::kU64;
// Double matrices with all combinations of overhead storage.
CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF64, uint64_t,
uint64_t, double);
CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF64, uint64_t,
uint32_t, double);
CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF64, uint64_t,
uint16_t, double);
CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF64, uint64_t,
uint8_t, double);
CASE(OverheadType::kU32, OverheadType::kU64, PrimaryType::kF64, uint32_t,
uint64_t, double);
CASE(OverheadType::kU32, OverheadType::kU32, PrimaryType::kF64, uint32_t,
uint32_t, double);
CASE(OverheadType::kU32, OverheadType::kU16, PrimaryType::kF64, uint32_t,
uint16_t, double);
CASE(OverheadType::kU32, OverheadType::kU8, PrimaryType::kF64, uint32_t,
uint8_t, double);
CASE(OverheadType::kU16, OverheadType::kU64, PrimaryType::kF64, uint16_t,
uint64_t, double);
CASE(OverheadType::kU16, OverheadType::kU32, PrimaryType::kF64, uint16_t,
uint32_t, double);
CASE(OverheadType::kU16, OverheadType::kU16, PrimaryType::kF64, uint16_t,
uint16_t, double);
CASE(OverheadType::kU16, OverheadType::kU8, PrimaryType::kF64, uint16_t,
uint8_t, double);
CASE(OverheadType::kU8, OverheadType::kU64, PrimaryType::kF64, uint8_t,
uint64_t, double);
CASE(OverheadType::kU8, OverheadType::kU32, PrimaryType::kF64, uint8_t,
uint32_t, double);
CASE(OverheadType::kU8, OverheadType::kU16, PrimaryType::kF64, uint8_t,
uint16_t, double);
CASE(OverheadType::kU8, OverheadType::kU8, PrimaryType::kF64, uint8_t,
uint8_t, double);
// Float matrices with all combinations of overhead storage.
CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF32, uint64_t,
uint64_t, float);
CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF32, uint64_t,
uint32_t, float);
CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF32, uint64_t,
uint16_t, float);
CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF32, uint64_t,
uint8_t, float);
CASE(OverheadType::kU32, OverheadType::kU64, PrimaryType::kF32, uint32_t,
uint64_t, float);
CASE(OverheadType::kU32, OverheadType::kU32, PrimaryType::kF32, uint32_t,
uint32_t, float);
CASE(OverheadType::kU32, OverheadType::kU16, PrimaryType::kF32, uint32_t,
uint16_t, float);
CASE(OverheadType::kU32, OverheadType::kU8, PrimaryType::kF32, uint32_t,
uint8_t, float);
CASE(OverheadType::kU16, OverheadType::kU64, PrimaryType::kF32, uint16_t,
uint64_t, float);
CASE(OverheadType::kU16, OverheadType::kU32, PrimaryType::kF32, uint16_t,
uint32_t, float);
CASE(OverheadType::kU16, OverheadType::kU16, PrimaryType::kF32, uint16_t,
uint16_t, float);
CASE(OverheadType::kU16, OverheadType::kU8, PrimaryType::kF32, uint16_t,
uint8_t, float);
CASE(OverheadType::kU8, OverheadType::kU64, PrimaryType::kF32, uint8_t,
uint64_t, float);
CASE(OverheadType::kU8, OverheadType::kU32, PrimaryType::kF32, uint8_t,
uint32_t, float);
CASE(OverheadType::kU8, OverheadType::kU16, PrimaryType::kF32, uint8_t,
uint16_t, float);
CASE(OverheadType::kU8, OverheadType::kU8, PrimaryType::kF32, uint8_t,
uint8_t, float);
// Integral matrices with both overheads of the same type.
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI64, uint64_t, int64_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI32, uint64_t, int32_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI16, uint64_t, int16_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI8, uint64_t, int8_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI32, uint32_t, int32_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI16, uint32_t, int16_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI8, uint32_t, int8_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI32, uint16_t, int32_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI16, uint16_t, int16_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI8, uint16_t, int8_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI32, uint8_t, int32_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI16, uint8_t, int16_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI8, uint8_t, int8_t);
// Complex matrices with wide overhead.
CASE_SECSAME(OverheadType::kU64, PrimaryType::kC64, uint64_t, complex64);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kC32, uint64_t, complex32);
// Unsupported case (add above if needed).
fputs("unsupported combination of types\n", stderr);
exit(1);
}
#undef CASE
#undef CASE_SECSAME
/// Methods that provide direct access to values. /// Methods that provide direct access to values.
#define IMPL_SPARSEVALUES(VNAME, V) \ #define IMPL_SPARSEVALUES(VNAME, V) \
void _mlir_ciface_sparseValues##VNAME(StridedMemRefType<V, 1> *ref, \ void _mlir_ciface_sparseValues##VNAME(StridedMemRefType<V, 1> *ref, \
void *tensor) { \ void *tensor) { \
assert(ref &&tensor); \ assert(ref &&tensor); \
std::vector<V> *v; \ std::vector<V> *v; \
static_cast<SparseTensorStorageBase *>(tensor)->getValues(&v); \ static_cast<SparseTensorStorageBase *>(tensor)->getValues(&v); \
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines void *_mlir_ciface_addElt##VNAME(void *coo, V value, \
static_cast<SparseTensorCOO<V> *>(coo)->add(indices, value); \ static_cast<SparseTensorCOO<V> *>(coo)->add(indices, value); \
return coo; \ return coo; \
} }
FOREVERY_SIMPLEX_V(IMPL_ADDELT) FOREVERY_SIMPLEX_V(IMPL_ADDELT)
// `complex64` apparently doesn't encounter any ABI issues (yet). // `complex64` apparently doesn't encounter any ABI issues (yet).
IMPL_ADDELT(C64, complex64) IMPL_ADDELT(C64, complex64)
// TODO: cleaner way to avoid ABI padding problem? // TODO: cleaner way to avoid ABI padding problem?
IMPL_ADDELT(C32ABI, complex32) IMPL_ADDELT(C32ABI, complex32)
void *_mlir_ciface_addEltC32(void *tensor, float r, float i, void *_mlir_ciface_addEltC32(void *coo, float r, float i,
StridedMemRefType<index_type, 1> *iref, StridedMemRefType<index_type, 1> *iref,
StridedMemRefType<index_type, 1> *pref) { StridedMemRefType<index_type, 1> *pref) {
return _mlir_ciface_addEltC32ABI(tensor, complex32(r, i), iref, pref); return _mlir_ciface_addEltC32ABI(coo, complex32(r, i), iref, pref);
} }
#undef IMPL_ADDELT #undef IMPL_ADDELT
/// Helper to enumerate elements of coordinate scheme, one per value type. /// Helper to enumerate elements of coordinate scheme, one per value type.
#define IMPL_GETNEXT(VNAME, V) \ #define IMPL_GETNEXT(VNAME, V) \
bool _mlir_ciface_getNext##VNAME(void *coo, \ bool _mlir_ciface_getNext##VNAME(void *coo, \
StridedMemRefType<index_type, 1> *iref, \ StridedMemRefType<index_type, 1> *iref, \
StridedMemRefType<V, 0> *vref) { \ StridedMemRefType<V, 0> *vref) { \
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines void _mlir_ciface_expInsert##VNAME( \
bool *filled = fref->data + fref->offset; \ bool *filled = fref->data + fref->offset; \
index_type *added = aref->data + aref->offset; \ index_type *added = aref->data + aref->offset; \
static_cast<SparseTensorStorageBase *>(tensor)->expInsert( \ static_cast<SparseTensorStorageBase *>(tensor)->expInsert( \
cursor, values, filled, added, count); \ cursor, values, filled, added, count); \
} }
FOREVERY_V(IMPL_EXPINSERT) FOREVERY_V(IMPL_EXPINSERT)
#undef IMPL_EXPINSERT #undef IMPL_EXPINSERT
// Assume index_type is in fact uint64_t, so that _mlir_ciface_newSparseTensor
// can safely rewrite kIndex to kU64. We make this assertion to guarantee
// that this file cannot get out of sync with its header.
static_assert(std::is_same<index_type, uint64_t>::value,
"Expected index_type == uint64_t");
/// Constructs a new sparse tensor. This is the "swiss army knife"
/// method for materializing sparse tensors into the computation.
///
/// Action:
/// kEmpty = returns empty storage to fill later
/// kFromFile = returns storage, where ptr contains filename to read
/// kFromCOO = returns storage, where ptr contains coordinate scheme to assign
/// kEmptyCOO = returns empty coordinate scheme to fill and use with kFromCOO
/// kToCOO = returns coordinate scheme from storage in ptr to use with kFromCOO
/// kToIterator = returns iterator from storage in ptr (call getNext() to use)
void *
_mlir_ciface_newSparseTensor(StridedMemRefType<DimLevelType, 1> *aref, // NOLINT
StridedMemRefType<index_type, 1> *sref,
StridedMemRefType<index_type, 1> *pref,
OverheadType ptrTp, OverheadType indTp,
PrimaryType valTp, Action action, void *ptr) {
assert(aref && sref && pref);
assert(aref->strides[0] == 1 && sref->strides[0] == 1 &&
pref->strides[0] == 1);
assert(aref->sizes[0] == sref->sizes[0] && sref->sizes[0] == pref->sizes[0]);
const DimLevelType *sparsity = aref->data + aref->offset;
const index_type *shape = sref->data + sref->offset;
const index_type *perm = pref->data + pref->offset;
uint64_t rank = aref->sizes[0];
// Rewrite kIndex to kU64, to avoid introducing a bunch of new cases.
// This is safe because of the static_assert above.
if (ptrTp == OverheadType::kIndex)
ptrTp = OverheadType::kU64;
if (indTp == OverheadType::kIndex)
indTp = OverheadType::kU64;
// Double matrices with all combinations of overhead storage.
CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF64, uint64_t,
uint64_t, double);
CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF64, uint64_t,
uint32_t, double);
CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF64, uint64_t,
uint16_t, double);
CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF64, uint64_t,
uint8_t, double);
CASE(OverheadType::kU32, OverheadType::kU64, PrimaryType::kF64, uint32_t,
uint64_t, double);
CASE(OverheadType::kU32, OverheadType::kU32, PrimaryType::kF64, uint32_t,
uint32_t, double);
CASE(OverheadType::kU32, OverheadType::kU16, PrimaryType::kF64, uint32_t,
uint16_t, double);
CASE(OverheadType::kU32, OverheadType::kU8, PrimaryType::kF64, uint32_t,
uint8_t, double);
CASE(OverheadType::kU16, OverheadType::kU64, PrimaryType::kF64, uint16_t,
uint64_t, double);
CASE(OverheadType::kU16, OverheadType::kU32, PrimaryType::kF64, uint16_t,
uint32_t, double);
CASE(OverheadType::kU16, OverheadType::kU16, PrimaryType::kF64, uint16_t,
uint16_t, double);
CASE(OverheadType::kU16, OverheadType::kU8, PrimaryType::kF64, uint16_t,
uint8_t, double);
CASE(OverheadType::kU8, OverheadType::kU64, PrimaryType::kF64, uint8_t,
uint64_t, double);
CASE(OverheadType::kU8, OverheadType::kU32, PrimaryType::kF64, uint8_t,
uint32_t, double);
CASE(OverheadType::kU8, OverheadType::kU16, PrimaryType::kF64, uint8_t,
uint16_t, double);
CASE(OverheadType::kU8, OverheadType::kU8, PrimaryType::kF64, uint8_t,
uint8_t, double);
// Float matrices with all combinations of overhead storage.
CASE(OverheadType::kU64, OverheadType::kU64, PrimaryType::kF32, uint64_t,
uint64_t, float);
CASE(OverheadType::kU64, OverheadType::kU32, PrimaryType::kF32, uint64_t,
uint32_t, float);
CASE(OverheadType::kU64, OverheadType::kU16, PrimaryType::kF32, uint64_t,
uint16_t, float);
CASE(OverheadType::kU64, OverheadType::kU8, PrimaryType::kF32, uint64_t,
uint8_t, float);
CASE(OverheadType::kU32, OverheadType::kU64, PrimaryType::kF32, uint32_t,
uint64_t, float);
CASE(OverheadType::kU32, OverheadType::kU32, PrimaryType::kF32, uint32_t,
uint32_t, float);
CASE(OverheadType::kU32, OverheadType::kU16, PrimaryType::kF32, uint32_t,
uint16_t, float);
CASE(OverheadType::kU32, OverheadType::kU8, PrimaryType::kF32, uint32_t,
uint8_t, float);
CASE(OverheadType::kU16, OverheadType::kU64, PrimaryType::kF32, uint16_t,
uint64_t, float);
CASE(OverheadType::kU16, OverheadType::kU32, PrimaryType::kF32, uint16_t,
uint32_t, float);
CASE(OverheadType::kU16, OverheadType::kU16, PrimaryType::kF32, uint16_t,
uint16_t, float);
CASE(OverheadType::kU16, OverheadType::kU8, PrimaryType::kF32, uint16_t,
uint8_t, float);
CASE(OverheadType::kU8, OverheadType::kU64, PrimaryType::kF32, uint8_t,
uint64_t, float);
CASE(OverheadType::kU8, OverheadType::kU32, PrimaryType::kF32, uint8_t,
uint32_t, float);
CASE(OverheadType::kU8, OverheadType::kU16, PrimaryType::kF32, uint8_t,
uint16_t, float);
CASE(OverheadType::kU8, OverheadType::kU8, PrimaryType::kF32, uint8_t,
uint8_t, float);
// Integral matrices with both overheads of the same type.
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI64, uint64_t, int64_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI32, uint64_t, int32_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI16, uint64_t, int16_t);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kI8, uint64_t, int8_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI32, uint32_t, int32_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI16, uint32_t, int16_t);
CASE_SECSAME(OverheadType::kU32, PrimaryType::kI8, uint32_t, int8_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI32, uint16_t, int32_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI16, uint16_t, int16_t);
CASE_SECSAME(OverheadType::kU16, PrimaryType::kI8, uint16_t, int8_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI32, uint8_t, int32_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI16, uint8_t, int16_t);
CASE_SECSAME(OverheadType::kU8, PrimaryType::kI8, uint8_t, int8_t);
// Complex matrices with wide overhead.
CASE_SECSAME(OverheadType::kU64, PrimaryType::kC64, uint64_t, complex64);
CASE_SECSAME(OverheadType::kU64, PrimaryType::kC32, uint64_t, complex32);
// Unsupported case (add above if needed).
fputs("unsupported combination of types\n", stderr);
exit(1);
}
#undef CASE
#undef CASE_SECSAME
/// Output a sparse tensor, one per value type. /// Output a sparse tensor, one per value type.
#define IMPL_OUTSPARSETENSOR(VNAME, V) \ #define IMPL_OUTSPARSETENSOR(VNAME, V) \
void outSparseTensor##VNAME(void *coo, void *dest, bool sort) { \ void outSparseTensor##VNAME(void *coo, void *dest, bool sort) { \
return outSparseTensor<V>(coo, dest, sort); \ return outSparseTensor<V>(coo, dest, sort); \
} }
FOREVERY_V(IMPL_OUTSPARSETENSOR) FOREVERY_V(IMPL_OUTSPARSETENSOR)
#undef IMPL_OUTSPARSETENSOR #undef IMPL_OUTSPARSETENSOR
▲ Show 20 Lines • Show All 110 Lines • Show Last 20 Lines