1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
//===--- Kernel.cpp - OpenMP device kernel interface -------------- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the kernel entry points for the device.
//
//===----------------------------------------------------------------------===//
#include "Debug.h"
#include "Interface.h"
#include "Mapping.h"
#include "State.h"
#include "Synchronization.h"
#include "Types.h"
#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
using namespace ompx;
#pragma omp begin declare target device_type(nohost)
static void inititializeRuntime(bool IsSPMD) {
// Order is important here.
synchronize::init(IsSPMD);
mapping::init(IsSPMD);
state::init(IsSPMD);
}
/// Simple generic state machine for worker threads.
static void genericStateMachine(IdentTy *Ident) {
FunctionTracingRAII();
uint32_t TId = mapping::getThreadIdInBlock();
do {
ParallelRegionFnTy WorkFn = nullptr;
// Wait for the signal that we have a new work function.
synchronize::threads(atomic::seq_cst);
// Retrieve the work function from the runtime.
bool IsActive = __kmpc_kernel_parallel(&WorkFn);
// If there is nothing more to do, break out of the state machine by
// returning to the caller.
if (!WorkFn)
return;
if (IsActive) {
ASSERT(!mapping::isSPMDMode());
((void (*)(uint32_t, uint32_t))WorkFn)(0, TId);
__kmpc_kernel_end_parallel();
}
synchronize::threads(atomic::seq_cst);
} while (true);
}
extern "C" {
/// Initialization
///
/// \param Ident Source location identification, can be NULL.
///
int32_t __kmpc_target_init(IdentTy *Ident, int8_t Mode,
bool UseGenericStateMachine) {
FunctionTracingRAII();
const bool IsSPMD =
Mode & llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD;
if (IsSPMD) {
inititializeRuntime(/* IsSPMD */ true);
synchronize::threadsAligned(atomic::relaxed);
} else {
inititializeRuntime(/* IsSPMD */ false);
// No need to wait since only the main threads will execute user
// code and workers will run into a barrier right away.
}
if (IsSPMD) {
state::assumeInitialState(IsSPMD);
// Synchronize to ensure the assertions above are in an aligned region.
// The barrier is eliminated later.
synchronize::threadsAligned(atomic::relaxed);
return -1;
}
if (mapping::isInitialThreadInLevel0(IsSPMD))
return -1;
// Enter the generic state machine if enabled and if this thread can possibly
// be an active worker thread.
//
// The latter check is important for NVIDIA Pascal (but not Volta) and AMD
// GPU. In those cases, a single thread can apparently satisfy a barrier on
// behalf of all threads in the same warp. Thus, it would not be safe for
// other threads in the main thread's warp to reach the first
// synchronize::threads call in genericStateMachine before the main thread
// reaches its corresponding synchronize::threads call: that would permit all
// active worker threads to proceed before the main thread has actually set
// state::ParallelRegionFn, and then they would immediately quit without
// doing any work. mapping::getBlockSize() does not include any of the main
// thread's warp, so none of its threads can ever be active worker threads.
if (UseGenericStateMachine &&
mapping::getThreadIdInBlock() < mapping::getBlockSize(IsSPMD)) {
genericStateMachine(Ident);
} else {
// Retrieve the work function just to ensure we always call
// __kmpc_kernel_parallel even if a custom state machine is used.
// TODO: this is not super pretty. The problem is we create the call to
// __kmpc_kernel_parallel in the openmp-opt pass but while we optimize it is
// not there yet. Thus, we assume we never reach it from
// __kmpc_target_deinit. That allows us to remove the store in there to
// ParallelRegionFn, which leads to bad results later on.
ParallelRegionFnTy WorkFn = nullptr;
__kmpc_kernel_parallel(&WorkFn);
ASSERT(WorkFn == nullptr);
}
return mapping::getThreadIdInBlock();
}
/// De-Initialization
///
/// In non-SPMD, this function releases the workers trapped in a state machine
/// and also any memory dynamically allocated by the runtime.
///
/// \param Ident Source location identification, can be NULL.
///
void __kmpc_target_deinit(IdentTy *Ident, int8_t Mode) {
FunctionTracingRAII();
const bool IsSPMD =
Mode & llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_SPMD;
if (IsSPMD)
return;
// Signal the workers to exit the state machine and exit the kernel.
state::ParallelRegionFn = nullptr;
}
int8_t __kmpc_is_spmd_exec_mode() {
FunctionTracingRAII();
return mapping::isSPMDMode();
}
}
#pragma omp end declare target
|
