Skip to content

Commit 2e6b2b5

Browse files
vchuravyvchuravy
committed
WIP: Profiler in Julia
1 parent 260a295 commit 2e6b2b5

File tree

8 files changed

+990
-1
lines changed

8 files changed

+990
-1
lines changed

‎gen/rocprof/generator.jl

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
using Clang.Generators
2+
using JuliaFormatter
3+
4+
include_dir = normpath("/opt/rocm/include")
5+
rocfft_dir = joinpath(include_dir, "rocprofiler/v2")
6+
options = load_options("rocprof/rocprof-generator.toml")
7+
8+
args = get_default_args()
9+
push!(args, "-I$include_dir")
10+
11+
headers = [
12+
joinpath(rocfft_dir, header)
13+
for header in readdir(rocfft_dir)
14+
if endswith(header, ".h")
15+
]
16+
17+
ctx = create_context(headers, args, options)
18+
build!(ctx)
19+
20+
path = options["general"]["output_file_path"]
21+
format_file(path, YASStyle())

‎gen/rocprof/rocprof-generator.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
[general]
2+
library_name = "librocprofiler64v2"
3+
output_file_path = "./librocprof.jl"
4+
export_symbol_prefixes = []
5+
6+
[codegen]
7+
use_ccall_macro = true

‎prof.jl

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# ENV["HSA_TOOLS_LIB"] = "/opt/rocm/lib/librocprofiler64v2.so"
2+
# ENV["ROCPROFILER_METRICS_PATH"] = "/opt/rocm/libexec/rocprofiler/counters/derived_counters.xml"
3+
4+
using AMDGPU
5+
import AMDGPU: Profiler, @check
6+
7+
@show Profiler.version()
8+
9+
# Application tracing
10+
11+
## Initialize tools
12+
@check Profiler.rocprofiler_initialize()
13+
14+
## create session with replay mode
15+
r_id = Ref{Profiler.rocprofiler_session_id_t}()
16+
@check Profiler.rocprofiler_create_session(Profiler.ROCPROFILER_NONE_REPLAY_MODE, r_id)
17+
id = r_id[]
18+
19+
## Create output buffer for the data
20+
function output_callback(record, end_record, session_id, buffer_id)
21+
@info "Output callback" record, end_record, session_id, buffer_id
22+
return nothing
23+
end
24+
25+
r_buffer_id = Ref{Profiler.rocprofiler_buffer_id_t}()
26+
@check Profiler.rocprofiler_create_buffer(
27+
id,
28+
@cfunction(output_callback, Cvoid, (
29+
Ptr{Profiler.rocprofiler_record_header_t},
30+
Ptr{Profiler.rocprofiler_record_header_t},
31+
Profiler.rocprofiler_session_id_t,
32+
Profiler.rocprofiler_buffer_id_t)),
33+
0x9999, r_buffer_id
34+
)
35+
buffer_id = r_buffer_id[]
36+
37+
# Specifying the APIs to be traced in a vector
38+
apis_requested = Vector{Profiler.rocprofiler_tracer_activity_domain_t}(undef, 0)
39+
push!(apis_requested, Profiler.ACTIVITY_DOMAIN_HIP_API)
40+
push!(apis_requested, Profiler.ACTIVITY_DOMAIN_HIP_OPS)
41+
push!(apis_requested, Profiler.ACTIVITY_DOMAIN_HSA_API)
42+
push!(apis_requested, Profiler.ACTIVITY_DOMAIN_HSA_OPS)
43+
push!(apis_requested, Profiler.ACTIVITY_DOMAIN_ROCTX)
44+
45+
GC.@preserve apis_requested begin
46+
# Looking at the code we can release the pointer after rocprofiler_create_filter
47+
filter_data = Ref{Profiler.rocprofiler_filter_data_t}()
48+
GC.@preserve filter_data begin
49+
ptr = Base.unsafe_convert(Ptr{Profiler.rocprofiler_filter_data_t}, filter_data)
50+
Base.memset(ptr, UInt8(0), sizeof(Profiler.rocprofiler_filter_data_t))
51+
52+
ptr.trace_apis = pointer(apis_requested)
53+
end
54+
55+
r_api_tracing_filter_id = Ref{Profiler.rocprofiler_filter_id_t}()
56+
filter_property = Ref{Profiler.rocprofiler_filter_property_t}()
57+
GC.@preserve filter_property begin
58+
ptr = Base.unsafe_convert(Ptr{Profiler.rocprofiler_filter_property_t}, filter_property)
59+
Base.memset(ptr, UInt8(0), sizeof(Profiler.rocprofiler_filter_property_t))
60+
end
61+
62+
# Creating filter for tracing APIs
63+
@check Profiler.rocprofiler_create_filter(
64+
id, Profiler.ROCPROFILER_API_TRACE,
65+
filter_data[], length(apis_requested),
66+
r_api_tracing_filter_id, filter_property[]
67+
)
68+
api_tracing_filter_id = r_api_tracing_filter_id[]
69+
end
70+
api_tracing_filter_id
71+
72+
function timestamp()
73+
r = Ref{Profiler.rocprofiler_timestamp_t}()
74+
Profiler.rocprofiler_get_timestamp(r)
75+
return r[].value
76+
end
77+
78+
function trace_sync_callback(record::Profiler.rocprofiler_record_tracer_t, session_id)
79+
if record.domain == Profiler.ACTIVITY_DOMAIN_HSA_API ||
80+
record.domain == Profiler.ACTIVITY_DOMAIN_HIP_API
81+
r_fn_name = Ref{Ptr{Cchar}}()
82+
83+
@check Profiler.rocprofiler_query_tracer_operation_name(
84+
record.domain, record.operation_id, r_fn_name
85+
)
86+
fn_name = Base.unsafe_string(r_fn_name[])
87+
else
88+
fn_name = nothing
89+
end
90+
91+
if record.phase == Profiler.ROCPROFILER_PHASE_ENTER
92+
ts_begin = timestamp()
93+
ts_end = nothing
94+
elseif record.phase == Profiler.ROCPROFILER_PHASE_EXIT
95+
ts_begin = nothing
96+
ts_end = timestamp()
97+
else
98+
ts_begin = record.timestamps._begin.value
99+
ts_end = record.timestamps._end.value
100+
end
101+
102+
if record.name != C_NULL
103+
name = Base.unsafe_string(record.name)
104+
else
105+
name = nothing
106+
end
107+
108+
@info "trace_sync" domain=record.domain phase=record.phase correlation=record.correlation_id.value ts_begin ts_end fn_name record.name
109+
return nothing
110+
end
111+
112+
@check Profiler.rocprofiler_set_filter_buffer(id, api_tracing_filter_id, buffer_id)
113+
@check Profiler.rocprofiler_set_api_trace_sync_callback(
114+
id, api_tracing_filter_id,
115+
@cfunction(trace_sync_callback, Cvoid, (
116+
Profiler.rocprofiler_record_tracer_t,
117+
Profiler.rocprofiler_session_id_t,
118+
)),
119+
)
120+
121+
# Kernel tracing
122+
filter_data = Ref{Profiler.rocprofiler_filter_data_t}()
123+
GC.@preserve filter_data begin
124+
ptr = Base.unsafe_convert(Ptr{Profiler.rocprofiler_filter_data_t}, filter_data)
125+
Base.memset(ptr, UInt8(0), sizeof(Profiler.rocprofiler_filter_data_t))
126+
end
127+
128+
r_kernel_tracing_filter_id = Ref{Profiler.rocprofiler_filter_id_t}()
129+
@check Profiler.rocprofiler_create_filter(
130+
id, Profiler.ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION,
131+
filter_data[], 0, r_kernel_tracing_filter_id, filter_property[]
132+
)
133+
kernel_tracing_filter_id = r_kernel_tracing_filter_id[]
134+
135+
function kernel()
136+
return nothing
137+
end
138+
139+
@roc kernel()
140+
141+
Profiler.rocprofiler_start_session(id)
142+
143+
@roc kernel()
144+
AMDGPU.synchronize()
145+
146+
Profiler.rocprofiler_terminate_session(id)
147+
Profiler.rocprofiler_flush_data(id, buffer_id)
148+
Profiler.rocprofiler_destroy_session(id)
149+
Profiler.rocprofiler_finalize()

‎src/AMDGPU.jl

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,9 @@ include("ROCKernels.jl")
140140
import .ROCKernels: ROCBackend
141141
export ROCBackend
142142

143+
include("profiler/profiler.jl")
144+
145+
143146
function __init__()
144147
# Used to shutdown hostcalls if any is running.
145148
atexit(() -> begin Runtime.RT_EXITING[] = true end)

‎src/discovery/discovery.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ function __init__()
103103
global librocrand = find_rocm_library(lib_prefix * "rocrand"; rocm_path)
104104
global librocfft = find_rocm_library(lib_prefix * "rocfft"; rocm_path)
105105
global libMIOpen_path = find_rocm_library(lib_prefix * "MIOpen"; rocm_path)
106+
global librocprofiler64v2 = get_library(lib_prefix * "rocprofiler64v2"; rocm_path)
106107
catch err
107108
@error """ROCm discovery failed!
108109
Discovered ROCm path: $rocm_path.

0 commit comments

Comments
 (0)