Genie Profile¶
Genie Profile APIs provides means to profile performance metrics of GenAI models. Profiling can be enabled on genie-t2t-run and genie-t2e-run by passing the “–profile FILENAME” argument. This dumps the collected profiling data to a JSON file with the provided name.
Profiling JSON Output Schema¶
{
"header": {
"type": "object",
"properties": {
"header_version": {
"type": "object",
"properties": {
"major": {"type": "integer"},
"minor": {"type": "integer"},
"patch": {"type": "integer"}
},
"required": [
"major",
"minor",
"patch"
]
},
"version": {
"type": "object",
"properties": {
"major": {"type": "integer"},
"minor": {"type": "integer"},
"patch": {"type": "integer"}
},
"required": [
"major",
"minor",
"patch"
]
},
"artifact_type": { "type": "string" }
},
"required": [
"header_version",
"version",
"artifact_type"
]
},
"metadata": {
"type": "object",
"properties": {
"timestamp": {"type": "integer"}
},
"required": [
"timestamp"
]
},
"components": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {"type": "string"},
"type": {"type": "string", "enum" : ["dialog", "embedding"]},
"events": {
"type": "array",
"items": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum" : [
"GenieDialog_create", "GenieDialog_query", "GenieDialog_free",
"GenieEmbedding_create", "GenieEmbedding_generate","GenieEmbedding_free"
]
},
"duration": {"type": "integer"},
"start": {"type": "integer"},
"stop": {"type": "integer"},
"init-time": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"num-prompt-tokens": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"prompt-processing-rate": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"time-to-first-token": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"num-generated-tokens": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"token-generation-rate": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
},
"token-generation-time": {
"type": "object",
"properties": {
"value": {"type": "integer"},
"unit": {"type": "string"}
},
"required": [
"value",
"unit"
]
}
}
},
"required": [
"type",
"duration",
"start",
"stop"
]
}
}
},
"required": [
"name",
"type",
"events"
]
},
"traceEvents" : {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {"type": "string"},
"cat": {"type": "string", "enum": ["function"]},
"dur": {"type": "integer"},
"ph": {"type": "string", "enum": ["X"]},
"pid": {"type": "integer"},
"tid": {"type": "integer"},
"ts": {"type": "integer"},
"args": {
"type": "object",
"properties": {
"stackDepth": {"type": "integer"}
}
}
}
}
}
}
Example on how to use Profiling APIs¶
# Create Profile Handle
GenieProfile_Handle_t profileHandle = NULL;
GenieProfile_create(nullptr, &profileHandle);
# Create Dialog Config
GenieDialogConfig_Handle_t dialogConfigHandle = NULL;
GenieDialogConfig_createFromJson(dialogConfigStr, &dialogConfigHandle);
# Bind Profile Handle to Dialog Config
GenieDialogConfig_bindProfiler(dialogConfigHandle, profileHandle);
# Create Dialog
GenieDialog_Handle_t dialogHandle = NULL;
GenieDialog_create(dialogConfigHandle, &dialogHandle);
# Run Dialog Query API
GenieDialog_query(dialogHandle, promptStr, GenieDialog_SentenceCode_t::GENIE_DIALOG_SENTENCE_COMPLETE, queryCallback);
# Get Profiling Data
const char* jsonData = nullptr;
const Genie_AllocCallback_t callback([](size_t size, const char** data) {
*data = (char*)malloc(size);
});
GenieProfile_getJsonData(profileHandle, callback, &jsonData);
# Dump to JSON File and Free the Allocated Data
std::ofstream outFile;
outFile.open(profilePath);
outFile << jsonData;
outFile.close();
free((char*)jsonData);
# Free Profile Handle
GenieProfile_free(profileHandle);
Trace Profiling Example¶
Tracing can be enabled on profile handles in order to collect detailed event data. When tracing is enabled, The JSON output from GenieProfile_getJsonData contains a “traceEvents” section which adheres to the Chrometrace JSON schema. The resultant JSON file can be loaded into Chrometrace viewer tools, such as chrome://tracing on Chrome-based web browsers, or Perfetto Trace Visualizer.
The following section demonstrates how to enable trace profiling in genie-app with the example profile config located at ${SDK_ROOT}/examples/Genie/configs/profile/trace.json:
# genie-app trace profiling script
profile config create pconfig1 path/to/trace.json
profile create profile1 pconfig1
dialog config create config1 path/to/dialogConfig.json
dialog config bind profile config1 profile1
dialog create dialog1 config1
dialog query dialog1 "Here's a summary of llamas in 10 words."
profile save profile1 profile_trace.json
# Clean up
dialog config free config1
dialog free dialog1
profile config free pconfig1
profile free profile1
The script will generate a profile_trace.json which can be directly loaded into Chrometrace viewers:
