Skip to content

Commit c2380fd

Browse files
asamal4asamal4
committed
openai-e2e: change model
1 parent b48b0f7 commit c2380fd

File tree

3 files changed

+219
-24
lines changed

3 files changed

+219
-24
lines changed

‎tests/config/operator_install/olsconfig.crd.openai_introspection.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ spec:
1515
name: llmcreds
1616
models:
1717
- name: gpt-4o-mini
18+
- name: gpt-4o
19+
- name: gpt-4
20+
- name: gpt-4-turbo
21+
- name: gpt-4.1-nano
1822
name: openai
1923
type: openai
2024
ols:

‎tests/e2e/test_query_endpoint.py

Lines changed: 196 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,13 +199,117 @@ def test_too_long_question() -> None:
199199
@pytest.mark.introspection
200200
@pytest.mark.smoketest
201201
@pytest.mark.rag
202-
def test_valid_question() -> None:
202+
def test_valid_question_4o_mini() -> None:
203203
"""Check the REST API /v1/query with POST HTTP method for valid question and no yaml."""
204204
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
205205
cid = suid.get_suid()
206206
response = pytest.client.post(
207207
QUERY_ENDPOINT,
208-
json={"conversation_id": cid, "query": "what is kubernetes?"},
208+
json={
209+
"conversation_id": cid,
210+
"query": "what is kubernetes?",
211+
"provider": "openai",
212+
"model": "gpt-4o-mini",
213+
},
214+
timeout=test_api.LLM_REST_API_TIMEOUT,
215+
)
216+
assert response.status_code == requests.codes.ok
217+
218+
response_utils.check_content_type(response, "application/json")
219+
print(vars(response))
220+
json_response = response.json()
221+
222+
# checking a few major information from response
223+
assert json_response["conversation_id"] == cid
224+
assert "Kubernetes is" in json_response["response"]
225+
assert re.search(
226+
r"orchestration (tool|system|platform|engine)",
227+
json_response["response"],
228+
re.IGNORECASE,
229+
)
230+
assert json_response["input_tokens"] > 0
231+
assert json_response["output_tokens"] > 0
232+
233+
234+
@pytest.mark.introspection
235+
def test_valid_question_4o() -> None:
236+
"""Check the REST API /v1/query with POST HTTP method for valid question and no yaml."""
237+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
238+
cid = suid.get_suid()
239+
response = pytest.client.post(
240+
QUERY_ENDPOINT,
241+
json={
242+
"conversation_id": cid,
243+
"query": "what is kubernetes?",
244+
"provider": "openai",
245+
"model": "gpt-4o",
246+
},
247+
timeout=test_api.LLM_REST_API_TIMEOUT,
248+
)
249+
assert response.status_code == requests.codes.ok
250+
251+
response_utils.check_content_type(response, "application/json")
252+
print(vars(response))
253+
json_response = response.json()
254+
255+
# checking a few major information from response
256+
assert json_response["conversation_id"] == cid
257+
assert "Kubernetes is" in json_response["response"]
258+
assert re.search(
259+
r"orchestration (tool|system|platform|engine)",
260+
json_response["response"],
261+
re.IGNORECASE,
262+
)
263+
assert json_response["input_tokens"] > 0
264+
assert json_response["output_tokens"] > 0
265+
266+
267+
@pytest.mark.introspection
268+
def test_valid_question_4() -> None:
269+
"""Check the REST API /v1/query with POST HTTP method for valid question and no yaml."""
270+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
271+
cid = suid.get_suid()
272+
response = pytest.client.post(
273+
QUERY_ENDPOINT,
274+
json={
275+
"conversation_id": cid,
276+
"query": "what is kubernetes?",
277+
"provider": "openai",
278+
"model": "gpt-4",
279+
},
280+
timeout=test_api.LLM_REST_API_TIMEOUT,
281+
)
282+
assert response.status_code == requests.codes.ok
283+
284+
response_utils.check_content_type(response, "application/json")
285+
print(vars(response))
286+
json_response = response.json()
287+
288+
# checking a few major information from response
289+
assert json_response["conversation_id"] == cid
290+
assert "Kubernetes is" in json_response["response"]
291+
assert re.search(
292+
r"orchestration (tool|system|platform|engine)",
293+
json_response["response"],
294+
re.IGNORECASE,
295+
)
296+
assert json_response["input_tokens"] > 0
297+
assert json_response["output_tokens"] > 0
298+
299+
300+
@pytest.mark.introspection
301+
def test_valid_question_nano() -> None:
302+
"""Check the REST API /v1/query with POST HTTP method for valid question and no yaml."""
303+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
304+
cid = suid.get_suid()
305+
response = pytest.client.post(
306+
QUERY_ENDPOINT,
307+
json={
308+
"conversation_id": cid,
309+
"query": "what is kubernetes?",
310+
"provider": "openai",
311+
"model": "gpt-4.1-nano",
312+
},
209313
timeout=test_api.LLM_REST_API_TIMEOUT,
210314
)
211315
assert response.status_code == requests.codes.ok
@@ -348,7 +452,6 @@ def test_token_counters_for_query_call_with_improper_payload() -> None:
348452

349453
@pytest.mark.introspection
350454
@pytest.mark.rag
351-
@retry(max_attempts=3, wait_between_runs=10)
352455
def test_rag_question() -> None:
353456
"""Ensure responses include rag references."""
354457
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
@@ -585,8 +688,94 @@ def test_query_with_unknown_model() -> None:
585688

586689

587690
@pytest.mark.introspection
588-
@retry(max_attempts=3, wait_between_runs=10)
589-
def test_tool_calling() -> None:
691+
def test_tool_calling_4o_mini() -> None:
692+
"""Check the REST API /v1/query with POST HTTP method for tool calling."""
693+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
694+
cid = suid.get_suid()
695+
response = pytest.client.post(
696+
QUERY_ENDPOINT,
697+
json={
698+
"conversation_id": cid,
699+
"query": "show me pods in openshift-lightspeed namespace",
700+
"provider": "openai",
701+
"model": "gpt-4o-mini",
702+
},
703+
timeout=test_api.LLM_REST_API_TIMEOUT,
704+
)
705+
assert response.status_code == requests.codes.ok
706+
707+
response_utils.check_content_type(response, "application/json")
708+
print(vars(response))
709+
json_response = response.json()
710+
711+
# checking a few major information from response
712+
assert json_response["conversation_id"] == cid
713+
714+
assert "lightspeed-app-server" in json_response["response"].lower()
715+
assert json_response["input_tokens"] > 0
716+
assert json_response["output_tokens"] > 0
717+
718+
719+
@pytest.mark.introspection
720+
def test_tool_calling_4o() -> None:
721+
"""Check the REST API /v1/query with POST HTTP method for tool calling."""
722+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
723+
cid = suid.get_suid()
724+
response = pytest.client.post(
725+
QUERY_ENDPOINT,
726+
json={
727+
"conversation_id": cid,
728+
"query": "show me pods in openshift-lightspeed namespace",
729+
"provider": "openai",
730+
"model": "gpt-4o",
731+
},
732+
timeout=test_api.LLM_REST_API_TIMEOUT,
733+
)
734+
assert response.status_code == requests.codes.ok
735+
736+
response_utils.check_content_type(response, "application/json")
737+
print(vars(response))
738+
json_response = response.json()
739+
740+
# checking a few major information from response
741+
assert json_response["conversation_id"] == cid
742+
743+
assert "lightspeed-app-server" in json_response["response"].lower()
744+
assert json_response["input_tokens"] > 0
745+
assert json_response["output_tokens"] > 0
746+
747+
748+
@pytest.mark.introspection
749+
def test_tool_calling_4() -> None:
750+
"""Check the REST API /v1/query with POST HTTP method for tool calling."""
751+
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
752+
cid = suid.get_suid()
753+
response = pytest.client.post(
754+
QUERY_ENDPOINT,
755+
json={
756+
"conversation_id": cid,
757+
"query": "show me pods in openshift-lightspeed namespace",
758+
"provider": "openai",
759+
"model": "gpt-4",
760+
},
761+
timeout=test_api.LLM_REST_API_TIMEOUT,
762+
)
763+
assert response.status_code == requests.codes.ok
764+
765+
response_utils.check_content_type(response, "application/json")
766+
print(vars(response))
767+
json_response = response.json()
768+
769+
# checking a few major information from response
770+
assert json_response["conversation_id"] == cid
771+
772+
assert "lightspeed-app-server" in json_response["response"].lower()
773+
assert json_response["input_tokens"] > 0
774+
assert json_response["output_tokens"] > 0
775+
776+
777+
@pytest.mark.introspection
778+
def test_tool_calling_nano() -> None:
590779
"""Check the REST API /v1/query with POST HTTP method for tool calling."""
591780
with metrics_utils.RestAPICallCounterChecker(pytest.metrics_client, QUERY_ENDPOINT):
592781
cid = suid.get_suid()
@@ -595,6 +784,8 @@ def test_tool_calling() -> None:
595784
json={
596785
"conversation_id": cid,
597786
"query": "show me pods in openshift-lightspeed namespace",
787+
"provider": "openai",
788+
"model": "gpt-4.1-nano",
598789
},
599790
timeout=test_api.LLM_REST_API_TIMEOUT,
600791
)

‎tests/scripts/test-e2e-cluster.sh

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -37,35 +37,35 @@ function run_suites() {
3737
# runsuite arguments:
3838
# suiteid test_tags provider provider_keypath model ols_image
3939
# empty test_tags means run all tests
40-
run_suite "azure_openai" "not certificates and not (introspection and not smoketest and not rag)" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "n"
41-
(( rc = rc || $? ))
42-
43-
# # BAM is currently not working, commenting for now
44-
# run_suite "bam" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "bam" "$BAM_PROVIDER_KEY_PATH" "ibm/granite-3-8b-instruct" "$OLS_IMAGE" "n"
40+
# run_suite "azure_openai" "not certificates and not (introspection and not smoketest and not rag)" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "n"
4541
# (( rc = rc || $? ))
4642

47-
run_suite "openai" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "n"
48-
(( rc = rc || $? ))
43+
# # # BAM is currently not working, commenting for now
44+
# # run_suite "bam" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "bam" "$BAM_PROVIDER_KEY_PATH" "ibm/granite-3-8b-instruct" "$OLS_IMAGE" "n"
45+
# # (( rc = rc || $? ))
4946

50-
run_suite "watsonx" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-3-8b-instruct" "$OLS_IMAGE" "n"
51-
(( rc = rc || $? ))
47+
# run_suite "openai" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "n"
48+
# (( rc = rc || $? ))
5249

53-
# smoke tests for RHOAI VLLM-compatible provider
54-
run_suite "rhoai_vllm" "smoketest" "rhoai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "n"
55-
(( rc = rc || $? ))
50+
# run_suite "watsonx" "not azure_entra_id and not certificates and not (introspection and not smoketest and not rag)" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-3-8b-instruct" "$OLS_IMAGE" "n"
51+
# (( rc = rc || $? ))
5652

57-
# smoke tests for RHELAI VLLM-compatible provider
58-
run_suite "rhelai_vllm" "smoketest" "rhelai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "n"
59-
(( rc = rc || $? ))
53+
# # smoke tests for RHOAI VLLM-compatible provider
54+
# run_suite "rhoai_vllm" "smoketest" "rhoai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "n"
55+
# (( rc = rc || $? ))
56+
57+
# # smoke tests for RHELAI VLLM-compatible provider
58+
# run_suite "rhelai_vllm" "smoketest" "rhelai_vllm" "$OPENAI_PROVIDER_KEY_PATH" "gpt-3.5-turbo" "$OLS_IMAGE" "n"
59+
# (( rc = rc || $? ))
6060

6161
# TODO: Reduce execution time. Sequential execution will take more time. Parallel execution will have cluster claim issue.
6262
# Run tool calling - Enable introspection
63-
run_suite "azure_openai_introspection" "introspection" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "y"
64-
(( rc = rc || $? ))
63+
# run_suite "azure_openai_introspection" "introspection" "azure_openai" "$AZUREOPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "y"
64+
# (( rc = rc || $? ))
6565
run_suite "openai_introspection" "introspection" "openai" "$OPENAI_PROVIDER_KEY_PATH" "gpt-4o-mini" "$OLS_IMAGE" "y"
6666
(( rc = rc || $? ))
67-
run_suite "watsonx_introspection" "introspection" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-3-2-8b-instruct" "$OLS_IMAGE" "y"
68-
(( rc = rc || $? ))
67+
# run_suite "watsonx_introspection" "introspection" "watsonx" "$WATSONX_PROVIDER_KEY_PATH" "ibm/granite-3-2-8b-instruct" "$OLS_IMAGE" "y"
68+
# (( rc = rc || $? ))
6969

7070
set -e
7171

0 commit comments

Comments
 (0)