szaszm commented on code in PR #2044: URL: https://github.com/apache/nifi-minifi-cpp/pull/2044#discussion_r2514153353
########## extensions/azure/tests/features/steps/azure_server_container.py: ########## @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +from docker.errors import ContainerError +from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.helpers import run_cmd_in_docker_image +from minifi_test_framework.core.helpers import wait_for_condition +from minifi_test_framework.core.minifi_test_context import MinifiTestContext + + +class AzureServerContainer(Container): + def __init__(self, test_context: MinifiTestContext): + super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0", + f"azure-storage-server-{test_context.scenario_id}", + test_context.network) + azure_storage_hostname = f"azure-storage-server-{test_context.scenario_id}" + self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \ + f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;"; + + def deploy(self): + super().deploy() + finished_str = "Azurite Queue service is successfully listening at" + return wait_for_condition(condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=None) + + def check_azure_storage_server_data(self, test_data): + (code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"]) + if code != 0: + return False + data_file = output.strip() + (code, file_data) = self.exec_run(["cat", data_file]) + return code == 0 and test_data in file_data + + def add_test_blob(self, blob_name, content="test_data", with_snapshot=False) -> bool: + cmd_create = (f'az storage container create --name "test-container" ' + f'--connection-string "{self.azure_connection_string}"') + try: + run_cmd_in_docker_image("mcr.microsoft.com/azure-cli", cmd_create, self.network.name) + except ContainerError as e: + logging.error(e) + return False + + cmd_upload = (f'az storage blob upload --container-name "test-container" ' + f'--name "{blob_name}" --data "{content}" ' + f'--connection-string "{self.azure_connection_string}"') + try: + run_cmd_in_docker_image("mcr.microsoft.com/azure-cli", cmd_upload, self.network.name) + except ContainerError as e: + logging.error(e) + return False + + if with_snapshot: + cmd_snapshot = (f'az storage blob snapshot --container-name "test-container" ' + f'--name "{blob_name}" ' + f'--connection-string "{self.azure_connection_string}"') + try: + run_cmd_in_docker_image("mcr.microsoft.com/azure-cli", cmd_snapshot, self.network.name) + except ContainerError as e: + logging.error(e) + return False + + return True + + def __get_blob_and_snapshot_count(self): + cmd = (f'az storage blob list --container-name "test-container" ' + f'--include deleted --query "length(@)" --output tsv ' + f'--connection-string "{self.azure_connection_string}"') Review Comment: ```suggestion cmd = ["az", "storage", "blob", "list", "--container-name", "test-container", "--include", "deleted", "--query", '"length(@)"', "--output", "tsv", "--connection-string", self.azure_connection_string] ``` ########## behave_framework/src/minifi_test_framework/core/helpers.py: ########## @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable + +import docker +from minifi_test_framework.core.minifi_test_context import MinifiTestContext + + +def log_due_to_failure(context: MinifiTestContext | None): + if context is not None: + for container in context.containers: + container.log_app_output() + context.minifi_container.log_app_output() + + +def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, bail_condition: Callable[[], bool], + context: MinifiTestContext | None) -> bool: + if bail_condition(): + logging.warning("Bail condition evaluated to 'True', aborting wait.") + log_due_to_failure(context) + return False + start_time = time.monotonic() + try: + while time.monotonic() - start_time < timeout_seconds: + if condition(): + return True + if bail_condition(): + logging.warning("Bail condition evaluated to 'True', aborting wait.") + log_due_to_failure(context) + return False + remaining_time = timeout_seconds - (time.monotonic() - start_time) + target_sleep = 1.0 if timeout_seconds < 30 else 10.0 + sleep_time = min(target_sleep, remaining_time) + if sleep_time > 0: + time.sleep(sleep_time) + except (Exception,): + logging.warning("Exception while waiting for condition") + log_due_to_failure(context) + return False + logging.warning("Timed out after %d seconds while waiting for condition", timeout_seconds) + log_due_to_failure(context) + return False + + +def run_cmd_in_docker_image(image_name: str, cmd: str, network: str) -> str: + client = docker.from_env() + output = client.containers.run(image=image_name, + command=["/bin/sh", "-c", cmd], + remove=True, + stdout=True, + stderr=True, + network=network, + detach=False) + return output.decode("utf-8") Review Comment: I would avoid forcing a shell layer on every caller when it's not necessary, and have cmd be directly forwarded as either a str or list to the command kwarg ```suggestion def run_cmd_in_docker_image(image_name: str, cmd: str|list, network: str) -> str: client = docker.from_env() output = client.containers.run(image=image_name, command=cmd, remove=True, stdout=True, stderr=True, network=network, detach=False) return output.decode("utf-8") def run_shell_cmd_in_docker_image(image_name: str, cmd: str, network: str) -> str: return run_cmd_in_docker_image(image_name, ["/bin/sh", "-c", cmd], network) ``` Also if @lordgamez has figured out how to fail on non-zero exit code, that would be a nice improvement. ########## extensions/azure/tests/features/steps/azure_server_container.py: ########## @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +from docker.errors import ContainerError +from minifi_test_framework.containers.container import Container +from minifi_test_framework.core.helpers import run_cmd_in_docker_image +from minifi_test_framework.core.helpers import wait_for_condition +from minifi_test_framework.core.minifi_test_context import MinifiTestContext + + +class AzureServerContainer(Container): + def __init__(self, test_context: MinifiTestContext): + super().__init__("mcr.microsoft.com/azure-storage/azurite:3.35.0", + f"azure-storage-server-{test_context.scenario_id}", + test_context.network) + azure_storage_hostname = f"azure-storage-server-{test_context.scenario_id}" + self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \ + f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;"; + + def deploy(self): + super().deploy() + finished_str = "Azurite Queue service is successfully listening at" + return wait_for_condition(condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=None) + + def check_azure_storage_server_data(self, test_data): + (code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"]) + if code != 0: + return False + data_file = output.strip() + (code, file_data) = self.exec_run(["cat", data_file]) + return code == 0 and test_data in file_data + + def add_test_blob(self, blob_name, content="test_data", with_snapshot=False) -> bool: + cmd_create = (f'az storage container create --name "test-container" ' + f'--connection-string "{self.azure_connection_string}"') + try: + run_cmd_in_docker_image("mcr.microsoft.com/azure-cli", cmd_create, self.network.name) + except ContainerError as e: + logging.error(e) + return False + + cmd_upload = (f'az storage blob upload --container-name "test-container" ' + f'--name "{blob_name}" --data "{content}" ' + f'--connection-string "{self.azure_connection_string}"') + try: + run_cmd_in_docker_image("mcr.microsoft.com/azure-cli", cmd_upload, self.network.name) + except ContainerError as e: + logging.error(e) + return False + + if with_snapshot: + cmd_snapshot = (f'az storage blob snapshot --container-name "test-container" ' + f'--name "{blob_name}" ' + f'--connection-string "{self.azure_connection_string}"') Review Comment: ```suggestion cmd_snapshot = ["az", "storage", "blob", "snapshot", "--container-name", "test-container", "--name", blob_name, "--connection-string", self.azure_connection_string] ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
