From 9d311c501b47c3055447832ede2809da9c5b6855 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 19:55:16 -0600 Subject: [PATCH 1/3] add redis to uses_services plugin --- .../orquesta_runner/tests/integration/BUILD | 1 + pants-plugins/uses_services/BUILD | 1 + pants-plugins/uses_services/redis_rules.py | 168 ++++++++++++++++++ .../uses_services/redis_rules_test.py | 90 ++++++++++ .../uses_services/scripts/is_redis_running.py | 48 +++++ st2tests/integration/orquesta/BUILD | 1 + 6 files changed, 309 insertions(+) create mode 100644 pants-plugins/uses_services/redis_rules.py create mode 100644 pants-plugins/uses_services/redis_rules_test.py create mode 100644 pants-plugins/uses_services/scripts/is_redis_running.py diff --git a/contrib/runners/orquesta_runner/tests/integration/BUILD b/contrib/runners/orquesta_runner/tests/integration/BUILD index 36ed6e8c52..e856b5ab2c 100644 --- a/contrib/runners/orquesta_runner/tests/integration/BUILD +++ b/contrib/runners/orquesta_runner/tests/integration/BUILD @@ -5,4 +5,5 @@ __defaults__( python_tests( name="tests", + uses=["redis"], ) diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index b488ceefc3..c04e91a0fa 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -16,5 +16,6 @@ python_tests( # overrides={ # "mongo_rules_test.py": {"uses": ["mongo"]}, # "rabbitmq_rules_test.py": {"uses": ["rabbitmq"]}, + # "redis_rules_test.py": {"uses": ["redis"]}, # }, ) diff --git a/pants-plugins/uses_services/redis_rules.py b/pants-plugins/uses_services/redis_rules.py new file mode 100644 index 0000000000..9530011b0d --- /dev/null +++ b/pants-plugins/uses_services/redis_rules.py @@ -0,0 +1,168 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, + rules as pex_rules, +) +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.process import FallibleProcessResult, ProcessCacheScope +from pants.engine.target import Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from uses_services.exceptions import ServiceMissingError, ServiceSpecificMessages +from uses_services.platform_rules import Platform +from uses_services.scripts.is_redis_running import ( + __file__ as is_redis_running_full_path, +) +from uses_services.target_types import UsesServicesField + + +@dataclass(frozen=True) +class UsesRedisRequest: + """One or more targets need a running redis service using these settings. + + The coord_* attributes represent the coordination settings from st2.conf. + In st2 code, they come from: + oslo_config.cfg.CONF.coordination.url + """ + + # These config opts for integration tests are in: + # conf/st2.dev.conf (copied to conf/st2.ci.conf) + # TODO: for int tests: set url by either modifying st2.{dev,ci}.conf on the fly or via env vars. + + # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. + + coord_url: str = "redis://127.0.0.1:6379" + + +@dataclass(frozen=True) +class RedisIsRunning: + pass + + +class PytestUsesRedisRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(UsesServicesField): + return False + uses = target.get(UsesServicesField).value + return uses is not None and "redis" in uses + + +@rule( + desc="Ensure redis is running and accessible before running tests.", + level=LogLevel.DEBUG, +) +async def redis_is_running_for_pytest( + request: PytestUsesRedisRequest, +) -> PytestPluginSetup: + # this will raise an error if redis is not running + _ = await Get(RedisIsRunning, UsesRedisRequest()) + + return PytestPluginSetup() + + +@rule( + desc="Test to see if redis is running and accessible.", + level=LogLevel.DEBUG, +) +async def redis_is_running( + request: UsesRedisRequest, platform: Platform +) -> RedisIsRunning: + script_path = "./is_redis_running.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(is_redis_running_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest, tooz_pex = await MultiGet( + Get(Digest, CreateDigest([FileContent(script_path, script_contents)])), + Get( + VenvPex, + PexRequest( + output_filename="tooz.pex", + internal_only=True, + requirements=PexRequirements({"tooz", "redis"}), + ), + ), + ) + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + tooz_pex, + argv=( + script_path, + request.coord_url, + ), + input_digest=script_digest, + description="Checking to see if Redis is up and accessible.", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_SESSION, + level=LogLevel.DEBUG, + ), + ) + is_running = result.exit_code == 0 + + if is_running: + return RedisIsRunning() + + # redis is not running, so raise an error with instructions. + raise ServiceMissingError.generate( + platform=platform, + messages=ServiceSpecificMessages( + service="redis", + service_start_cmd_el_7="service redis start", + service_start_cmd_el="systemctl start redis", + not_installed_clause_el="this is one way to install it:", + install_instructions_el=dedent( + """\ + sudo yum -y install redis + # Don't forget to start redis. + """ + ), + service_start_cmd_deb="systemctl start redis", + not_installed_clause_deb="this is one way to install it:", + install_instructions_deb=dedent( + """\ + sudo apt-get install -y mongodb redis + # Don't forget to start redis. + """ + ), + service_start_cmd_generic="systemctl start redis", + ), + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestUsesRedisRequest), + *pex_rules(), + ] diff --git a/pants-plugins/uses_services/redis_rules_test.py b/pants-plugins/uses_services/redis_rules_test.py new file mode 100644 index 0000000000..53e8808c37 --- /dev/null +++ b/pants-plugins/uses_services/redis_rules_test.py @@ -0,0 +1,90 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import pytest + +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .data_fixtures import platform, platform_samples +from .exceptions import ServiceMissingError +from .redis_rules import ( + RedisIsRunning, + UsesRedisRequest, + rules as redis_rules, +) +from .platform_rules import Platform + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *redis_rules(), + QueryRule(RedisIsRunning, (UsesRedisRequest, Platform)), + ], + target_types=[], + ) + + +def run_redis_is_running( + rule_runner: RuleRunner, + uses_redis_request: UsesRedisRequest, + mock_platform: Platform, + *, + extra_args: list[str] | None = None, +) -> RedisIsRunning: + rule_runner.set_options( + [ + "--backend-packages=uses_services", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + result = rule_runner.request( + RedisIsRunning, + [uses_redis_request, mock_platform], + ) + return result + + +# Warning this requires that redis be running +def test_redis_is_running(rule_runner: RuleRunner) -> None: + request = UsesRedisRequest() + mock_platform = platform(os="TestMock") + + # we are asserting that this does not raise an exception + is_running = run_redis_is_running(rule_runner, request, mock_platform) + assert is_running + + +@pytest.mark.parametrize("mock_platform", platform_samples) +def test_redis_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: + request = UsesRedisRequest( + coord_url="redis://127.100.20.7:10", # 10 is an unassigned port, unlikely to be used + ) + + with pytest.raises(ExecutionError) as exception_info: + run_redis_is_running(rule_runner, request, mock_platform) + + execution_error = exception_info.value + assert len(execution_error.wrapped_exceptions) == 1 + + exc = execution_error.wrapped_exceptions[0] + assert isinstance(exc, ServiceMissingError) + + assert exc.service == "redis" + assert "The redis service does not seem to be running" in str(exc) + assert exc.instructions != "" diff --git a/pants-plugins/uses_services/scripts/is_redis_running.py b/pants-plugins/uses_services/scripts/is_redis_running.py new file mode 100644 index 0000000000..4a54533381 --- /dev/null +++ b/pants-plugins/uses_services/scripts/is_redis_running.py @@ -0,0 +1,48 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import sys + + +def _is_redis_running(coord_url: str) -> bool: + """Connect to redis with connection logic that mirrors the st2 code. + + In particular, this is based on: + - st2common.services.coordination.coordinator_setup() + + This should not import the st2 code as it should be self-contained. + """ + # late import so that __file__ can be imported in the pants plugin without these imports + from tooz import ToozError, coordination + + member_id = "pants-uses_services-redis" + coordinator = coordination.get_coordinator(coord_url, member_id) + try: + coordinator.start(start_heart=False) + except ToozError: + return False + return True + + +if __name__ == "__main__": + args = dict((k, v) for k, v in enumerate(sys.argv)) + + # unit tests do not use redis, they use use an in-memory coordinator: "zake://" + # integration tests use this url with a conf file derived from conf/st2.dev.conf + coord_url = args.get(1, "redis://127.0.0.1:6379") + + is_running = _is_redis_running(coord_url) + exit_code = 0 if is_running else 1 + sys.exit(exit_code) diff --git a/st2tests/integration/orquesta/BUILD b/st2tests/integration/orquesta/BUILD index 0eea8b1cf1..e2d2f13e20 100644 --- a/st2tests/integration/orquesta/BUILD +++ b/st2tests/integration/orquesta/BUILD @@ -2,4 +2,5 @@ python_sources() python_tests( name="tests", + uses=["redis"], ) From 6eab77f81599c70085fee4f5c896425cabce54c4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 7 Feb 2023 11:12:20 -0600 Subject: [PATCH 2/3] add redis service to GHA workflow --- .github/workflows/test.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b05b886fce..334479b702 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -54,6 +54,19 @@ jobs: - 5672:5672/tcp # AMQP standard port - 15672:15672/tcp # Management: HTTP, CLI + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + env: COLUMNS: '120' From a8fd0d3e65229133c3f851d9abd4bd0bffe40098 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 10:53:37 -0600 Subject: [PATCH 3/3] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 135b91afee..cebfca4671 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884 #5893 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805