diff --git a/demos/opensearch-rag/load-embeddings-from-git.yaml b/demos/opensearch-rag/load-embeddings-from-git.yaml index 4980bc4c..7316737b 100644 --- a/demos/opensearch-rag/load-embeddings-from-git.yaml +++ b/demos/opensearch-rag/load-embeddings-from-git.yaml @@ -18,21 +18,28 @@ data: from requests.auth import HTTPBasicAuth # Configuration - OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST', 'opensearch') - OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT', '9200')) + OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS', 'https://opensearch:9200') + OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME', 'opensearch') + OPENSEARCH_PORT = os.getenv('OPENSEARCH_PORT', '9200') + OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL', 'https') OPENSEARCH_USER = os.getenv('OPENSEARCH_USER', 'admin') OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD', 'adminadmin') + OPENSEARCH_DASHBOARDS_HOST = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards') + OPENSEARCH_DASHBOARDS_PORT = os.getenv('DASHBOARDS_PORT', '5601') INDEX_NAME = os.getenv('INDEX_NAME', 'rag-documents') EMBEDDINGS_FILE = os.getenv('EMBEDDINGS_FILE', '/data/stackable-docs-embeddings.json') + OPENSEARCH_BASIC_AUTH = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD) + OPENSEARCH_DASHBOARDS_BASE_URL = f'http://{OPENSEARCH_DASHBOARDS_HOST}:{OPENSEARCH_DASHBOARDS_PORT}' + def wait_for_opensearch(): """Wait for OpenSearch to be ready.""" print("Waiting for OpenSearch to be ready...") for i in range(60): try: response = requests.get( - f'https://{OPENSEARCH_HOST}:{OPENSEARCH_PORT}', - auth=HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD), + OPENSEARCH_HOSTS, + auth=OPENSEARCH_BASIC_AUTH, verify=False, timeout=5 ) @@ -46,6 +53,27 @@ data: print("[ERROR] OpenSearch did not become ready in time") return False + def wait_for_opensearch_dashboards(): + """Wait for OpenSearch Dashboards to be ready.""" + print("Waiting for OpenSearch Dashboards to be ready...") + for i in range(60): + try: + response = requests.get( + OPENSEARCH_DASHBOARDS_BASE_URL, + auth=OPENSEARCH_BASIC_AUTH, + verify=False, + timeout=5 + ) + if response.status_code == 200: + print("[OK] OpenSearch Dashboards is ready!") + return True + except Exception: + print(f"Waiting for OpenSearch Dashboards... ({i+1}/60)") + time.sleep(5) + + print("[ERROR] OpenSearch Dashboards did not become ready in time") + return False + def index_needs_loading(client): """Check if index exists and has documents. Returns True if loading is needed.""" if not client.indices.exists(index=INDEX_NAME): @@ -135,17 +163,13 @@ data: def create_index_pattern(): """Create index pattern in OpenSearch Dashboards and set as default.""" - dashboards_host = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards') - dashboards_port = os.getenv('DASHBOARDS_PORT', '5601') - base_url = f'http://{dashboards_host}:{dashboards_port}' headers = {"osd-xsrf": "true"} - auth = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD) # Create index pattern - response = requests.put( - f'{base_url}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true', + response = requests.post( + f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true', json={"attributes": {"title": INDEX_NAME, "timeFieldName": "timestamp"}}, - auth=auth, headers=headers, timeout=10 + auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10 ) if response.status_code not in (200, 201, 409): print(f"[ERROR] Failed to create index pattern: {response.status_code} - {response.text}") @@ -154,9 +178,9 @@ data: # Set as default response = requests.post( - f'{base_url}/api/opensearch-dashboards/settings', + f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/opensearch-dashboards/settings', json={"changes": {"defaultIndex": INDEX_NAME}}, - auth=auth, headers=headers, timeout=10 + auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10 ) if response.status_code in (200, 201): print(f"[OK] Set {INDEX_NAME} as default index pattern") @@ -201,10 +225,13 @@ data: if not wait_for_opensearch(): return 1 + if not wait_for_opensearch_dashboards(): + return 1 + client = OpenSearch( - hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}], + hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}], http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD), - use_ssl=True, + use_ssl=OPENSEARCH_PROTOCOL == 'https', verify_certs=False, ssl_show_warn=False ) @@ -277,10 +304,6 @@ spec: pip install -q opensearch-py requests urllib3 python -u /scripts/load.py env: - - name: OPENSEARCH_HOST - value: "opensearch" - - name: OPENSEARCH_PORT - value: "9200" - name: OPENSEARCH_USER value: "admin" - name: OPENSEARCH_PASSWORD @@ -292,6 +315,9 @@ spec: value: "rag-documents" - name: EMBEDDINGS_FILE value: "/data/stackable-docs-embeddings.json" + envFrom: + - configMapRef: + name: opensearch volumeMounts: - name: script mountPath: /scripts diff --git a/docs/modules/demos/images/logging/tenant.png b/docs/modules/demos/images/logging/tenant.png deleted file mode 100644 index 6e9df68c..00000000 Binary files a/docs/modules/demos/images/logging/tenant.png and /dev/null differ diff --git a/docs/modules/demos/pages/logging.adoc b/docs/modules/demos/pages/logging.adoc index 5d984730..9b6810bf 100644 --- a/docs/modules/demos/pages/logging.adoc +++ b/docs/modules/demos/pages/logging.adoc @@ -56,7 +56,8 @@ $ stackablectl stacklet list ┌───────────────────────┬───────────────────────┬───────────┬────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ │ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ ╞═══════════════════════╪═══════════════════════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ -│ opensearch ┆ opensearch ┆ default ┆ nodes-default-http http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │ +│ opensearch ┆ opensearch ┆ default ┆ -http http://opensearch.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ nodes-default-http http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ zookeeper ┆ simple-zk ┆ default ┆ server-zk simple-zk-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ @@ -79,10 +80,6 @@ Log in with the username `admin` and password `adminadmin`. NOTE: On first login, you will be presented with some options. Feel free to bypass them to get to the logs. -Select the `Global` tenant to have access to the pre-configured `vector-*` index pattern. - -image::logging/tenant.png[] - Click _Discover_ in the menu on the left to view the recent logs. If you do not see anything, increase the search window to greater than _Last 15 minutes_. diff --git a/docs/modules/demos/pages/opensearch-rag.adoc b/docs/modules/demos/pages/opensearch-rag.adoc index 166e969e..2b3d9744 100644 --- a/docs/modules/demos/pages/opensearch-rag.adoc +++ b/docs/modules/demos/pages/opensearch-rag.adoc @@ -274,7 +274,7 @@ Log in at http://localhost:5601 with: * *Username*: `admin` * *Password*: `adminadmin` -Select the *Global* tenant, then navigate to *Discover* to browse the `rag-documents` index: +Navigate to *Discover* to browse the `rag-documents` index: * Document titles and content * Categories (airflow-operator, kafka-operator, trino-operator, etc.) diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml index 15fbdc6f..fe0b511a 100644 --- a/stacks/_templates/opensearch-dashboards.yaml +++ b/stacks/_templates/opensearch-dashboards.yaml @@ -6,7 +6,6 @@ repo: url: https://opensearch-project.github.io/helm-charts version: {{ opensearchVersion }} options: - opensearchHosts: https://opensearch:9200 image: repository: oci.stackable.tech/sdp/opensearch-dashboards tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}" @@ -16,11 +15,17 @@ options: type: NodePort port: 5601 annotations: - stackable.tech/logging-view-logs: |- - /app/discover?security_tenant=global#/view/logs + stackable.tech/logging-view-logs: /app/discover#/view/logs stackable.tech/logging-credentials-secret: opensearch-user labels: stackable.tech/vendor: Stackable + opensearchHosts: null # Use the discovery ConfigMap instead + extraEnvs: + - name: OPENSEARCH_HOSTS + valueFrom: + configMapKeyRef: + name: opensearch + key: OPENSEARCH_HOSTS opensearchAccount: secret: opensearch-dashboard-user serviceAccount: @@ -28,3 +33,5 @@ options: # Use the ServiceAccount of OpenSearch because its permissions are already configured to work on # OpenShift. name: opensearch-serviceaccount +config: + opensearch_security.multitenancy.enabled: false diff --git a/stacks/logging/opensearch.yaml b/stacks/logging/opensearch.yaml index 632a333a..d96d4610 100644 --- a/stacks/logging/opensearch.yaml +++ b/stacks/logging/opensearch.yaml @@ -88,6 +88,8 @@ stringData: authentication_backend: type: intern authz: {} + kibana: + multitenancy_enabled: false internal_users.yml: | --- _meta: diff --git a/stacks/logging/setup-opensearch-dashboards.yaml b/stacks/logging/setup-opensearch-dashboards.yaml index c2ece40c..edec00fa 100644 --- a/stacks/logging/setup-opensearch-dashboards.yaml +++ b/stacks/logging/setup-opensearch-dashboards.yaml @@ -28,7 +28,6 @@ spec: --retry-delay 5 \ --header "Content-Type:application/json" \ --header "osd-xsrf:true" \ - --header "securitytenant: global" \ --data '{ "attributes": { "title": "vector-*", @@ -39,7 +38,6 @@ spec: curl \ --header "Content-Type:application/json" \ --header "osd-xsrf:true" \ - --header "securitytenant: global" \ --data '{ "attributes": { "title":"Logs", diff --git a/stacks/opensearch-rag/jupyterlab.yaml b/stacks/opensearch-rag/jupyterlab.yaml index 1cff928c..9eabc005 100644 --- a/stacks/opensearch-rag/jupyterlab.yaml +++ b/stacks/opensearch-rag/jupyterlab.yaml @@ -31,10 +31,6 @@ spec: env: - name: JUPYTER_PORT value: "8888" - - name: OPENSEARCH_HOST - value: "opensearch" - - name: OPENSEARCH_PORT - value: "9200" - name: OPENSEARCH_USER value: "admin" - name: OPENSEARCH_PASSWORD @@ -48,6 +44,9 @@ spec: value: "11434" - name: OLLAMA_LLM_MODEL value: "{{ ollamaLlmModel }}" + envFrom: + - configMapRef: + name: opensearch ports: - name: http containerPort: 8888 diff --git a/stacks/opensearch-rag/opensearch-rag.ipynb b/stacks/opensearch-rag/opensearch-rag.ipynb index b75a0684..63368f23 100644 --- a/stacks/opensearch-rag/opensearch-rag.ipynb +++ b/stacks/opensearch-rag/opensearch-rag.ipynb @@ -68,8 +68,10 @@ "warnings.filterwarnings('ignore')\n", "\n", "# Configuration from environment variables\n", - "OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST')\n", + "OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS')\n", + "OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME')\n", "OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT'))\n", + "OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL')\n", "OPENSEARCH_USER = os.getenv('OPENSEARCH_USER')\n", "OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD')\n", "OLLAMA_HOST = os.getenv('OLLAMA_HOST')\n", @@ -77,7 +79,7 @@ "OLLAMA_LLM_MODEL = os.getenv('OLLAMA_LLM_MODEL')\n", "INDEX_NAME = 'rag-documents'\n", "\n", - "print(f\"OpenSearch: {OPENSEARCH_HOST}:{OPENSEARCH_PORT}\")\n", + "print(f\"OpenSearch: {OPENSEARCH_HOSTS}\")\n", "print(f\"Ollama: {OLLAMA_HOST}:{OLLAMA_PORT} using {OLLAMA_LLM_MODEL} to generate responses\")\n", "print(f\"Index: {INDEX_NAME}\")" ] @@ -101,9 +103,9 @@ "source": [ "# Initialize OpenSearch client\n", "opensearch_client = OpenSearch(\n", - " hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}],\n", + " hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}],\n", " http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD),\n", - " use_ssl=True,\n", + " use_ssl=OPENSEARCH_PROTOCOL == 'https',\n", " verify_certs=False,\n", " ssl_show_warn=False\n", ")\n", diff --git a/stacks/opensearch-rag/opensearch.yaml b/stacks/opensearch-rag/opensearch.yaml index bc359a92..57a57220 100644 --- a/stacks/opensearch-rag/opensearch.yaml +++ b/stacks/opensearch-rag/opensearch.yaml @@ -90,6 +90,8 @@ stringData: authentication_backend: type: intern authz: {} + kibana: + multitenancy_enabled: false internal_users.yml: | --- _meta: diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index cccf69e2..25ba0cc9 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -101,7 +101,7 @@ stacks: default: kibanaserverkibanaserver - name: opensearchVersion description: Version of OpenSearch and OpenSearch Dashboards to deploy - default: 3.1.0 + default: 3.4.0 - name: stackableReleaseVersion description: The Stackable release to be used for the OpenSearch Dashboards image tag default: 0.0.0-dev @@ -725,7 +725,7 @@ stacks: default: kibanaserverkibanaserver - name: opensearchVersion description: Version of OpenSearch and OpenSearch Dashboards to deploy - default: 3.1.0 + default: 3.4.0 - name: stackableReleaseVersion description: The Stackable release to be used for the OpenSearch Dashboards image tag default: 0.0.0-dev