Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 45 additions & 19 deletions demos/opensearch-rag/load-embeddings-from-git.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,28 @@ data:
from requests.auth import HTTPBasicAuth

# Configuration
OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST', 'opensearch')
OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT', '9200'))
OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS', 'https://opensearch:9200')
OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME', 'opensearch')
OPENSEARCH_PORT = os.getenv('OPENSEARCH_PORT', '9200')
OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL', 'https')
OPENSEARCH_USER = os.getenv('OPENSEARCH_USER', 'admin')
OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD', 'adminadmin')
OPENSEARCH_DASHBOARDS_HOST = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards')
OPENSEARCH_DASHBOARDS_PORT = os.getenv('DASHBOARDS_PORT', '5601')
INDEX_NAME = os.getenv('INDEX_NAME', 'rag-documents')
EMBEDDINGS_FILE = os.getenv('EMBEDDINGS_FILE', '/data/stackable-docs-embeddings.json')

OPENSEARCH_BASIC_AUTH = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD)
OPENSEARCH_DASHBOARDS_BASE_URL = f'http://{OPENSEARCH_DASHBOARDS_HOST}:{OPENSEARCH_DASHBOARDS_PORT}'

def wait_for_opensearch():
"""Wait for OpenSearch to be ready."""
print("Waiting for OpenSearch to be ready...")
for i in range(60):
try:
response = requests.get(
f'https://{OPENSEARCH_HOST}:{OPENSEARCH_PORT}',
auth=HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD),
OPENSEARCH_HOSTS,
auth=OPENSEARCH_BASIC_AUTH,
verify=False,
timeout=5
)
Expand All @@ -46,6 +53,27 @@ data:
print("[ERROR] OpenSearch did not become ready in time")
return False

def wait_for_opensearch_dashboards():
"""Wait for OpenSearch Dashboards to be ready."""
print("Waiting for OpenSearch Dashboards to be ready...")
for i in range(60):
try:
response = requests.get(
OPENSEARCH_DASHBOARDS_BASE_URL,
auth=OPENSEARCH_BASIC_AUTH,
verify=False,
timeout=5
)
if response.status_code == 200:
print("[OK] OpenSearch Dashboards is ready!")
return True
except Exception:
print(f"Waiting for OpenSearch Dashboards... ({i+1}/60)")
time.sleep(5)

print("[ERROR] OpenSearch Dashboards did not become ready in time")
return False

def index_needs_loading(client):
"""Check if index exists and has documents. Returns True if loading is needed."""
if not client.indices.exists(index=INDEX_NAME):
Expand Down Expand Up @@ -135,17 +163,13 @@ data:

def create_index_pattern():
"""Create index pattern in OpenSearch Dashboards and set as default."""
dashboards_host = os.getenv('DASHBOARDS_HOST', 'opensearch-dashboards')
dashboards_port = os.getenv('DASHBOARDS_PORT', '5601')
base_url = f'http://{dashboards_host}:{dashboards_port}'
headers = {"osd-xsrf": "true"}
auth = HTTPBasicAuth(OPENSEARCH_USER, OPENSEARCH_PASSWORD)

# Create index pattern
response = requests.put(
f'{base_url}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true',
response = requests.post(
f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/saved_objects/index-pattern/{INDEX_NAME}?overwrite=true',
json={"attributes": {"title": INDEX_NAME, "timeFieldName": "timestamp"}},
auth=auth, headers=headers, timeout=10
auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10
)
if response.status_code not in (200, 201, 409):
print(f"[ERROR] Failed to create index pattern: {response.status_code} - {response.text}")
Expand All @@ -154,9 +178,9 @@ data:

# Set as default
response = requests.post(
f'{base_url}/api/opensearch-dashboards/settings',
f'{OPENSEARCH_DASHBOARDS_BASE_URL}/api/opensearch-dashboards/settings',
json={"changes": {"defaultIndex": INDEX_NAME}},
auth=auth, headers=headers, timeout=10
auth=OPENSEARCH_BASIC_AUTH, headers=headers, timeout=10
)
if response.status_code in (200, 201):
print(f"[OK] Set {INDEX_NAME} as default index pattern")
Expand Down Expand Up @@ -201,10 +225,13 @@ data:
if not wait_for_opensearch():
return 1

if not wait_for_opensearch_dashboards():
return 1

client = OpenSearch(
hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}],
hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}],
http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD),
use_ssl=True,
use_ssl=OPENSEARCH_PROTOCOL == 'https',
verify_certs=False,
ssl_show_warn=False
)
Expand Down Expand Up @@ -277,10 +304,6 @@ spec:
pip install -q opensearch-py requests urllib3
python -u /scripts/load.py
env:
- name: OPENSEARCH_HOST
value: "opensearch"
- name: OPENSEARCH_PORT
value: "9200"
- name: OPENSEARCH_USER
value: "admin"
- name: OPENSEARCH_PASSWORD
Expand All @@ -292,6 +315,9 @@ spec:
value: "rag-documents"
- name: EMBEDDINGS_FILE
value: "/data/stackable-docs-embeddings.json"
envFrom:
- configMapRef:
name: opensearch
volumeMounts:
- name: script
mountPath: /scripts
Expand Down
Binary file removed docs/modules/demos/images/logging/tenant.png
Binary file not shown.
7 changes: 2 additions & 5 deletions docs/modules/demos/pages/logging.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ $ stackablectl stacklet list
┌───────────────────────┬───────────────────────┬───────────┬────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐
│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │
╞═══════════════════════╪═══════════════════════╪═══════════╪════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡
│ opensearch ┆ opensearch ┆ default ┆ nodes-default-http http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │
│ opensearch ┆ opensearch ┆ default ┆ -http http://opensearch.default.svc.cluster.local:9200 ┆ Available, Reconciling, Running │
│ ┆ ┆ ┆ nodes-default-http http://opensearch-nodes-default.default.svc.cluster.local:9200 ┆ │
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
│ zookeeper ┆ simple-zk ┆ default ┆ server-zk simple-zk-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
Expand All @@ -79,10 +80,6 @@ Log in with the username `admin` and password `adminadmin`.

NOTE: On first login, you will be presented with some options. Feel free to bypass them to get to the logs.

Select the `Global` tenant to have access to the pre-configured `vector-*` index pattern.

image::logging/tenant.png[]

Click _Discover_ in the menu on the left to view the recent logs.
If you do not see anything, increase the search window to greater than _Last 15 minutes_.

Expand Down
2 changes: 1 addition & 1 deletion docs/modules/demos/pages/opensearch-rag.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ Log in at http://localhost:5601 with:
* *Username*: `admin`
* *Password*: `adminadmin`

Select the *Global* tenant, then navigate to *Discover* to browse the `rag-documents` index:
Navigate to *Discover* to browse the `rag-documents` index:

* Document titles and content
* Categories (airflow-operator, kafka-operator, trino-operator, etc.)
Expand Down
13 changes: 10 additions & 3 deletions stacks/_templates/opensearch-dashboards.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ repo:
url: https://opensearch-project.github.io/helm-charts
version: {{ opensearchVersion }}
options:
opensearchHosts: https://opensearch:9200
image:
repository: oci.stackable.tech/sdp/opensearch-dashboards
tag: "{{ opensearchVersion }}-stackable{{ stackableReleaseVersion }}"
Expand All @@ -16,15 +15,23 @@ options:
type: NodePort
port: 5601
annotations:
stackable.tech/logging-view-logs: |-
/app/discover?security_tenant=global#/view/logs
stackable.tech/logging-view-logs: /app/discover#/view/logs
stackable.tech/logging-credentials-secret: opensearch-user
labels:
stackable.tech/vendor: Stackable
opensearchHosts: null # Use the discovery ConfigMap instead
extraEnvs:
- name: OPENSEARCH_HOSTS
valueFrom:
configMapKeyRef:
name: opensearch
key: OPENSEARCH_HOSTS
opensearchAccount:
secret: opensearch-dashboard-user
serviceAccount:
create: false
# Use the ServiceAccount of OpenSearch because its permissions are already configured to work on
# OpenShift.
name: opensearch-serviceaccount
config:
opensearch_security.multitenancy.enabled: false
2 changes: 2 additions & 0 deletions stacks/logging/opensearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ stringData:
authentication_backend:
type: intern
authz: {}
kibana:
multitenancy_enabled: false
internal_users.yml: |
---
_meta:
Expand Down
2 changes: 0 additions & 2 deletions stacks/logging/setup-opensearch-dashboards.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ spec:
--retry-delay 5 \
--header "Content-Type:application/json" \
--header "osd-xsrf:true" \
--header "securitytenant: global" \
--data '{
"attributes": {
"title": "vector-*",
Expand All @@ -39,7 +38,6 @@ spec:
curl \
--header "Content-Type:application/json" \
--header "osd-xsrf:true" \
--header "securitytenant: global" \
--data '{
"attributes": {
"title":"Logs",
Expand Down
7 changes: 3 additions & 4 deletions stacks/opensearch-rag/jupyterlab.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,6 @@ spec:
env:
- name: JUPYTER_PORT
value: "8888"
- name: OPENSEARCH_HOST
value: "opensearch"
- name: OPENSEARCH_PORT
value: "9200"
- name: OPENSEARCH_USER
value: "admin"
- name: OPENSEARCH_PASSWORD
Expand All @@ -48,6 +44,9 @@ spec:
value: "11434"
- name: OLLAMA_LLM_MODEL
value: "{{ ollamaLlmModel }}"
envFrom:
- configMapRef:
name: opensearch
ports:
- name: http
containerPort: 8888
Expand Down
10 changes: 6 additions & 4 deletions stacks/opensearch-rag/opensearch-rag.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -68,16 +68,18 @@
"warnings.filterwarnings('ignore')\n",
"\n",
"# Configuration from environment variables\n",
"OPENSEARCH_HOST = os.getenv('OPENSEARCH_HOST')\n",
"OPENSEARCH_HOSTS = os.getenv('OPENSEARCH_HOSTS')\n",
"OPENSEARCH_HOSTNAME = os.getenv('OPENSEARCH_HOSTNAME')\n",
"OPENSEARCH_PORT = int(os.getenv('OPENSEARCH_PORT'))\n",
"OPENSEARCH_PROTOCOL = os.getenv('OPENSEARCH_PROTOCOL')\n",
"OPENSEARCH_USER = os.getenv('OPENSEARCH_USER')\n",
"OPENSEARCH_PASSWORD = os.getenv('OPENSEARCH_PASSWORD')\n",
"OLLAMA_HOST = os.getenv('OLLAMA_HOST')\n",
"OLLAMA_PORT = int(os.getenv('OLLAMA_PORT'))\n",
"OLLAMA_LLM_MODEL = os.getenv('OLLAMA_LLM_MODEL')\n",
"INDEX_NAME = 'rag-documents'\n",
"\n",
"print(f\"OpenSearch: {OPENSEARCH_HOST}:{OPENSEARCH_PORT}\")\n",
"print(f\"OpenSearch: {OPENSEARCH_HOSTS}\")\n",
"print(f\"Ollama: {OLLAMA_HOST}:{OLLAMA_PORT} using {OLLAMA_LLM_MODEL} to generate responses\")\n",
"print(f\"Index: {INDEX_NAME}\")"
]
Expand All @@ -101,9 +103,9 @@
"source": [
"# Initialize OpenSearch client\n",
"opensearch_client = OpenSearch(\n",
" hosts=[{'host': OPENSEARCH_HOST, 'port': OPENSEARCH_PORT}],\n",
" hosts=[{'host': OPENSEARCH_HOSTNAME, 'port': OPENSEARCH_PORT}],\n",
" http_auth=(OPENSEARCH_USER, OPENSEARCH_PASSWORD),\n",
" use_ssl=True,\n",
" use_ssl=OPENSEARCH_PROTOCOL == 'https',\n",
" verify_certs=False,\n",
" ssl_show_warn=False\n",
")\n",
Expand Down
2 changes: 2 additions & 0 deletions stacks/opensearch-rag/opensearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ stringData:
authentication_backend:
type: intern
authz: {}
kibana:
multitenancy_enabled: false
internal_users.yml: |
---
_meta:
Expand Down
4 changes: 2 additions & 2 deletions stacks/stacks-v2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ stacks:
default: kibanaserverkibanaserver
- name: opensearchVersion
description: Version of OpenSearch and OpenSearch Dashboards to deploy
default: 3.1.0
default: 3.4.0
- name: stackableReleaseVersion
description: The Stackable release to be used for the OpenSearch Dashboards image tag
default: 0.0.0-dev
Expand Down Expand Up @@ -725,7 +725,7 @@ stacks:
default: kibanaserverkibanaserver
- name: opensearchVersion
description: Version of OpenSearch and OpenSearch Dashboards to deploy
default: 3.1.0
default: 3.4.0
- name: stackableReleaseVersion
description: The Stackable release to be used for the OpenSearch Dashboards image tag
default: 0.0.0-dev
Expand Down