From 846da3e6bf6a215e364b7c7895558c83a9ca1330 Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 14:15:05 +0200 Subject: [PATCH 1/5] clean doc --- Makefile | 80 ++++++++++++++++++----- README.md | 187 ++++++++++++++++++++++++++---------------------------- 2 files changed, 155 insertions(+), 112 deletions(-) diff --git a/Makefile b/Makefile index 4338af3..522f894 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,37 @@ -# Build the docker images contained in this repo +# Makefile +# Build and run local stack for Pyronear + +SHELL := /bin/sh +COMPOSE := docker compose + +.PHONY: help init build build-external build-all run-backend run-engine run-tools run run-all stop ps logs test + +help: + @echo "Targets:" + @echo " init Create .env from .env.test if missing" + @echo " build Build local images in this repo" + @echo " build-external Build images in sibling repos" + @echo " build-all Build local and external images" + @echo " run-backend Start base services without profiles" + @echo " run-engine Start engine profile only" + @echo " run-tools Start tools profile only" + @echo " run Start everything except engine" + @echo " run-all Start everything including engine" + @echo " stop Stop and remove all services and volumes" + @echo " ps Show compose status" + @echo " logs Follow logs" + @echo " test Run pytest" + +# ------------------------------------------------------------------- +# Init +# ------------------------------------------------------------------- + +init: + @test -f .env || cp .env.test .env + +# ------------------------------------------------------------------- +# Build images in this repo +# ------------------------------------------------------------------- build: docker build -f containers/init_script/Dockerfile -t pyronear/pyro-api-init:latest containers/init_script/ @@ -6,36 +39,51 @@ build: docker build -f containers/reolinkdev2/Dockerfile -t pyronear/reolinkdev2:latest containers/reolinkdev2/ docker build -f containers/notebooks/Dockerfile -t pyronear/notebooks:latest containers/notebooks/ -build-external: - cd ../pyro-api/; make build - cd ../pyro-engine/; make build-lib - cd ../pyro-engine/; make build-app - cd ../pyro-platform/; make build +# ------------------------------------------------------------------- +# Build images from sibling repositories +# ------------------------------------------------------------------- +build-external: + cd ../pyro-api && make build + cd ../pyro-engine && make build-lib + cd ../pyro-engine && make build-app + cd ../pyro-platform && make build build-all: build build-external +# ------------------------------------------------------------------- +# Run targets +# ------------------------------------------------------------------- + run-backend: - docker compose up -d + $(COMPOSE) up -d run-engine: - docker compose --profile engine up -d + $(COMPOSE) --profile engine up -d run-tools: - cp .env.test .env - docker compose --profile tools up -d - -run-etl: - docker compose --profile etl up -d + $(COMPOSE) --profile tools up -d +# Start everything except the engine: base services plus front + tools run: - docker compose --profile front up -d + $(COMPOSE) --profile front --profile tools up -d +# Start everything including the engine run-all: - docker compose --profile front --profile engine up -d + $(COMPOSE) --profile front --profile tools --profile engine up -d stop: - docker compose --profile front --profile engine --profile etl --profile tools down -v + $(COMPOSE) --profile front --profile engine --profile tools down -v + +ps: + $(COMPOSE) ps + +logs: + $(COMPOSE) logs -f --tail=200 + +# ------------------------------------------------------------------- +# Tests +# ------------------------------------------------------------------- test: pytest -s tests/* diff --git a/README.md b/README.md index 74dff59..1130095 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,142 @@ -**README** +# Pyronear Dev Environment -This Docker Compose configuration sets up a development environment for Pyronear's API along with supporting services like a PostgreSQL database, MinIO for S3 emulation, Pyro Engine, and Promtail for log shipping. - -## Services -1. **pyro-api**: Runs the Pyronear API using uvicorn. -2. **db**: PostgreSQL database for the API. -3. **minio**: Emulates AWS S3 using [MinIO](https://github.com/minio/minio). -4. **pyro-engine**: Pyro Engine service. -5. **reolinkdev**: a service which imitate a reolink camera by sending back pictures of fire. -6. **frontend**: our webapp available on the 8085 port. +This repository provides a Docker Compose configuration to run a full Pyronear development environment with the API, database, S3 emulation, frontend, notebooks, and optional camera engine. +--- - _Additional services (helpers):_ -7. **notebooks** : Python notebook server to run scripts on api without having to install python -8. **db-ui** (pgadmin): UI to visualize and manipulate the data in PostgreSQL database +## 🚀 Quick Start -## Installation -### Prerequisites -- Docker and Docker Compose installed on your system. -- Precommit hook installed on this repo +### Init -### Running everything (engine + predefined alerts) +```bash +make init +make build +``` -Start the Docker services using the following commands: +### Run ```bash -make build -make run-all +make run ``` -This will launch the full stack including the engine and the predefined alert generation. +* Send an alert by opening [http://0.0.0.0:8889/notebooks/notebooks/send_real_alerts.ipynb](http://0.0.0.0:8889/notebooks/notebooks/send_real_alerts.ipynb) +* Observe the alert on the frontend at [http://0.0.0.0:8050/](http://0.0.0.0:8050/) +* Use credentials from `data/csv/API_DATA_DEV/users.csv` +* Or check directly on the API at [http://0.0.0.0:5050/docs](http://0.0.0.0:5050/docs) with the same creds --- -you can check that everyhing is working thanks to the following commands : -```bash -docker logs init -docker logs engine -``` +## 🧩 Services +* **pyro-api**: Pyronear API (uvicorn) +* **db**: PostgreSQL database +* **minio**: S3-compatible storage (via MinIO) +* **frontend**: Web app (Dash) +* **pyro-engine**: Engine service (requires cameras, optional) +* **reolinkdev1 / reolinkdev2**: Fake Reolink cameras sending test images +* **notebooks**: Jupyter server to run helper notebooks +* **db-ui**: pgAdmin to browse/manage the database -### Running services partially -If you want to launch only the engine and two dev-cameras you can use : -```bash -make run-engine -``` +--- -If you want to launch only the backend (in order to develop the frontend) : -```bash -make run-backend -``` +## ⚙️ Installation -If you want to launch only the additional tools, you can use : -```bash -make run-tools -``` +### Prerequisites -Also you need to tell your computer where your S3 is. -For that you will have to add this line to you /etc/hosts : +* Docker and Docker Compose +* Pre-commit hook installed in this repo +* Add this line to `/etc/hosts` so the MinIO endpoint resolves correctly: + + ``` + 127.0.0.1 minio + ``` + +--- + +## ▶️ Running + +### Full stack with engine ```bash -127.0.0.1 minio +make build +make run-all ``` -### Running customized alerts using personal notebooks (not in docker) - -Install the notebook dependencies: +This launches everything including the engine and simulated alerts. +You can check health with: ```bash -pip install -r notebooks/requirements.txt +docker logs init +docker logs engine ``` +### Partial runs -## Access -### Accessing the API -Once the services are up and running, you can access the Pyronear API at `http://localhost:5050/docs`. +* Backend only (API, DB, S3): + ```bash + make run-backend + ``` +* Engine only: -### Accessing the Dash web-app + ```bash + make run-engine + ``` +* Tools only (notebooks, db-ui): -Since Dash can be a bit capricious, you should launch a private window from you browser and access the web app at `http://localhost:8050` + ```bash + make run-tools + ``` -29/01/2024 : For the moment, the ADMIN access doesn't show the alerts sent by the camera. For that you will have to use a user account which are defined in data/csv/users.csv +--- -### Launch the web app manually from the pyro-platform directory +## 🔑 Access -You can launch the backend (alert-api, postgresdb & s3) : +* **API**: [http://localhost:5050/docs](http://localhost:5050/docs) +* **Frontend (Dash app)**: [http://localhost:8050](http://localhost:8050) -```bash -make run-backend -``` + * If issues: use a private browser window + * Admin access currently does not display camera alerts, use user creds from `data/csv/users.csv` +* **Notebooks**: [http://localhost:8889](http://localhost:8889) +* **pgAdmin (db-ui)**: [http://localhost:8888/browser/](http://localhost:8888/browser/) -And, in your pyro-platform/.env use this API_URL env var : -```bash -API_URL=http://localhost:5050 -``` + * Login: `DB_UI_MAIL` / `DB_UI_PWD` (set in `.env`) + * First connection: register server with host `db`, user/password from `.env` +* **MinIO console (S3 GUI)**: [http://localhost:9001](http://localhost:9001) -### Access the service notebooks -Access at the address : http://localhost:8889/ + * Manage buckets, upload/delete files -### Access the service db-ui -You can access the db-ui service (pgadmin) at `http://localhost:8888/browser/` +--- -Log in with the mail/pwd specified in the env file (`DB_UI_MAIL`/`DB_UI_PWD`) +## 📂 Data Usage -At the first connection, the db server must be configured: -- Register a server with those data : -- Name: "pyro-db" -- Host name/address: "db" -- Maintenance database : see POSTGRES_DB in .env -- User : see POSTGRES_USER in .env -- Password : see POSTGRES_PASSWORD in .env +### Update the last image for a camera -### Access the S3 GUI MinIO console -You can access S3 GUI MinIO console at `http://localhost:9001/` -And then upload/download/delete files, create buckets. +1. Upload a new image in MinIO under the bucket ending with `...-alert-api-{organisation_id}` +2. In pgAdmin, update the `cameras` table: -## How to use data -#### How to update the last image for a camera -- In S3 GUI MinIO console, open the directory finishing by "...-alert-api-{organisation_id}" and upload the image -- In db-ui, open the table "cameras" and update - - the column last_image with the filename from above - - the column last_active_at + * `last_image` with the filename + * `last_active_at` timestamp -### More images in the Reoling Dev Camera -you need to create a directory data/images before launching the env, with the images inside ! +### Add more images to Reolink Dev -### How to create alerts -Use one of the provided notebooks to send custom alerts manually. +Create a directory `data/images` before starting the environment and put your images inside. -For example, to send real alerts based on selected examples, run: +### Send custom alerts -```bash -notebooks/send_real_alerts.ipynb -``` +Use Jupyter notebooks (e.g., `notebooks/send_real_alerts.ipynb`). +When running notebooks **inside Docker**, set: -Then, you will be able to connect to the API thanks to the credentials in the .env file -Note : if the notebook is run in a container, please change the following variable -in the files .ipynb +```python API_URL = "http://api:5050" +``` + +--- + +## 🛑 Cleanup + +Stop and remove everything: -## Cleanup -To stop and remove the Docker services, run: ```bash make stop ``` From 6dd607acb2cd879b6b521dca255127aee1605dcd Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 15:20:04 +0200 Subject: [PATCH 2/5] reorder --- README.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1130095..f4c2d3c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,19 @@ This repository provides a Docker Compose configuration to run a full Pyronear d --- +## ⚙️ Installation + +### Prerequisites + +* Docker and Docker Compose +* Add this line to `/etc/hosts` so the MinIO endpoint resolves correctly: + + ``` + 127.0.0.1 minio + ``` + +--- + ## 🚀 Quick Start ### Init @@ -39,20 +52,6 @@ make run --- -## ⚙️ Installation - -### Prerequisites - -* Docker and Docker Compose -* Pre-commit hook installed in this repo -* Add this line to `/etc/hosts` so the MinIO endpoint resolves correctly: - - ``` - 127.0.0.1 minio - ``` - ---- - ## ▶️ Running ### Full stack with engine From 74d99dbcd152ca5bad192dcb6d90a48afafc5da1 Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 16:26:04 +0200 Subject: [PATCH 3/5] simplify --- Makefile | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 522f894..a57cd21 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,6 @@ # Makefile # Build and run local stack for Pyronear -SHELL := /bin/sh -COMPOSE := docker compose - -.PHONY: help init build build-external build-all run-backend run-engine run-tools run run-all stop ps logs test - help: @echo "Targets:" @echo " init Create .env from .env.test if missing" @@ -56,30 +51,30 @@ build-all: build build-external # ------------------------------------------------------------------- run-backend: - $(COMPOSE) up -d + docker compose up -d run-engine: - $(COMPOSE) --profile engine up -d + docker compose --profile engine up -d run-tools: - $(COMPOSE) --profile tools up -d + docker compose --profile tools up -d # Start everything except the engine: base services plus front + tools run: - $(COMPOSE) --profile front --profile tools up -d + docker compose --profile front --profile tools up -d # Start everything including the engine run-all: - $(COMPOSE) --profile front --profile tools --profile engine up -d + docker compose --profile front --profile tools --profile engine up -d stop: - $(COMPOSE) --profile front --profile engine --profile tools down -v + docker compose --profile front --profile engine --profile tools down -v ps: - $(COMPOSE) ps + docker compose ps logs: - $(COMPOSE) logs -f --tail=200 + docker compose logs -f --tail=200 # ------------------------------------------------------------------- # Tests From 64f6ea051c0fa6abc4d4e75fe43439ded9d7473d Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 16:31:50 +0200 Subject: [PATCH 4/5] clean comments --- Makefile | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index a57cd21..ddd9e03 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Makefile -# Build and run local stack for Pyronear +# Build and run the local stack for Pyronear help: @echo "Targets:" @@ -7,11 +7,11 @@ help: @echo " build Build local images in this repo" @echo " build-external Build images in sibling repos" @echo " build-all Build local and external images" - @echo " run-backend Start base services without profiles" - @echo " run-engine Start engine profile only" - @echo " run-tools Start tools profile only" - @echo " run Start everything except engine" - @echo " run-all Start everything including engine" + @echo " run-backend Start base services only" + @echo " run-engine Start base services plus engine profile" + @echo " run-tools Start base services plus tools profile" + @echo " run Start base services plus front and tools profiles" + @echo " run-all Start base services plus front tools and engine profiles" @echo " stop Stop and remove all services and volumes" @echo " ps Show compose status" @echo " logs Follow logs" @@ -50,31 +50,34 @@ build-all: build build-external # Run targets # ------------------------------------------------------------------- +# Base services: db, minio, pyro_api, init_script run-backend: - docker compose up -d + docker compose up -d +# Engine profile adds pyro_engine, reolinkdev1, reolinkdev2 run-engine: - docker compose --profile engine up -d + docker compose --profile engine up -d +# Tools profile adds notebooks, db-ui run-tools: - docker compose --profile tools up -d + docker compose --profile tools up -d -# Start everything except the engine: base services plus front + tools +# Front profile adds frontend, here we also include tools run: - docker compose --profile front --profile tools up -d + docker compose --profile front --profile tools up -d -# Start everything including the engine +# Everything including engine run-all: - docker compose --profile front --profile tools --profile engine up -d + docker compose --profile front --profile tools --profile engine up -d stop: - docker compose --profile front --profile engine --profile tools down -v + docker compose --profile front --profile engine --profile tools down -v ps: - docker compose ps + docker compose ps logs: - docker compose logs -f --tail=200 + docker compose logs -f --tail=200 # ------------------------------------------------------------------- # Tests From 613e09da49e9bede23c91bf8806b39dc80fc167b Mon Sep 17 00:00:00 2001 From: MateoLOSTANLEN Date: Wed, 1 Oct 2025 16:34:11 +0200 Subject: [PATCH 5/5] order --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f4c2d3c..f3213ff 100644 --- a/README.md +++ b/README.md @@ -109,14 +109,6 @@ docker logs engine ## 📂 Data Usage -### Update the last image for a camera - -1. Upload a new image in MinIO under the bucket ending with `...-alert-api-{organisation_id}` -2. In pgAdmin, update the `cameras` table: - - * `last_image` with the filename - * `last_active_at` timestamp - ### Add more images to Reolink Dev Create a directory `data/images` before starting the environment and put your images inside. @@ -128,6 +120,16 @@ When running notebooks **inside Docker**, set: ```python API_URL = "http://api:5050" + +### Update the last image for a camera + +1. Upload a new image in MinIO under the bucket ending with `...-alert-api-{organisation_id}` +2. In pgAdmin, update the `cameras` table: + + * `last_image` with the filename + * `last_active_at` timestamp + + ``` ---