From dbea4a0329d3c1d758fc8d15f0f7054b18d430b7 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Tue, 12 Apr 2022 11:30:18 +0200 Subject: [PATCH 01/23] MAINT: Removed conflicting dependencies from setup.py --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 7a94c06..a470e34 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ], }, install_requires=[ - "flask<2.0", + "flask", "pymongo", "alembic", "flask-sqlalchemy", @@ -32,9 +32,7 @@ "flask-smorest", "flask-cors", "dtoolcore>=3.18.0", - "dtool_irods", "dtool_s3", - "dtool_ecs", "flask-jwt-extended[asymmetric_crypto]>=4.0", "pyyaml", ], From 6ee07f2dee4b1b7466872473189a1c55c29f7f2f Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Tue, 12 Apr 2022 15:39:56 +0200 Subject: [PATCH 02/23] MAINT: Needs marshmallow-sqlalchemy --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a470e34..52d26b4 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ "dtoolcore>=3.18.0", "dtool_s3", "flask-jwt-extended[asymmetric_crypto]>=4.0", + "marshmallow-sqlalchemy", "pyyaml", ], download_url="{}/tarball/{}".format(url, version), From 0e0a2c0a3cf5edd251743ee20fdd3ef4122fb6fe Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Tue, 12 Apr 2022 15:40:47 +0200 Subject: [PATCH 03/23] MAINT: server should not depend explicitly on dtool_s3 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 52d26b4..ab79e89 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ "flask-smorest", "flask-cors", "dtoolcore>=3.18.0", - "dtool_s3", "flask-jwt-extended[asymmetric_crypto]>=4.0", "marshmallow-sqlalchemy", "pyyaml", From d6c808314febca2408d8a7ce6cf6ceb9adbbbb9e Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 14 Apr 2022 00:49:57 +0200 Subject: [PATCH 04/23] MAINT: launch script for quick testing --- devel/README | 1 + devel/run.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 devel/README create mode 100644 devel/run.sh diff --git a/devel/README b/devel/README new file mode 100644 index 0000000..a98c519 --- /dev/null +++ b/devel/README @@ -0,0 +1 @@ +Simple development environment diff --git a/devel/run.sh b/devel/run.sh new file mode 100644 index 0000000..b15e6cf --- /dev/null +++ b/devel/run.sh @@ -0,0 +1,30 @@ +# run from repository root + +export PYTHONUNBUFFERED=1 +export FLASK_APP=dtool_lookup_server +export JWT_PUBLIC_KEY_FILE=$(pwd)/keys/jwt.pub +export JWT_PRIVATE_KEY_FILE=$(pwd)/keys/jwt + +openssl genrsa -out ${JWT_PRIVATE_KEY_FILE} 2048 +openssl rsa -in ${JWT_PRIVATE_KEY_FILE} -pubout -outform PEM -out ${JWT_PUBLIC_KEY_FILE} + +docker run -d -p 27017:27017 -v $(pwd)/data:/data/db mongo + +flask db init +flask db migrate +flask db upgrade + +flask base_uri add s3://test-bucket +flask base_uri add smb://test-share + +flask base_uri index s3://test-bucket +flask base_uri index smb://test-share + +flask user add testuser +flask user search_permission testuser s3://test-bucket +flask user register_permission testuser s3://test-bucket + +flask run + +# run after server is up +# flask user token testuser From 980cb1657aaf06525b237b6bacd8888d2a12c161 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 14 Apr 2022 09:34:26 +0200 Subject: [PATCH 05/23] MAINT: exclude auto-generated keys and pycharmm .idea folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index df5d925..112bf69 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ dist/* venv/* old-provision/* jwt-spike/* +keys/* + +.idea/* From 3e6fdfa5ed570d2de9464e6d9bf20d903087fe65 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 14 Apr 2022 13:24:33 +0200 Subject: [PATCH 06/23] typo --- dtool_lookup_server/dataset_routes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dtool_lookup_server/dataset_routes.py b/dtool_lookup_server/dataset_routes.py index 975e589..78abee7 100644 --- a/dtool_lookup_server/dataset_routes.py +++ b/dtool_lookup_server/dataset_routes.py @@ -161,7 +161,7 @@ def manifest(query: UriSchema): try: manifest_ = get_manifest_from_uri_by_user(username, uri) except AuthenticationError: - current_app.logger.info("AuthenticaitonError") + current_app.logger.info("AuthenticationError") abort(401) except AuthorizationError: current_app.logger.info("AuthorizationError") @@ -189,7 +189,7 @@ def readme(query: UriSchema): try: readme_ = get_readme_from_uri_by_user(username, uri) except AuthenticationError: - current_app.logger.info("AuthenticaitonError") + current_app.logger.info("AuthenticationError") abort(401) except AuthorizationError: current_app.logger.info("AuthorizationError") @@ -218,7 +218,7 @@ def annotations(query: UriSchema): try: annotations_ = get_annotations_from_uri_by_user(username, uri) except AuthenticationError: - current_app.logger.info("AuthenticaitonError") + current_app.logger.info("AuthenticationError") abort(401) except AuthorizationError: current_app.logger.info("AuthorizationError") From 3802e57a7b85788c5aaa5dafb4c21049b4403904 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Tue, 19 Apr 2022 23:01:28 +0200 Subject: [PATCH 07/23] MAINT: devel requirements.txt for quick testing setup --- devel/requirements.txt | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 devel/requirements.txt diff --git a/devel/requirements.txt b/devel/requirements.txt new file mode 100644 index 0000000..b1823ba --- /dev/null +++ b/devel/requirements.txt @@ -0,0 +1,68 @@ +setuptools_scm +alembic==1.7.7 +apispec==5.1.1 +boto3==1.21.40 +botocore==1.24.40 +certifi==2021.10.8 +cffi==1.15.0 +charset-normalizer==2.0.12 +click==8.1.2 +click-plugins==1.1.1 +cryptography==36.0.2 +dtool==3.26.2 +dtool-annotation==0.1.1 +dtool-cli==0.7.1 +dtool-config==0.4.1 +dtool-create==0.23.4 +dtool-http==0.5.1 +dtool-info==0.16.2 +dtool-lookup-client==0.1.0 +dtool-overlay==0.3.1 +dtool-s3==0.14.1 +dtool-smb==0.1.0 +dtool-symlink==0.3.1 +dtool-tag==0.1.1 +dtoolcore==3.18.2 +Flask==2.1.1 +Flask-Cors==3.0.10 +Flask-JWT-Extended==4.3.1 +flask-marshmallow==0.14.0 +Flask-Migrate==3.1.0 +Flask-PyMongo==2.3.0 +flask-smorest==0.37.0 +Flask-SQLAlchemy==2.5.1 +greenlet==1.1.2 +idna==3.3 +importlib-metadata==4.11.3 +importlib-resources==5.7.0 +itsdangerous==2.1.2 +Jinja2==3.1.1 +jmespath==1.0.0 +Mako==1.2.0 +MarkupSafe==2.1.1 +marshmallow==3.15.0 +marshmallow-sqlalchemy==0.28.0 +packaging==21.3 +parse==1.19.0 +pyasn1==0.4.8 +pycparser==2.21 +Pygments==2.11.2 +PyJWT==2.3.0 +pymongo==4.1.1 +pyparsing==3.0.8 +pysmb==1.2.7 +python-dateutil==2.8.2 +PyYAML==6.0 +requests==2.27.1 +ruamel.yaml==0.17.21 +ruamel.yaml.clib==0.2.6 +s3transfer==0.5.2 +six==1.16.0 +SQLAlchemy==1.4.35 +urllib3==1.26.9 +webargs==8.1.0 +Werkzeug==2.1.1 +zipp==3.8.0 +-e git+https://github.com/jotelha/dtool-lookup-server.git@openapi#egg=dtool_lookup_server +-e git+https://github.com/livMatS/dtool-lookup-server-plugin-scaffolding.git@openapi#egg=dtool-lookup-server-plugin-scaffolding +-e git+https://github.com/livMatS/dtool-lookup-server-dependency-graph-plugin.git@openapi#egg=dtool-lookup-server-dependency-graph-plugin From a64a661f5c1fc5537a33c57339f051fe8b5e15bd Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Wed, 20 Apr 2022 17:25:05 +0200 Subject: [PATCH 08/23] MAINT: AnnotationsSchema --- dtool_lookup_server/dataset_routes.py | 3 ++- dtool_lookup_server/schemas.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dtool_lookup_server/dataset_routes.py b/dtool_lookup_server/dataset_routes.py index 78abee7..1114722 100644 --- a/dtool_lookup_server/dataset_routes.py +++ b/dtool_lookup_server/dataset_routes.py @@ -29,6 +29,7 @@ ValidationError, ) from dtool_lookup_server.schemas import ( + AnnotationSchema, UriSchema, RegisterDatasetSchema, SearchDatasetSchema, @@ -206,7 +207,7 @@ def readme(query: UriSchema): @bp.route("/annotations", methods=["POST"]) @bp.arguments(UriSchema) -@bp.response(200, Dict) +@bp.response(200, AnnotationsSchema) @jwt_required() def annotations(query: UriSchema): """Request the dataset annotations.""" diff --git a/dtool_lookup_server/schemas.py b/dtool_lookup_server/schemas.py index 60309c4..a67154b 100644 --- a/dtool_lookup_server/schemas.py +++ b/dtool_lookup_server/schemas.py @@ -66,6 +66,7 @@ class SearchDatasetSchema(Schema): uuids = List(UUID) tags = List(String) + class SummarySchema(Schema): number_of_datasets = Integer() creator_usernames = List(String) @@ -81,3 +82,7 @@ class UserResponseSchema(Schema): is_admin = Boolean() register_permissions_on_base_uris = List(String) search_permissions_on_base_uris = List(String) + + +class AnnotationsSchema(Schema): + annotations = Dict() From 70289aa1f10d7b385af2cfa916d1cfd91afe8fa6 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 14:05:03 +0200 Subject: [PATCH 09/23] DOC: development environment and openapi-generator-cli examples --- devel/.gitignore | 3 ++ devel/README | 1 - devel/README.md | 38 ++++++++++++++++++++++ devel/export_token.sh | 11 +++++++ devel/openapi-generator-example.sh | 46 +++++++++++++++++++++++++++ devel/python-client-config.yml | 45 ++++++++++++++++++++++++++ devel/python-legacy-client-config.yml | 31 ++++++++++++++++++ devel/requirements.txt | 3 +- devel/run.sh | 9 +++--- devel/setup.sh | 15 +++++++++ 10 files changed, 195 insertions(+), 7 deletions(-) create mode 100644 devel/.gitignore delete mode 100644 devel/README create mode 100644 devel/README.md create mode 100644 devel/export_token.sh create mode 100644 devel/openapi-generator-example.sh create mode 100644 devel/python-client-config.yml create mode 100644 devel/python-legacy-client-config.yml create mode 100644 devel/setup.sh diff --git a/devel/.gitignore b/devel/.gitignore new file mode 100644 index 0000000..185fb12 --- /dev/null +++ b/devel/.gitignore @@ -0,0 +1,3 @@ +data/ +migrations/ +out/ diff --git a/devel/README b/devel/README deleted file mode 100644 index a98c519..0000000 --- a/devel/README +++ /dev/null @@ -1 +0,0 @@ -Simple development environment diff --git a/devel/README.md b/devel/README.md new file mode 100644 index 0000000..404bd87 --- /dev/null +++ b/devel/README.md @@ -0,0 +1,38 @@ +# Development environment + +Simple development environment. + + +## Setup + +`setup.sh` creates a Python virtual environment within the repository's root and installs requirements specified within `requirements.txt`. + + +## Start + +`run.sh` starts a local dtool lookup server on port 5000. Expects valid `~/.config/dtool/dtool.json` or setup via environment variables. + +## Token + +`source export_token.sh` generates JWT token and exports $TOKEN and $HEADER environment variables for authentification. Run after starting server. + +## Environement variables + +`source env.rc` sets environment variables for flask. Use to manually issue commands like `flask run`. + +## openapi-generator-cli + +`openapi-generator-example.sh` contains a few examples on how to use openapi-generator-cli with the dtool-lookup-server. +Different Python API generators exist. Only `python-legacy` is able to generate `asyncio`-based API. +Running `openapi-generator-cli` will produce two auto-generated Python packages, `out/python` and `out/python-legacy`, +former `urllib3`-based, latter `asyncio`-based. + +The `python*client-config.yml` files configure Python-specific behavior for openapi-generator-cli. + +Use + + export GIT_TOKEN=xyz + cd out/python-legacy + bash git_push.sh + +to push the generated API to a remote repository if in possession of a valid token. diff --git a/devel/export_token.sh b/devel/export_token.sh new file mode 100644 index 0000000..1c13f3c --- /dev/null +++ b/devel/export_token.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Script in $APPDIR" +source ${APPDIR}/env.rc + +TOKEN=$(flask user token testuser) +echo "Generated token '${TOKEN}'" +export TOKEN + +HEADER="Authorization: Bearer $TOKEN" +export HEADER diff --git a/devel/openapi-generator-example.sh b/devel/openapi-generator-example.sh new file mode 100644 index 0000000..507b2e3 --- /dev/null +++ b/devel/openapi-generator-example.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Generate openapi python client from openapi description. +# +# Validate local description file: +# +# docker run --rm -v "${PWD}:/local" openapitools/openapi-generator-cli validate -i /local/openapi.json +# +# Validate openapi descrition provided by local lookup server: +# +# docker run --rm --network="host" openapitools/openapi-generator-cli validate -i http://127.0.0.1:5000/doc/openapi.json +# +# Generate python API from local descritption file: +# +# docker run --rm -v "${PWD}:/local" openapitools/openapi-generator-cli generate \ +# -i /local/openapi.json \ +# -g python \ +# -o /local/out/python +# +# Generate python client api generator sample config file: +# +# docker run --rm openapitools/openapi-generator-cli config-help -g python -f yamlsample > python-client-config.yml +# +# Generate python API from openapi descrition provided by local lookup server: +# +docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ + --package-name "dtool_lookup_openapi_client" \ + --git-user-id "jotelha" \ + --git-repo-id "dtool-lookup-openapi-client" \ + --release-note "auto-generated openapi client for dtool-lookup-server" \ + -c /local/python-client-config.yml \ + -i http://127.0.0.1:5000/doc/openapi.json \ + -o /local/out/python \ + -g python \ + --verbose + +docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ + --package-name "dtool_lookup_openapi_client" \ + --git-user-id "jotelha" \ + --git-repo-id "dtool-lookup-openapi-python-legacy-client" \ + --release-note "auto-generated openapi client for dtool-lookup-server" \ + -c /local/python-legacy-client-config.yml \ + -i http://127.0.0.1:5000/doc/openapi.json \ + -o /local/out/python-legacy \ + -g python-legacy \ + --verbose diff --git a/devel/python-client-config.yml b/devel/python-client-config.yml new file mode 100644 index 0000000..23d4745 --- /dev/null +++ b/devel/python-client-config.yml @@ -0,0 +1,45 @@ +# Description: python package name (convention: snake_case). +# packageName: dtool_lookup_openapi_client + +# Description: python project name in setup.py (e.g. petstore-api). +# projectName: + +# Description: python package version. +# packageVersion: 1.0.0 + +# Description: python package URL. +# packageUrl: + +# Description: Hides the generation timestamp when files are generated. +# hideGenerationTimestamp: true + +# Description: Specifies that only a library source code is to be generated. +# generateSourceCodeOnly: false + +# Description: use the nose test framework +# useNose: false + +# Description: Set the recursion limit. If not set, use the system default value. +# recursionLimit: + +# Description: library template (sub-template) to use: asyncio, tornado, urllib3 +# library: asyncio +# will lead to +# Exception in thread "main" java.lang.RuntimeException: Only the `urllib3` library is supported in the refactored `python` client generator at the moment. Please fall back to `python-legacy` client generator for the time being. We welcome contributions to add back `asyncio`, `tornado` support to the `python` client generator. + +library: urllib3 + +# Description: when accessing unset attribute, return `None` instead of raising `ApiAttributeError` +# pythonAttrNoneIfUnset: false + +# Description: If set to true then the required variables are included as positional arguments in __init__ and _from_openapi_data methods. Note: this can break some composition use cases. To learn more read PR #8802. +# initRequiredVars: false + +# Description: If false, the 'additionalProperties' implementation (set to true by default) is compliant with the OAS and JSON schema specifications. If true (default), keep the old (incorrect) behaviour that 'additionalProperties' is set to false by default. +# Available Values: +# false +# The 'additionalProperties' implementation is compliant with the OAS and JSON schema specifications. +# true +# Keep the old (incorrect) behaviour that 'additionalProperties' is set to false by default. NOTE: this option breaks composition and will be removed in 6.0.0 +# disallowAdditionalPropertiesIfNotPresent: false + diff --git a/devel/python-legacy-client-config.yml b/devel/python-legacy-client-config.yml new file mode 100644 index 0000000..e20d65f --- /dev/null +++ b/devel/python-legacy-client-config.yml @@ -0,0 +1,31 @@ +# Description: python package name (convention: snake_case). +packageName: dtool_lookup_openapi_client + +# Description: python project name in setup.py (e.g. petstore-api). +projectName: dtool-lookup-openapi-python-legacy-client + +# Description: python package version. +# packageVersion: 0.0.1 + +# Description: python package URL. +packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-legacy-client + +# Description: Sort method arguments to place required parameters before optional parameters. +# sortParamsByRequiredFlag: true + +# Description: Hides the generation timestamp when files are generated. +# hideGenerationTimestamp: true + +# Description: Specifies that only a library source code is to be generated. +# generateSourceCodeOnly: false + +# Description: use the nose test framework +# useNose: false + +# Description: Set the recursion limit. If not set, use the system default value. +# recursionLimit: + +# Description: library template (sub-template) to use: asyncio, tornado, urllib3 +# library: urllib3 +library: asyncio + diff --git a/devel/requirements.txt b/devel/requirements.txt index b1823ba..3f1aaa4 100644 --- a/devel/requirements.txt +++ b/devel/requirements.txt @@ -1,4 +1,3 @@ -setuptools_scm alembic==1.7.7 apispec==5.1.1 boto3==1.21.40 @@ -66,3 +65,5 @@ zipp==3.8.0 -e git+https://github.com/jotelha/dtool-lookup-server.git@openapi#egg=dtool_lookup_server -e git+https://github.com/livMatS/dtool-lookup-server-plugin-scaffolding.git@openapi#egg=dtool-lookup-server-plugin-scaffolding -e git+https://github.com/livMatS/dtool-lookup-server-dependency-graph-plugin.git@openapi#egg=dtool-lookup-server-dependency-graph-plugin +-e git+https://github.com/livMatS/dtool-lookup-server-direct-mongo-plugin.git@openapi#egg=dtool-lookup-server-direct-mongo-plugin +-e git+https://github.com/livMatS/dtool-lookup-server-notification-plugin.git@openapi#egg=dtool-lookup-server-notification-plugin diff --git a/devel/run.sh b/devel/run.sh index b15e6cf..a62cd4c 100644 --- a/devel/run.sh +++ b/devel/run.sh @@ -1,9 +1,8 @@ +#!/usr/bin/env bash # run from repository root - -export PYTHONUNBUFFERED=1 -export FLASK_APP=dtool_lookup_server -export JWT_PUBLIC_KEY_FILE=$(pwd)/keys/jwt.pub -export JWT_PRIVATE_KEY_FILE=$(pwd)/keys/jwt +APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +echo "Script in $APPDIR" +source ${APPDIR}/env.rc openssl genrsa -out ${JWT_PRIVATE_KEY_FILE} 2048 openssl rsa -in ${JWT_PRIVATE_KEY_FILE} -pubout -outform PEM -out ${JWT_PUBLIC_KEY_FILE} diff --git a/devel/setup.sh b/devel/setup.sh new file mode 100644 index 0000000..21b0069 --- /dev/null +++ b/devel/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash -x +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" + +python3 -m venv "${ROOTDIR}/venv" + +echo "Activate venv in ${ROOTDIR}/venv" +source "${ROOTDIR}/venv/bin/activate" + +pip install --upgrade pip + +pip install setuptools_scm + +pip install -r "${ROOTDIR}/devel/requirements.txt" + +pip install -e "${ROOTDIR}" From f86de8321f96ca509b03aa2797faee696584eaea Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 14:06:15 +0200 Subject: [PATCH 10/23] MAINT: empty ConfigSchema --- dtool_lookup_server/config_routes.py | 3 +++ dtool_lookup_server/dataset_routes.py | 2 +- dtool_lookup_server/schemas.py | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dtool_lookup_server/config_routes.py b/dtool_lookup_server/config_routes.py index af69aec..dfa9a95 100644 --- a/dtool_lookup_server/config_routes.py +++ b/dtool_lookup_server/config_routes.py @@ -11,6 +11,8 @@ from dtool_lookup_server import AuthenticationError +from dtool_lookup_server.schemas import ConfigSchema + from dtool_lookup_server.utils import config_to_dict @@ -18,6 +20,7 @@ @bp.route("/info", methods=["GET"]) +@bp.response(200, ConfigSchema) @jwt_required() def server_config(): """Return the JSON-serialized server configuration.""" diff --git a/dtool_lookup_server/dataset_routes.py b/dtool_lookup_server/dataset_routes.py index 1114722..4a3d3d0 100644 --- a/dtool_lookup_server/dataset_routes.py +++ b/dtool_lookup_server/dataset_routes.py @@ -29,7 +29,7 @@ ValidationError, ) from dtool_lookup_server.schemas import ( - AnnotationSchema, + AnnotationsSchema, UriSchema, RegisterDatasetSchema, SearchDatasetSchema, diff --git a/dtool_lookup_server/schemas.py b/dtool_lookup_server/schemas.py index a67154b..073e58f 100644 --- a/dtool_lookup_server/schemas.py +++ b/dtool_lookup_server/schemas.py @@ -85,4 +85,8 @@ class UserResponseSchema(Schema): class AnnotationsSchema(Schema): - annotations = Dict() + pass + + +class ConfigSchema(Schema): + pass From abf62ea61a0fc11c1a29f7ba36b50a714e90ddc2 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 18:46:11 +0200 Subject: [PATCH 11/23] DOC: Working auto-generated openapi client --- ...tool_lookup_openapi_client_usage_sample.py | 36 +++++++++++++++++++ devel/openapi-generator-example.sh | 15 +++++++- devel/python-client-config.yml | 6 ++-- devel/python-experimental-client-config.yml | 27 ++++++++++++++ devel/run.sh | 8 +++-- 5 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 devel/dtool_lookup_openapi_client_usage_sample.py create mode 100644 devel/python-experimental-client-config.yml mode change 100644 => 100755 devel/run.sh diff --git a/devel/dtool_lookup_openapi_client_usage_sample.py b/devel/dtool_lookup_openapi_client_usage_sample.py new file mode 100644 index 0000000..5618972 --- /dev/null +++ b/devel/dtool_lookup_openapi_client_usage_sample.py @@ -0,0 +1,36 @@ +import os +import time +import dtool_lookup_openapi_client +from pprint import pprint +from dtool_lookup_openapi_client.api import dataset_api +# from dtool_lookup_openapi_client.model.base_uri import BaseURI +from dtool_lookup_openapi_client.model.dataset_sql_alchemy import DatasetSQLAlchemy as Dataset +from dtool_lookup_openapi_client.model.error import Error +from dtool_lookup_openapi_client.model.pagination_metadata import PaginationMetadata +# Defining the host is optional and defaults to http://localhost +# See configuration.py for a list of all supported configuration parameters. + +# The client must configure the authentication and authorization parameters +# in accordance with the API server security policy. +# Examples for each auth method are provided below, use the example that +# satisfies your auth use case. + +token = os.environ["TOKEN"] + +# Configure Bearer authorization (JWT): bearerAuth +configuration = dtool_lookup_openapi_client.Configuration(host="http://localhost:5000", access_token=token) + + +# Enter a context with an instance of the API client +with dtool_lookup_openapi_client.ApiClient(configuration) as api_client: + # Create an instance of the API class + api_instance = dataset_api.DatasetApi(api_client) + page = 1 # int | (optional) (default to 1) + page_size = 10 # int | (optional) (default to 10) + + try: + # List all base_uris. + api_response = api_instance.dataset_list_get(page=page, page_size=page_size) + pprint(api_response) + except dtool_lookup_openapi_client.ApiException as e: + print("Exception when calling BaseUriApi->admin_base_uri_list_get: %s\n" % e) diff --git a/devel/openapi-generator-example.sh b/devel/openapi-generator-example.sh index 507b2e3..858593e 100644 --- a/devel/openapi-generator-example.sh +++ b/devel/openapi-generator-example.sh @@ -26,7 +26,7 @@ docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ --package-name "dtool_lookup_openapi_client" \ --git-user-id "jotelha" \ - --git-repo-id "dtool-lookup-openapi-client" \ + --git-repo-id "dtool-lookup-openapi-python-client" \ --release-note "auto-generated openapi client for dtool-lookup-server" \ -c /local/python-client-config.yml \ -i http://127.0.0.1:5000/doc/openapi.json \ @@ -44,3 +44,16 @@ docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitool -o /local/out/python-legacy \ -g python-legacy \ --verbose + +# workaround for https://github.com/OpenAPITools/openapi-generator/issues/12196 +curl http://127.0.0.1:5000/doc/openapi.json | jq '. + {"x-original-swagger-version": .openapi}' > openapi.json +docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ + --package-name "dtool_lookup_openapi_client" \ + --git-user-id "jotelha" \ + --git-repo-id "dtool-lookup-openapi-python-experimental-client" \ + --release-note "auto-generated openapi client for dtool-lookup-server" \ + -c /local/python-experimental-client-config.yml \ + -i /local/openapi.json \ + -o /local/out/python-experimental \ + -g python-experimental \ + --verbose diff --git a/devel/python-client-config.yml b/devel/python-client-config.yml index 23d4745..13bcf3f 100644 --- a/devel/python-client-config.yml +++ b/devel/python-client-config.yml @@ -1,14 +1,14 @@ # Description: python package name (convention: snake_case). -# packageName: dtool_lookup_openapi_client +packageName: dtool_lookup_openapi_client # Description: python project name in setup.py (e.g. petstore-api). -# projectName: +projectName: dtool-lookup-openapi-python-client # Description: python package version. # packageVersion: 1.0.0 # Description: python package URL. -# packageUrl: +packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-client # Description: Hides the generation timestamp when files are generated. # hideGenerationTimestamp: true diff --git a/devel/python-experimental-client-config.yml b/devel/python-experimental-client-config.yml new file mode 100644 index 0000000..bc4be77 --- /dev/null +++ b/devel/python-experimental-client-config.yml @@ -0,0 +1,27 @@ +# Description: python package name (convention: snake_case). +packageName: dtool_lookup_openapi_client + +# Description: python project name in setup.py (e.g. petstore-api). +projectName: dtool-lookup-openapi-python-experimental-client + +# Description: python package version. +# PackageVersion: 1.0.0 + +# Description: python package URL. +# packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-experimental-client + +# Description: Hides the generation timestamp when files are generated. +# hideGenerationTimestamp: true + +# Description: Specifies that only a library source code is to be generated. +# generateSourceCodeOnly: false + +# Description: use the nose test framework +# useNose: false + +# Description: Set the recursion limit. If not set, use the system default value. +# recursionLimit: + +# Description: library template (sub-template) to use: urllib3 +library: urllib3 + diff --git a/devel/run.sh b/devel/run.sh old mode 100644 new mode 100755 index a62cd4c..9a0b542 --- a/devel/run.sh +++ b/devel/run.sh @@ -1,13 +1,15 @@ -#!/usr/bin/env bash +#!/bin/bash -x # run from repository root +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "Script in $APPDIR" +echo "Repository in $ROOTDIR, Script in $APPDIR" source ${APPDIR}/env.rc openssl genrsa -out ${JWT_PRIVATE_KEY_FILE} 2048 openssl rsa -in ${JWT_PRIVATE_KEY_FILE} -pubout -outform PEM -out ${JWT_PUBLIC_KEY_FILE} -docker run -d -p 27017:27017 -v $(pwd)/data:/data/db mongo +mkdir -p ${ROOTDIR}/data +docker run --user ${UID}:${UID} -d -p 27017:27017 -v ${ROOTDIR}/data:/data/db mongo:latest flask db init flask db migrate From f61ba5aa3ae8d9208864f06f0184ad1ab1d6328e Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 18:53:32 +0200 Subject: [PATCH 12/23] DOC: three APIs --- devel/README.md | 5 +++-- devel/export_token.sh | 0 devel/openapi-generator-example.sh | 0 devel/setup.sh | 0 4 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 devel/export_token.sh mode change 100644 => 100755 devel/openapi-generator-example.sh mode change 100644 => 100755 devel/setup.sh diff --git a/devel/README.md b/devel/README.md index 404bd87..1ce65a6 100644 --- a/devel/README.md +++ b/devel/README.md @@ -23,9 +23,10 @@ Simple development environment. ## openapi-generator-cli `openapi-generator-example.sh` contains a few examples on how to use openapi-generator-cli with the dtool-lookup-server. + Different Python API generators exist. Only `python-legacy` is able to generate `asyncio`-based API. -Running `openapi-generator-cli` will produce two auto-generated Python packages, `out/python` and `out/python-legacy`, -former `urllib3`-based, latter `asyncio`-based. +Running `openapi-generator-example.sh` will produce three auto-generated Python packages, `out/python`, `out/python-experimental`, and `out/python-legacy`, +former two `urllib3`-based, latter `asyncio`-based. Only `out/python` works for now against a running lookup server. The `python*client-config.yml` files configure Python-specific behavior for openapi-generator-cli. diff --git a/devel/export_token.sh b/devel/export_token.sh old mode 100644 new mode 100755 diff --git a/devel/openapi-generator-example.sh b/devel/openapi-generator-example.sh old mode 100644 new mode 100755 diff --git a/devel/setup.sh b/devel/setup.sh old mode 100644 new mode 100755 From eca0305dbe1ed1575e51f2b5367e72dbca6b1cab Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 18:54:41 +0200 Subject: [PATCH 13/23] * Different cases in BaseURI and BaseUri, BaseURISchema, BaseUriSchema in sql_models.py and schmemas.py caused troubles for autogenerated Python client API. For now, resorted to suboptimal suffix "SQLAlchemy" for the schemata defined within sql_models.py. Do we need actually need the different schema in sql_models.BaseURISQLAlchemySchema(ma.SQLAlchemyAutoSchema) and schemas.class BaseURISchema(Schema) ? * sql_models.Dataset(db.Model) defines columns frozen_at and created_at as db.DateTime(), its method as_dict() converts those with dtoolcore.utils.timestamp(self.frozen_at). This, however, results in the autogenerated client API to expect a serialized datetime object and throw an exception when receiving float. For now, inverted the previos behavior of the server to always return timestamps as floats to returning datetime objects instead. What should be the way here in the future? * Standardized spelling of Uri to URI in models and schema for all occurences. --- dtool_lookup_server/base_uri_routes.py | 8 ++++---- dtool_lookup_server/dataset_routes.py | 26 ++++++++++++------------ dtool_lookup_server/permission_routes.py | 6 +++--- dtool_lookup_server/schemas.py | 4 ++-- dtool_lookup_server/sql_models.py | 7 ++++--- dtool_lookup_server/user_admin_routes.py | 4 ++-- dtool_lookup_server/utils.py | 19 +++++++++-------- 7 files changed, 39 insertions(+), 35 deletions(-) diff --git a/dtool_lookup_server/base_uri_routes.py b/dtool_lookup_server/base_uri_routes.py index 73a87a0..a9ff4b7 100644 --- a/dtool_lookup_server/base_uri_routes.py +++ b/dtool_lookup_server/base_uri_routes.py @@ -10,7 +10,7 @@ from dtool_lookup_server import ( AuthenticationError, ) -from dtool_lookup_server.sql_models import BaseURISchema, BaseURI +from dtool_lookup_server.sql_models import BaseURISQLAlchemySchema, BaseURI from dtool_lookup_server.utils import ( base_uri_exists, get_user_obj, @@ -21,9 +21,9 @@ @bp.route("/register", methods=["POST"]) -@bp.arguments(BaseURISchema, required=True) +@bp.arguments(BaseURISQLAlchemySchema, required=True) @jwt_required() -def register(parameter: BaseURISchema): +def register(parameter: BaseURISQLAlchemySchema): """Register a base URI. The user needs to be admin. @@ -52,7 +52,7 @@ def register(parameter: BaseURISchema): @bp.route("/list", methods=["GET"]) @bp.paginate() -@bp.response(200, BaseURISchema(many=True)) +@bp.response(200, BaseURISQLAlchemySchema(many=True)) @jwt_required() def base_uri_list(pagination_parameters): """List all base_uris. diff --git a/dtool_lookup_server/dataset_routes.py b/dtool_lookup_server/dataset_routes.py index 4a3d3d0..a491ce1 100644 --- a/dtool_lookup_server/dataset_routes.py +++ b/dtool_lookup_server/dataset_routes.py @@ -12,8 +12,8 @@ from flask_smorest.pagination import PaginationParameters from .sql_models import ( - BaseURISchema, - DatasetSchema + BaseURISQLAlchemySchema, + DatasetSQLAlchemySchema ) from marshmallow.fields import ( @@ -30,7 +30,7 @@ ) from dtool_lookup_server.schemas import ( AnnotationsSchema, - UriSchema, + URISchema, RegisterDatasetSchema, SearchDatasetSchema, SummarySchema, @@ -66,7 +66,7 @@ def summary_of_datasets(): @bp.route("/list", methods=["GET"]) -@bp.response(200, DatasetSchema(many=True)) +@bp.response(200, DatasetSQLAlchemySchema(many=True)) @bp.paginate() @jwt_required() def list_datasets(pagination_parameters: PaginationParameters): @@ -83,7 +83,7 @@ def list_datasets(pagination_parameters: PaginationParameters): @bp.route("/lookup/", methods=["GET"]) -@bp.response(200, DatasetSchema(many=True)) +@bp.response(200, DatasetSQLAlchemySchema(many=True)) @bp.paginate() @jwt_required() def lookup_datasets(pagination_parameters: PaginationParameters, uuid): @@ -101,7 +101,7 @@ def lookup_datasets(pagination_parameters: PaginationParameters, uuid): @bp.route("/search", methods=["POST"]) @bp.arguments(SearchDatasetSchema(partial=True)) -@bp.response(200, DatasetSchema(many=True)) +@bp.response(200, DatasetSQLAlchemySchema(many=True)) @bp.paginate() @jwt_required() def search_datasets( @@ -121,7 +121,7 @@ def search_datasets( @bp.route("/register", methods=["POST"]) @bp.arguments(RegisterDatasetSchema(partial=("created_at",))) -@bp.response(201, UriSchema) +@bp.response(201, URISchema) @jwt_required() def register(dataset: RegisterDatasetSchema): """Register a dataset. The user needs to have register permissions on the base_uri.""" @@ -150,9 +150,9 @@ def register(dataset: RegisterDatasetSchema): @bp.route("/manifest", methods=["POST"]) -@bp.arguments(UriSchema) +@bp.arguments(URISchema) @jwt_required() -def manifest(query: UriSchema): +def manifest(query: URISchema): """Request the dataset manifest.""" username = get_jwt_identity() if "uri" not in query: @@ -178,9 +178,9 @@ def manifest(query: UriSchema): @bp.route("/readme", methods=["POST"]) -@bp.arguments(UriSchema) +@bp.arguments(URISchema) @jwt_required() -def readme(query: UriSchema): +def readme(query: URISchema): """Request the dataset readme.""" username = get_jwt_identity() if "uri" not in query: @@ -206,10 +206,10 @@ def readme(query: UriSchema): @bp.route("/annotations", methods=["POST"]) -@bp.arguments(UriSchema) +@bp.arguments(URISchema) @bp.response(200, AnnotationsSchema) @jwt_required() -def annotations(query: UriSchema): +def annotations(query: URISchema): """Request the dataset annotations.""" username = get_jwt_identity() if "uri" not in query: diff --git a/dtool_lookup_server/permission_routes.py b/dtool_lookup_server/permission_routes.py index 7d7420d..2605921 100644 --- a/dtool_lookup_server/permission_routes.py +++ b/dtool_lookup_server/permission_routes.py @@ -10,16 +10,16 @@ AuthenticationError, ValidationError ) -from dtool_lookup_server.schemas import BaseUriSchema, UriPermissionSchema +from dtool_lookup_server.schemas import BaseURISchema, UriPermissionSchema bp = Blueprint("permissions", __name__, url_prefix="/admin/permission") @bp.route("/info", methods=["POST"]) -@bp.arguments(BaseUriSchema) +@bp.arguments(BaseURISchema) @bp.response(200, UriPermissionSchema) @jwt_required() -def permission_info(data: BaseUriSchema): +def permission_info(data: BaseURISchema): """Get information about the permissions on a base URI. The user needs to be admin. diff --git a/dtool_lookup_server/schemas.py b/dtool_lookup_server/schemas.py index 073e58f..d7f6fb0 100644 --- a/dtool_lookup_server/schemas.py +++ b/dtool_lookup_server/schemas.py @@ -11,11 +11,11 @@ ) -class UriSchema(Schema): +class URISchema(Schema): uri = String() -class BaseUriSchema(Schema): +class BaseURISchema(Schema): base_uri = String() diff --git a/dtool_lookup_server/sql_models.py b/dtool_lookup_server/sql_models.py index b6a6729..484bea4 100644 --- a/dtool_lookup_server/sql_models.py +++ b/dtool_lookup_server/sql_models.py @@ -106,17 +106,18 @@ def as_dict(self): } -class BaseURISchema(ma.SQLAlchemyAutoSchema): +class BaseURISQLAlchemySchema(ma.SQLAlchemyAutoSchema): class Meta: model = BaseURI fields = ('base_uri',) -class UserSchema(ma.SQLAlchemyAutoSchema): +class UserSQLAlchemySchema(ma.SQLAlchemyAutoSchema): class Meta: model = User -class DatasetSchema(ma.SQLAlchemyAutoSchema): + +class DatasetSQLAlchemySchema(ma.SQLAlchemyAutoSchema): class Meta: model = Dataset fields = ('base_uri', 'created_at', 'creator_username', 'frozen_at', 'created_at', 'name', 'uri', 'uuid') \ No newline at end of file diff --git a/dtool_lookup_server/user_admin_routes.py b/dtool_lookup_server/user_admin_routes.py index 7d1e808..a762632 100644 --- a/dtool_lookup_server/user_admin_routes.py +++ b/dtool_lookup_server/user_admin_routes.py @@ -14,7 +14,7 @@ from dtool_lookup_server.schemas import RegisterUserSchema from dtool_lookup_server.sql_models import ( User, - UserSchema + UserSQLAlchemySchema ) from dtool_lookup_server.utils import ( get_user_obj, @@ -57,7 +57,7 @@ def register(data: RegisterUserSchema): @bp.route("/list", methods=["GET"]) @bp.paginate() -@bp.response(200, UserSchema(many=True)) +@bp.response(200, UserSQLAlchemySchema(many=True)) @jwt_required() def list_users(pagination_parameters: PaginationParameters): """List the users in the dtool lookup server. diff --git a/dtool_lookup_server/utils.py b/dtool_lookup_server/utils.py index abb7b2d..f8d95c9 100644 --- a/dtool_lookup_server/utils.py +++ b/dtool_lookup_server/utils.py @@ -346,7 +346,10 @@ def list_datasets_by_user(username): datasets = [] for base_uri in user.search_base_uris: for ds in base_uri.datasets: - datasets.append(ds.as_dict()) + ds_dict = ds.as_dict() + ds_dict['frozen_at'] = _extract_frozen_at_as_datetime(ds_dict) + ds_dict['created_at'] = _extract_created_at_as_datetime(ds_dict) + datasets.append(ds_dict) return datasets @@ -410,10 +413,10 @@ def search_datasets_by_user(username, query): ) for ds in cx: - # Convert datetime object to float timestamp. - for key in ("created_at", "frozen_at"): - datetime_obj = ds[key] - ds[key] = dtoolcore.utils.timestamp(datetime_obj) + # don't convert datetime object to float timestamp. + # for key in ("created_at", "frozen_at"): + # datetime_obj = ds[key] + # ds[key] = dtoolcore.utils.timestamp(datetime_obj) datasets.append(ds) return datasets @@ -598,7 +601,7 @@ def _extract_created_at_as_datetime(admin_metadata): return datetime.utcfromtimestamp(created_at) -def _extract_frozen_at_as_datatime(admin_metadata): +def _extract_frozen_at_as_datetime(admin_metadata): frozen_at = admin_metadata["frozen_at"] frozen_at = float(frozen_at) return datetime.utcfromtimestamp(frozen_at) @@ -608,7 +611,7 @@ def register_dataset_admin_metadata(admin_metadata): """Register the admin metadata in the dataset SQL table.""" base_uri = get_base_uri_obj(admin_metadata["base_uri"]) - frozen_at = _extract_frozen_at_as_datatime(admin_metadata) + frozen_at = _extract_frozen_at_as_datetime(admin_metadata) created_at = _extract_created_at_as_datetime(admin_metadata) dataset = Dataset( @@ -646,7 +649,7 @@ def _register_dataset_descriptive_metadata(collection, dataset_info): if not dataset_info_is_valid(dataset_info): return None - frozen_at = _extract_frozen_at_as_datatime(dataset_info) + frozen_at = _extract_frozen_at_as_datetime(dataset_info) created_at = _extract_created_at_as_datetime(dataset_info) dataset_info["frozen_at"] = frozen_at From 5b9a59dd75eb6be3bf167919debedf5873b42236 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 21:11:18 +0200 Subject: [PATCH 14/23] MAINT: Removed silly SQLAlchemy suffix from scheme where not necessary --- dtool_lookup_server/dataset_routes.py | 8 ++++---- dtool_lookup_server/sql_models.py | 4 ++-- dtool_lookup_server/user_admin_routes.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dtool_lookup_server/dataset_routes.py b/dtool_lookup_server/dataset_routes.py index a491ce1..98dd913 100644 --- a/dtool_lookup_server/dataset_routes.py +++ b/dtool_lookup_server/dataset_routes.py @@ -13,7 +13,7 @@ from .sql_models import ( BaseURISQLAlchemySchema, - DatasetSQLAlchemySchema + DatasetSchema ) from marshmallow.fields import ( @@ -66,7 +66,7 @@ def summary_of_datasets(): @bp.route("/list", methods=["GET"]) -@bp.response(200, DatasetSQLAlchemySchema(many=True)) +@bp.response(200, DatasetSchema(many=True)) @bp.paginate() @jwt_required() def list_datasets(pagination_parameters: PaginationParameters): @@ -83,7 +83,7 @@ def list_datasets(pagination_parameters: PaginationParameters): @bp.route("/lookup/", methods=["GET"]) -@bp.response(200, DatasetSQLAlchemySchema(many=True)) +@bp.response(200, DatasetSchema(many=True)) @bp.paginate() @jwt_required() def lookup_datasets(pagination_parameters: PaginationParameters, uuid): @@ -101,7 +101,7 @@ def lookup_datasets(pagination_parameters: PaginationParameters, uuid): @bp.route("/search", methods=["POST"]) @bp.arguments(SearchDatasetSchema(partial=True)) -@bp.response(200, DatasetSQLAlchemySchema(many=True)) +@bp.response(200, DatasetSchema(many=True)) @bp.paginate() @jwt_required() def search_datasets( diff --git a/dtool_lookup_server/sql_models.py b/dtool_lookup_server/sql_models.py index 484bea4..c7e18da 100644 --- a/dtool_lookup_server/sql_models.py +++ b/dtool_lookup_server/sql_models.py @@ -112,12 +112,12 @@ class Meta: fields = ('base_uri',) -class UserSQLAlchemySchema(ma.SQLAlchemyAutoSchema): +class UserSchema(ma.SQLAlchemyAutoSchema): class Meta: model = User -class DatasetSQLAlchemySchema(ma.SQLAlchemyAutoSchema): +class DatasetSchema(ma.SQLAlchemyAutoSchema): class Meta: model = Dataset fields = ('base_uri', 'created_at', 'creator_username', 'frozen_at', 'created_at', 'name', 'uri', 'uuid') \ No newline at end of file diff --git a/dtool_lookup_server/user_admin_routes.py b/dtool_lookup_server/user_admin_routes.py index a762632..7d1e808 100644 --- a/dtool_lookup_server/user_admin_routes.py +++ b/dtool_lookup_server/user_admin_routes.py @@ -14,7 +14,7 @@ from dtool_lookup_server.schemas import RegisterUserSchema from dtool_lookup_server.sql_models import ( User, - UserSQLAlchemySchema + UserSchema ) from dtool_lookup_server.utils import ( get_user_obj, @@ -57,7 +57,7 @@ def register(data: RegisterUserSchema): @bp.route("/list", methods=["GET"]) @bp.paginate() -@bp.response(200, UserSQLAlchemySchema(many=True)) +@bp.response(200, UserSchema(many=True)) @jwt_required() def list_users(pagination_parameters: PaginationParameters): """List the users in the dtool lookup server. From 8d5b1313520ba1654f2cd7088d6e93ae3e1c35a1 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 23:13:23 +0200 Subject: [PATCH 15/23] DOC: export token --- ...token.sh => export_token_from_local_test_server.sh} | 0 devel/export_token_from_remote_token_generator.sh | 10 ++++++++++ 2 files changed, 10 insertions(+) rename devel/{export_token.sh => export_token_from_local_test_server.sh} (100%) create mode 100755 devel/export_token_from_remote_token_generator.sh diff --git a/devel/export_token.sh b/devel/export_token_from_local_test_server.sh similarity index 100% rename from devel/export_token.sh rename to devel/export_token_from_local_test_server.sh diff --git a/devel/export_token_from_remote_token_generator.sh b/devel/export_token_from_remote_token_generator.sh new file mode 100755 index 0000000..21e9022 --- /dev/null +++ b/devel/export_token_from_remote_token_generator.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +TOKEN=$(curl --insecure -H "Content-Type: application/json" \ + -X POST -d '{"username": "testuser", "password": "test_password" }' \ + https://localhost:5001/token | jq -r '.token') +echo "Generated token '${TOKEN}'" +export TOKEN + +HEADER="Authorization: Bearer $TOKEN" +export HEADER From 9d1b8cc751f1d1e143bae10d9454d86be438dc13 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Thu, 21 Apr 2022 23:59:24 +0200 Subject: [PATCH 16/23] MAINT: changed back to timestamps as floats in dataset schema --- dtool_lookup_server/sql_models.py | 8 +++++++- dtool_lookup_server/utils.py | 14 +++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/dtool_lookup_server/sql_models.py b/dtool_lookup_server/sql_models.py index c7e18da..0e1badc 100644 --- a/dtool_lookup_server/sql_models.py +++ b/dtool_lookup_server/sql_models.py @@ -2,6 +2,8 @@ from dtool_lookup_server import ma from dtool_lookup_server import sql_db as db +from marshmallow.fields import Float + search_permissions = db.Table( "search_permissions", db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True), @@ -118,6 +120,10 @@ class Meta: class DatasetSchema(ma.SQLAlchemyAutoSchema): + created_at = Float() + frozen_at = Float() + class Meta: model = Dataset - fields = ('base_uri', 'created_at', 'creator_username', 'frozen_at', 'created_at', 'name', 'uri', 'uuid') \ No newline at end of file + additional = ('base_uri', 'creator_username', 'name', 'uri', 'uuid') + diff --git a/dtool_lookup_server/utils.py b/dtool_lookup_server/utils.py index f8d95c9..144f86e 100644 --- a/dtool_lookup_server/utils.py +++ b/dtool_lookup_server/utils.py @@ -346,10 +346,7 @@ def list_datasets_by_user(username): datasets = [] for base_uri in user.search_base_uris: for ds in base_uri.datasets: - ds_dict = ds.as_dict() - ds_dict['frozen_at'] = _extract_frozen_at_as_datetime(ds_dict) - ds_dict['created_at'] = _extract_created_at_as_datetime(ds_dict) - datasets.append(ds_dict) + datasets.append(ds.as_dict()) return datasets @@ -412,11 +409,10 @@ def search_datasets_by_user(username, query): }, ) for ds in cx: - - # don't convert datetime object to float timestamp. - # for key in ("created_at", "frozen_at"): - # datetime_obj = ds[key] - # ds[key] = dtoolcore.utils.timestamp(datetime_obj) + # convert datetime object to float timestamp. + for key in ("created_at", "frozen_at"): + datetime_obj = ds[key] + ds[key] = dtoolcore.utils.timestamp(datetime_obj) datasets.append(ds) return datasets From 31e71b6d812bbbe3dd92bdf46e0f8372665240b9 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Fri, 22 Apr 2022 01:02:33 +0200 Subject: [PATCH 17/23] MAINT: Treat UUIDs as String in schema --- dtool_lookup_server/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dtool_lookup_server/schemas.py b/dtool_lookup_server/schemas.py index d7f6fb0..ec29abd 100644 --- a/dtool_lookup_server/schemas.py +++ b/dtool_lookup_server/schemas.py @@ -38,7 +38,7 @@ class ManifestSchema(Schema): class RegisterDatasetSchema(Schema): - uuid = UUID() + uuid = String() base_uri = String() uri = String() # dtoolcore_version should be included when storing (based on current version) but not required on the request @@ -63,7 +63,7 @@ class SearchDatasetSchema(Schema): free_text = String() creator_usernames = List(String) base_uris = List(String) - uuids = List(UUID) + uuids = List(String) tags = List(String) From c5bdfea7add54aa2b2bf9c3cc3b1b5e77461f84b Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Fri, 22 Apr 2022 14:49:09 +0200 Subject: [PATCH 18/23] MAINT: Specify remote host and token generator in env vars --- devel/dtool_lookup_openapi_client_usage_sample.py | 6 +++--- devel/export_token_from_local_test_server.sh | 8 ++++---- devel/export_token_from_remote_token_generator.sh | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/devel/dtool_lookup_openapi_client_usage_sample.py b/devel/dtool_lookup_openapi_client_usage_sample.py index 5618972..da1ae2c 100644 --- a/devel/dtool_lookup_openapi_client_usage_sample.py +++ b/devel/dtool_lookup_openapi_client_usage_sample.py @@ -15,10 +15,10 @@ # Examples for each auth method are provided below, use the example that # satisfies your auth use case. -token = os.environ["TOKEN"] - +token = os.getenv("DTOOL_LOOKUP_SERVER_TOKEN") +host = os.getenv("DTOOL_LOOKUP_SERVER_URL", "http://localhost:5000") # Configure Bearer authorization (JWT): bearerAuth -configuration = dtool_lookup_openapi_client.Configuration(host="http://localhost:5000", access_token=token) +configuration = dtool_lookup_openapi_client.Configuration(host=host, access_token=token) # Enter a context with an instance of the API client diff --git a/devel/export_token_from_local_test_server.sh b/devel/export_token_from_local_test_server.sh index 1c13f3c..75361c5 100755 --- a/devel/export_token_from_local_test_server.sh +++ b/devel/export_token_from_local_test_server.sh @@ -3,9 +3,9 @@ APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" echo "Script in $APPDIR" source ${APPDIR}/env.rc -TOKEN=$(flask user token testuser) -echo "Generated token '${TOKEN}'" -export TOKEN +DTOOL_LOOKUP_SERVER_TOKEN=$(flask user token testuser) +echo "Generated token '${DTOOL_LOOKUP_SERVER_TOKEN}'" +export DTOOL_LOOKUP_SERVER_TOKEN -HEADER="Authorization: Bearer $TOKEN" +HEADER="Authorization: Bearer ${DTOOL_LOOKUP_SERVER_TOKEN}" export HEADER diff --git a/devel/export_token_from_remote_token_generator.sh b/devel/export_token_from_remote_token_generator.sh index 21e9022..bb81f8e 100755 --- a/devel/export_token_from_remote_token_generator.sh +++ b/devel/export_token_from_remote_token_generator.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash -TOKEN=$(curl --insecure -H "Content-Type: application/json" \ +DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL="${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL:-https://localhost:5001/token}" + +echo "Fetch token from ${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL}." +DTOOL_LOOKUP_SERVER_TOKEN=$(curl --insecure -H "Content-Type: application/json" \ -X POST -d '{"username": "testuser", "password": "test_password" }' \ - https://localhost:5001/token | jq -r '.token') -echo "Generated token '${TOKEN}'" -export TOKEN + "${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL}" | jq -r '.token') +echo "Generated token '${DTOOL_LOOKUP_SERVER_TOKEN}'" +export DTOOL_LOOKUP_SERVER_TOKEN -HEADER="Authorization: Bearer $TOKEN" +HEADER="Authorization: Bearer ${DTOOL_LOOKUP_SERVER_TOKEN}" export HEADER From b509c3e30a8894d88128f7cedd1fa7ee8694fe35 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Fri, 22 Apr 2022 18:31:58 +0200 Subject: [PATCH 19/23] MAINT: aiopenapi3 client sample --- ...l_lookup_aiopenapi3_client_usage_sample.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 devel/dtool_lookup_aiopenapi3_client_usage_sample.py diff --git a/devel/dtool_lookup_aiopenapi3_client_usage_sample.py b/devel/dtool_lookup_aiopenapi3_client_usage_sample.py new file mode 100644 index 0000000..c000af7 --- /dev/null +++ b/devel/dtool_lookup_aiopenapi3_client_usage_sample.py @@ -0,0 +1,28 @@ +# works with https://github.com/jotelha/aiopenapi3/tree/auto-generation-of-operation-ids-and-authorization-via-jwt +import os +import asyncio + +from aiopenapi3 import OpenAPI +from pprint import pprint + +token = os.getenv("DTOOL_LOOKUP_SERVER_TOKEN") +host = os.getenv("DTOOL_LOOKUP_SERVER_URL", "http://localhost:5000") + +url = f"{host}/doc/openapi.json" + +async def main(): + api = await OpenAPI.load_async(url) + + class Server: + def __init__(self, url): + self.url = url + + server = Server(host) + api._root.servers.append(server) + api.authenticate(bearerAuth=token) + ret = await api._._config_info_get() + + print(type(ret)) + + +asyncio.run(main()) From dcfadfafa698adc552740fff960b60c688f5c7d9 Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Wed, 21 Sep 2022 14:30:34 +0200 Subject: [PATCH 20/23] MAINT: removed all devel relics --- devel/.gitignore | 3 - devel/README.md | 39 ----------- ...l_lookup_aiopenapi3_client_usage_sample.py | 28 -------- ...tool_lookup_openapi_client_usage_sample.py | 36 ---------- devel/export_token_from_local_test_server.sh | 11 --- ...xport_token_from_remote_token_generator.sh | 13 ---- devel/openapi-generator-example.sh | 59 ---------------- devel/python-client-config.yml | 45 ------------ devel/python-experimental-client-config.yml | 27 -------- devel/python-legacy-client-config.yml | 31 --------- devel/requirements.txt | 69 ------------------- devel/run.sh | 31 --------- devel/setup.sh | 15 ---- 13 files changed, 407 deletions(-) delete mode 100644 devel/.gitignore delete mode 100644 devel/README.md delete mode 100644 devel/dtool_lookup_aiopenapi3_client_usage_sample.py delete mode 100644 devel/dtool_lookup_openapi_client_usage_sample.py delete mode 100755 devel/export_token_from_local_test_server.sh delete mode 100755 devel/export_token_from_remote_token_generator.sh delete mode 100755 devel/openapi-generator-example.sh delete mode 100644 devel/python-client-config.yml delete mode 100644 devel/python-experimental-client-config.yml delete mode 100644 devel/python-legacy-client-config.yml delete mode 100644 devel/requirements.txt delete mode 100755 devel/run.sh delete mode 100755 devel/setup.sh diff --git a/devel/.gitignore b/devel/.gitignore deleted file mode 100644 index 185fb12..0000000 --- a/devel/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -data/ -migrations/ -out/ diff --git a/devel/README.md b/devel/README.md deleted file mode 100644 index 1ce65a6..0000000 --- a/devel/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Development environment - -Simple development environment. - - -## Setup - -`setup.sh` creates a Python virtual environment within the repository's root and installs requirements specified within `requirements.txt`. - - -## Start - -`run.sh` starts a local dtool lookup server on port 5000. Expects valid `~/.config/dtool/dtool.json` or setup via environment variables. - -## Token - -`source export_token.sh` generates JWT token and exports $TOKEN and $HEADER environment variables for authentification. Run after starting server. - -## Environement variables - -`source env.rc` sets environment variables for flask. Use to manually issue commands like `flask run`. - -## openapi-generator-cli - -`openapi-generator-example.sh` contains a few examples on how to use openapi-generator-cli with the dtool-lookup-server. - -Different Python API generators exist. Only `python-legacy` is able to generate `asyncio`-based API. -Running `openapi-generator-example.sh` will produce three auto-generated Python packages, `out/python`, `out/python-experimental`, and `out/python-legacy`, -former two `urllib3`-based, latter `asyncio`-based. Only `out/python` works for now against a running lookup server. - -The `python*client-config.yml` files configure Python-specific behavior for openapi-generator-cli. - -Use - - export GIT_TOKEN=xyz - cd out/python-legacy - bash git_push.sh - -to push the generated API to a remote repository if in possession of a valid token. diff --git a/devel/dtool_lookup_aiopenapi3_client_usage_sample.py b/devel/dtool_lookup_aiopenapi3_client_usage_sample.py deleted file mode 100644 index c000af7..0000000 --- a/devel/dtool_lookup_aiopenapi3_client_usage_sample.py +++ /dev/null @@ -1,28 +0,0 @@ -# works with https://github.com/jotelha/aiopenapi3/tree/auto-generation-of-operation-ids-and-authorization-via-jwt -import os -import asyncio - -from aiopenapi3 import OpenAPI -from pprint import pprint - -token = os.getenv("DTOOL_LOOKUP_SERVER_TOKEN") -host = os.getenv("DTOOL_LOOKUP_SERVER_URL", "http://localhost:5000") - -url = f"{host}/doc/openapi.json" - -async def main(): - api = await OpenAPI.load_async(url) - - class Server: - def __init__(self, url): - self.url = url - - server = Server(host) - api._root.servers.append(server) - api.authenticate(bearerAuth=token) - ret = await api._._config_info_get() - - print(type(ret)) - - -asyncio.run(main()) diff --git a/devel/dtool_lookup_openapi_client_usage_sample.py b/devel/dtool_lookup_openapi_client_usage_sample.py deleted file mode 100644 index da1ae2c..0000000 --- a/devel/dtool_lookup_openapi_client_usage_sample.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import time -import dtool_lookup_openapi_client -from pprint import pprint -from dtool_lookup_openapi_client.api import dataset_api -# from dtool_lookup_openapi_client.model.base_uri import BaseURI -from dtool_lookup_openapi_client.model.dataset_sql_alchemy import DatasetSQLAlchemy as Dataset -from dtool_lookup_openapi_client.model.error import Error -from dtool_lookup_openapi_client.model.pagination_metadata import PaginationMetadata -# Defining the host is optional and defaults to http://localhost -# See configuration.py for a list of all supported configuration parameters. - -# The client must configure the authentication and authorization parameters -# in accordance with the API server security policy. -# Examples for each auth method are provided below, use the example that -# satisfies your auth use case. - -token = os.getenv("DTOOL_LOOKUP_SERVER_TOKEN") -host = os.getenv("DTOOL_LOOKUP_SERVER_URL", "http://localhost:5000") -# Configure Bearer authorization (JWT): bearerAuth -configuration = dtool_lookup_openapi_client.Configuration(host=host, access_token=token) - - -# Enter a context with an instance of the API client -with dtool_lookup_openapi_client.ApiClient(configuration) as api_client: - # Create an instance of the API class - api_instance = dataset_api.DatasetApi(api_client) - page = 1 # int | (optional) (default to 1) - page_size = 10 # int | (optional) (default to 10) - - try: - # List all base_uris. - api_response = api_instance.dataset_list_get(page=page, page_size=page_size) - pprint(api_response) - except dtool_lookup_openapi_client.ApiException as e: - print("Exception when calling BaseUriApi->admin_base_uri_list_get: %s\n" % e) diff --git a/devel/export_token_from_local_test_server.sh b/devel/export_token_from_local_test_server.sh deleted file mode 100755 index 75361c5..0000000 --- a/devel/export_token_from_local_test_server.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "Script in $APPDIR" -source ${APPDIR}/env.rc - -DTOOL_LOOKUP_SERVER_TOKEN=$(flask user token testuser) -echo "Generated token '${DTOOL_LOOKUP_SERVER_TOKEN}'" -export DTOOL_LOOKUP_SERVER_TOKEN - -HEADER="Authorization: Bearer ${DTOOL_LOOKUP_SERVER_TOKEN}" -export HEADER diff --git a/devel/export_token_from_remote_token_generator.sh b/devel/export_token_from_remote_token_generator.sh deleted file mode 100755 index bb81f8e..0000000 --- a/devel/export_token_from_remote_token_generator.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL="${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL:-https://localhost:5001/token}" - -echo "Fetch token from ${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL}." -DTOOL_LOOKUP_SERVER_TOKEN=$(curl --insecure -H "Content-Type: application/json" \ - -X POST -d '{"username": "testuser", "password": "test_password" }' \ - "${DTOOL_LOOKUP_SERVER_TOKEN_GENERATOR_URL}" | jq -r '.token') -echo "Generated token '${DTOOL_LOOKUP_SERVER_TOKEN}'" -export DTOOL_LOOKUP_SERVER_TOKEN - -HEADER="Authorization: Bearer ${DTOOL_LOOKUP_SERVER_TOKEN}" -export HEADER diff --git a/devel/openapi-generator-example.sh b/devel/openapi-generator-example.sh deleted file mode 100755 index 858593e..0000000 --- a/devel/openapi-generator-example.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -# Generate openapi python client from openapi description. -# -# Validate local description file: -# -# docker run --rm -v "${PWD}:/local" openapitools/openapi-generator-cli validate -i /local/openapi.json -# -# Validate openapi descrition provided by local lookup server: -# -# docker run --rm --network="host" openapitools/openapi-generator-cli validate -i http://127.0.0.1:5000/doc/openapi.json -# -# Generate python API from local descritption file: -# -# docker run --rm -v "${PWD}:/local" openapitools/openapi-generator-cli generate \ -# -i /local/openapi.json \ -# -g python \ -# -o /local/out/python -# -# Generate python client api generator sample config file: -# -# docker run --rm openapitools/openapi-generator-cli config-help -g python -f yamlsample > python-client-config.yml -# -# Generate python API from openapi descrition provided by local lookup server: -# -docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ - --package-name "dtool_lookup_openapi_client" \ - --git-user-id "jotelha" \ - --git-repo-id "dtool-lookup-openapi-python-client" \ - --release-note "auto-generated openapi client for dtool-lookup-server" \ - -c /local/python-client-config.yml \ - -i http://127.0.0.1:5000/doc/openapi.json \ - -o /local/out/python \ - -g python \ - --verbose - -docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ - --package-name "dtool_lookup_openapi_client" \ - --git-user-id "jotelha" \ - --git-repo-id "dtool-lookup-openapi-python-legacy-client" \ - --release-note "auto-generated openapi client for dtool-lookup-server" \ - -c /local/python-legacy-client-config.yml \ - -i http://127.0.0.1:5000/doc/openapi.json \ - -o /local/out/python-legacy \ - -g python-legacy \ - --verbose - -# workaround for https://github.com/OpenAPITools/openapi-generator/issues/12196 -curl http://127.0.0.1:5000/doc/openapi.json | jq '. + {"x-original-swagger-version": .openapi}' > openapi.json -docker run --rm --network="host" -v "${PWD}:/local" --user $UID:$UID openapitools/openapi-generator-cli:latest generate \ - --package-name "dtool_lookup_openapi_client" \ - --git-user-id "jotelha" \ - --git-repo-id "dtool-lookup-openapi-python-experimental-client" \ - --release-note "auto-generated openapi client for dtool-lookup-server" \ - -c /local/python-experimental-client-config.yml \ - -i /local/openapi.json \ - -o /local/out/python-experimental \ - -g python-experimental \ - --verbose diff --git a/devel/python-client-config.yml b/devel/python-client-config.yml deleted file mode 100644 index 13bcf3f..0000000 --- a/devel/python-client-config.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Description: python package name (convention: snake_case). -packageName: dtool_lookup_openapi_client - -# Description: python project name in setup.py (e.g. petstore-api). -projectName: dtool-lookup-openapi-python-client - -# Description: python package version. -# packageVersion: 1.0.0 - -# Description: python package URL. -packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-client - -# Description: Hides the generation timestamp when files are generated. -# hideGenerationTimestamp: true - -# Description: Specifies that only a library source code is to be generated. -# generateSourceCodeOnly: false - -# Description: use the nose test framework -# useNose: false - -# Description: Set the recursion limit. If not set, use the system default value. -# recursionLimit: - -# Description: library template (sub-template) to use: asyncio, tornado, urllib3 -# library: asyncio -# will lead to -# Exception in thread "main" java.lang.RuntimeException: Only the `urllib3` library is supported in the refactored `python` client generator at the moment. Please fall back to `python-legacy` client generator for the time being. We welcome contributions to add back `asyncio`, `tornado` support to the `python` client generator. - -library: urllib3 - -# Description: when accessing unset attribute, return `None` instead of raising `ApiAttributeError` -# pythonAttrNoneIfUnset: false - -# Description: If set to true then the required variables are included as positional arguments in __init__ and _from_openapi_data methods. Note: this can break some composition use cases. To learn more read PR #8802. -# initRequiredVars: false - -# Description: If false, the 'additionalProperties' implementation (set to true by default) is compliant with the OAS and JSON schema specifications. If true (default), keep the old (incorrect) behaviour that 'additionalProperties' is set to false by default. -# Available Values: -# false -# The 'additionalProperties' implementation is compliant with the OAS and JSON schema specifications. -# true -# Keep the old (incorrect) behaviour that 'additionalProperties' is set to false by default. NOTE: this option breaks composition and will be removed in 6.0.0 -# disallowAdditionalPropertiesIfNotPresent: false - diff --git a/devel/python-experimental-client-config.yml b/devel/python-experimental-client-config.yml deleted file mode 100644 index bc4be77..0000000 --- a/devel/python-experimental-client-config.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Description: python package name (convention: snake_case). -packageName: dtool_lookup_openapi_client - -# Description: python project name in setup.py (e.g. petstore-api). -projectName: dtool-lookup-openapi-python-experimental-client - -# Description: python package version. -# PackageVersion: 1.0.0 - -# Description: python package URL. -# packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-experimental-client - -# Description: Hides the generation timestamp when files are generated. -# hideGenerationTimestamp: true - -# Description: Specifies that only a library source code is to be generated. -# generateSourceCodeOnly: false - -# Description: use the nose test framework -# useNose: false - -# Description: Set the recursion limit. If not set, use the system default value. -# recursionLimit: - -# Description: library template (sub-template) to use: urllib3 -library: urllib3 - diff --git a/devel/python-legacy-client-config.yml b/devel/python-legacy-client-config.yml deleted file mode 100644 index e20d65f..0000000 --- a/devel/python-legacy-client-config.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Description: python package name (convention: snake_case). -packageName: dtool_lookup_openapi_client - -# Description: python project name in setup.py (e.g. petstore-api). -projectName: dtool-lookup-openapi-python-legacy-client - -# Description: python package version. -# packageVersion: 0.0.1 - -# Description: python package URL. -packageUrl: https://github.com/jotelha/dtool-lookup-openapi-python-legacy-client - -# Description: Sort method arguments to place required parameters before optional parameters. -# sortParamsByRequiredFlag: true - -# Description: Hides the generation timestamp when files are generated. -# hideGenerationTimestamp: true - -# Description: Specifies that only a library source code is to be generated. -# generateSourceCodeOnly: false - -# Description: use the nose test framework -# useNose: false - -# Description: Set the recursion limit. If not set, use the system default value. -# recursionLimit: - -# Description: library template (sub-template) to use: asyncio, tornado, urllib3 -# library: urllib3 -library: asyncio - diff --git a/devel/requirements.txt b/devel/requirements.txt deleted file mode 100644 index 3f1aaa4..0000000 --- a/devel/requirements.txt +++ /dev/null @@ -1,69 +0,0 @@ -alembic==1.7.7 -apispec==5.1.1 -boto3==1.21.40 -botocore==1.24.40 -certifi==2021.10.8 -cffi==1.15.0 -charset-normalizer==2.0.12 -click==8.1.2 -click-plugins==1.1.1 -cryptography==36.0.2 -dtool==3.26.2 -dtool-annotation==0.1.1 -dtool-cli==0.7.1 -dtool-config==0.4.1 -dtool-create==0.23.4 -dtool-http==0.5.1 -dtool-info==0.16.2 -dtool-lookup-client==0.1.0 -dtool-overlay==0.3.1 -dtool-s3==0.14.1 -dtool-smb==0.1.0 -dtool-symlink==0.3.1 -dtool-tag==0.1.1 -dtoolcore==3.18.2 -Flask==2.1.1 -Flask-Cors==3.0.10 -Flask-JWT-Extended==4.3.1 -flask-marshmallow==0.14.0 -Flask-Migrate==3.1.0 -Flask-PyMongo==2.3.0 -flask-smorest==0.37.0 -Flask-SQLAlchemy==2.5.1 -greenlet==1.1.2 -idna==3.3 -importlib-metadata==4.11.3 -importlib-resources==5.7.0 -itsdangerous==2.1.2 -Jinja2==3.1.1 -jmespath==1.0.0 -Mako==1.2.0 -MarkupSafe==2.1.1 -marshmallow==3.15.0 -marshmallow-sqlalchemy==0.28.0 -packaging==21.3 -parse==1.19.0 -pyasn1==0.4.8 -pycparser==2.21 -Pygments==2.11.2 -PyJWT==2.3.0 -pymongo==4.1.1 -pyparsing==3.0.8 -pysmb==1.2.7 -python-dateutil==2.8.2 -PyYAML==6.0 -requests==2.27.1 -ruamel.yaml==0.17.21 -ruamel.yaml.clib==0.2.6 -s3transfer==0.5.2 -six==1.16.0 -SQLAlchemy==1.4.35 -urllib3==1.26.9 -webargs==8.1.0 -Werkzeug==2.1.1 -zipp==3.8.0 --e git+https://github.com/jotelha/dtool-lookup-server.git@openapi#egg=dtool_lookup_server --e git+https://github.com/livMatS/dtool-lookup-server-plugin-scaffolding.git@openapi#egg=dtool-lookup-server-plugin-scaffolding --e git+https://github.com/livMatS/dtool-lookup-server-dependency-graph-plugin.git@openapi#egg=dtool-lookup-server-dependency-graph-plugin --e git+https://github.com/livMatS/dtool-lookup-server-direct-mongo-plugin.git@openapi#egg=dtool-lookup-server-direct-mongo-plugin --e git+https://github.com/livMatS/dtool-lookup-server-notification-plugin.git@openapi#egg=dtool-lookup-server-notification-plugin diff --git a/devel/run.sh b/devel/run.sh deleted file mode 100755 index 9a0b542..0000000 --- a/devel/run.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -x -# run from repository root -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -APPDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "Repository in $ROOTDIR, Script in $APPDIR" -source ${APPDIR}/env.rc - -openssl genrsa -out ${JWT_PRIVATE_KEY_FILE} 2048 -openssl rsa -in ${JWT_PRIVATE_KEY_FILE} -pubout -outform PEM -out ${JWT_PUBLIC_KEY_FILE} - -mkdir -p ${ROOTDIR}/data -docker run --user ${UID}:${UID} -d -p 27017:27017 -v ${ROOTDIR}/data:/data/db mongo:latest - -flask db init -flask db migrate -flask db upgrade - -flask base_uri add s3://test-bucket -flask base_uri add smb://test-share - -flask base_uri index s3://test-bucket -flask base_uri index smb://test-share - -flask user add testuser -flask user search_permission testuser s3://test-bucket -flask user register_permission testuser s3://test-bucket - -flask run - -# run after server is up -# flask user token testuser diff --git a/devel/setup.sh b/devel/setup.sh deleted file mode 100755 index 21b0069..0000000 --- a/devel/setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -x -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" - -python3 -m venv "${ROOTDIR}/venv" - -echo "Activate venv in ${ROOTDIR}/venv" -source "${ROOTDIR}/venv/bin/activate" - -pip install --upgrade pip - -pip install setuptools_scm - -pip install -r "${ROOTDIR}/devel/requirements.txt" - -pip install -e "${ROOTDIR}" From e900829d7729e34f45bcde69b860e57de780051a Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Wed, 21 Sep 2022 14:39:36 +0200 Subject: [PATCH 21/23] MAINT: removed redundant requirement from setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4735d78..17c6deb 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ "flask-cors", "dtoolcore>=3.18.0", "flask-jwt-extended[asymmetric_crypto]>=4.0", - "marshmallow-sqlalchemy", "pyyaml", ], download_url="{}/tarball/{}".format(url, version), From 3f0fed78a6ed8b8eae184b735660309af7829eea Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Wed, 21 Sep 2022 18:09:44 +0200 Subject: [PATCH 22/23] MAINT: get a few openapi config parameters from environment --- dtool_lookup_server/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dtool_lookup_server/config.py b/dtool_lookup_server/config.py index 9bba2be..4d7652a 100644 --- a/dtool_lookup_server/config.py +++ b/dtool_lookup_server/config.py @@ -41,13 +41,13 @@ class Config(object): API_TITLE = "dtool-lookup-server API" API_VERSION = "v1" OPENAPI_VERSION = "3.0.2" - OPENAPI_URL_PREFIX = "/doc" - OPENAPI_REDOC_PATH = "/redoc" - OPENAPI_REDOC_URL = ( + OPENAPI_URL_PREFIX = os.environ.get("OPENAPI_URL_PREFIX", "/doc") + OPENAPI_REDOC_PATH = os.environ.get("OPENAPI_REDOC_PATH", "/redoc") + OPENAPI_REDOC_URL = os.environ.get("OPENAPI_REDOC_URL", "https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js" ) - OPENAPI_SWAGGER_UI_PATH = "/swagger" - OPENAPI_SWAGGER_UI_URL = "https://cdn.jsdelivr.net/npm/swagger-ui-dist/" + OPENAPI_SWAGGER_UI_PATH = os.environ.get("OPENAPI_SWAGGER_UI_PATH", "/swagger") + OPENAPI_SWAGGER_UI_URL = os.environ.get("OPENAPI_SWAGGER_UI_URL", "https://cdn.jsdelivr.net/npm/swagger-ui-dist/") API_SPEC_OPTIONS = { "x-internal-id": "2", "security": [{"bearerAuth": []}], From 2ba61cc9dae796c5921b570cb94a6acc21c73bbe Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Wed, 21 Sep 2022 18:47:42 +0200 Subject: [PATCH 23/23] MAINT: load api specs from env if available --- dtool_lookup_server/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dtool_lookup_server/config.py b/dtool_lookup_server/config.py index 4d7652a..5422817 100644 --- a/dtool_lookup_server/config.py +++ b/dtool_lookup_server/config.py @@ -1,3 +1,4 @@ +import json import os import dtool_lookup_server @@ -48,7 +49,7 @@ class Config(object): ) OPENAPI_SWAGGER_UI_PATH = os.environ.get("OPENAPI_SWAGGER_UI_PATH", "/swagger") OPENAPI_SWAGGER_UI_URL = os.environ.get("OPENAPI_SWAGGER_UI_URL", "https://cdn.jsdelivr.net/npm/swagger-ui-dist/") - API_SPEC_OPTIONS = { + API_SPEC_OPTIONS = json.loads(os.environ.get("API_SPEC_OPTIONS", """{ "x-internal-id": "2", "security": [{"bearerAuth": []}], "components": { @@ -60,7 +61,7 @@ class Config(object): } } }, - } + }""")) @classmethod def to_dict(cls):