Bits and pieces on openapi implementation

.gitignore

@@ -18,3 +18,6 @@ dist/*
 venv/*
 old-provision/*
 jwt-spike/*
+keys/*
+
+.idea/*

dtool_lookup_server/base_uri_routes.py

@@ -10,7 +10,7 @@
 from dtool_lookup_server import (
     AuthenticationError,
 )
-from dtool_lookup_server.sql_models import BaseURISchema, BaseURI
+from dtool_lookup_server.sql_models import BaseURISQLAlchemySchema, BaseURI
 from dtool_lookup_server.utils import (
     base_uri_exists,
     get_user_obj,
@@ -21,9 +21,9 @@
 
 
 @bp.route("/register", methods=["POST"])
-@bp.arguments(BaseURISchema, required=True)
+@bp.arguments(BaseURISQLAlchemySchema, required=True)
 @jwt_required()
-def register(parameter: BaseURISchema):
+def register(parameter: BaseURISQLAlchemySchema):
     """Register a base URI.
 
     The user needs to be admin.
@@ -52,7 +52,7 @@ def register(parameter: BaseURISchema):
 
 @bp.route("/list", methods=["GET"])
 @bp.paginate()
-@bp.response(200, BaseURISchema(many=True))
+@bp.response(200, BaseURISQLAlchemySchema(many=True))
 @jwt_required()
 def base_uri_list(pagination_parameters):
     """List all base_uris.

dtool_lookup_server/config.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 import dtool_lookup_server
@@ -41,14 +42,14 @@ class Config(object):
     API_TITLE = "dtool-lookup-server API"
     API_VERSION = "v1"
     OPENAPI_VERSION = "3.0.2"
-    OPENAPI_URL_PREFIX = "/doc"
-    OPENAPI_REDOC_PATH = "/redoc"
-    OPENAPI_REDOC_URL = (
+    OPENAPI_URL_PREFIX = os.environ.get("OPENAPI_URL_PREFIX", "/doc")
+    OPENAPI_REDOC_PATH = os.environ.get("OPENAPI_REDOC_PATH", "/redoc")
+    OPENAPI_REDOC_URL = os.environ.get("OPENAPI_REDOC_URL",
         "https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js"
     )
-    OPENAPI_SWAGGER_UI_PATH = "/swagger"
-    OPENAPI_SWAGGER_UI_URL = "https://cdn.jsdelivr.net/npm/swagger-ui-dist/"
-    API_SPEC_OPTIONS = {
+    OPENAPI_SWAGGER_UI_PATH = os.environ.get("OPENAPI_SWAGGER_UI_PATH", "/swagger")
+    OPENAPI_SWAGGER_UI_URL = os.environ.get("OPENAPI_SWAGGER_UI_URL", "https://cdn.jsdelivr.net/npm/swagger-ui-dist/")
+    API_SPEC_OPTIONS = json.loads(os.environ.get("API_SPEC_OPTIONS", """{
         "x-internal-id": "2",
         "security": [{"bearerAuth": []}],
         "components": {
@@ -60,7 +61,7 @@ class Config(object):
                 }
             }
         },
-    }
+    }"""))
 
     @classmethod
     def to_dict(cls):

dtool_lookup_server/config_routes.py

@@ -11,13 +11,16 @@
 
 from dtool_lookup_server import AuthenticationError
 
+from dtool_lookup_server.schemas import ConfigSchema
+
 from dtool_lookup_server.utils import config_to_dict
 
 
 bp = Blueprint("config", __name__, url_prefix="/config")
 
 
 @bp.route("/info", methods=["GET"])
+@bp.response(200, ConfigSchema)
 @jwt_required()
 def server_config():
     """Return the JSON-serialized server configuration."""

dtool_lookup_server/dataset_routes.py

@@ -12,7 +12,7 @@
 from flask_smorest.pagination import PaginationParameters
 
 from .sql_models import (
-    BaseURISchema,
+    BaseURISQLAlchemySchema,
     DatasetSchema
 )
 
@@ -29,7 +29,8 @@
     ValidationError,
 )
 from dtool_lookup_server.schemas import (
-    UriSchema,
+    AnnotationsSchema,
+    URISchema,
     RegisterDatasetSchema,
     SearchDatasetSchema,
     SummarySchema,
@@ -120,7 +121,7 @@ def search_datasets(
 
 @bp.route("/register", methods=["POST"])
 @bp.arguments(RegisterDatasetSchema(partial=("created_at",)))
-@bp.response(201, UriSchema)
+@bp.response(201, URISchema)
 @jwt_required()
 def register(dataset: RegisterDatasetSchema):
     """Register a dataset. The user needs to have register permissions on the base_uri."""
@@ -149,9 +150,9 @@ def register(dataset: RegisterDatasetSchema):
 
 
 @bp.route("/manifest", methods=["POST"])
-@bp.arguments(UriSchema)
+@bp.arguments(URISchema)
 @jwt_required()
-def manifest(query: UriSchema):
+def manifest(query: URISchema):
     """Request the dataset manifest."""
     username = get_jwt_identity()
     if "uri" not in query:
@@ -161,7 +162,7 @@ def manifest(query: UriSchema):
     try:
         manifest_ = get_manifest_from_uri_by_user(username, uri)
     except AuthenticationError:
-        current_app.logger.info("AuthenticaitonError")
+        current_app.logger.info("AuthenticationError")
         abort(401)
     except AuthorizationError:
         current_app.logger.info("AuthorizationError")
@@ -177,9 +178,9 @@ def manifest(query: UriSchema):
 
 
 @bp.route("/readme", methods=["POST"])
-@bp.arguments(UriSchema)
+@bp.arguments(URISchema)
 @jwt_required()
-def readme(query: UriSchema):
+def readme(query: URISchema):
     """Request the dataset readme."""
     username = get_jwt_identity()
     if "uri" not in query:
@@ -189,7 +190,7 @@ def readme(query: UriSchema):
     try:
         readme_ = get_readme_from_uri_by_user(username, uri)
     except AuthenticationError:
-        current_app.logger.info("AuthenticaitonError")
+        current_app.logger.info("AuthenticationError")
         abort(401)
     except AuthorizationError:
         current_app.logger.info("AuthorizationError")
@@ -205,10 +206,10 @@ def readme(query: UriSchema):
 
 
 @bp.route("/annotations", methods=["POST"])
-@bp.arguments(UriSchema)
-@bp.response(200, Dict)
+@bp.arguments(URISchema)
+@bp.response(200, AnnotationsSchema)
 @jwt_required()
-def annotations(query: UriSchema):
+def annotations(query: URISchema):
     """Request the dataset annotations."""
     username = get_jwt_identity()
     if "uri" not in query:
@@ -218,7 +219,7 @@ def annotations(query: UriSchema):
     try:
         annotations_ = get_annotations_from_uri_by_user(username, uri)
     except AuthenticationError:
-        current_app.logger.info("AuthenticaitonError")
+        current_app.logger.info("AuthenticationError")
         abort(401)
     except AuthorizationError:
         current_app.logger.info("AuthorizationError")

dtool_lookup_server/permission_routes.py

@@ -10,16 +10,16 @@
     AuthenticationError,
     ValidationError
 )
-from dtool_lookup_server.schemas import BaseUriSchema, UriPermissionSchema
+from dtool_lookup_server.schemas import BaseURISchema, UriPermissionSchema
 
 bp = Blueprint("permissions", __name__, url_prefix="/admin/permission")
 
 
 @bp.route("/info", methods=["POST"])
-@bp.arguments(BaseUriSchema)
+@bp.arguments(BaseURISchema)
 @bp.response(200, UriPermissionSchema)
 @jwt_required()
-def permission_info(data: BaseUriSchema):
+def permission_info(data: BaseURISchema):
     """Get information about the permissions on a base URI.
 
     The user needs to be admin.

dtool_lookup_server/schemas.py

@@ -11,11 +11,11 @@
 )
 
 
-class UriSchema(Schema):
+class URISchema(Schema):
     uri = String()
 
 
-class BaseUriSchema(Schema):
+class BaseURISchema(Schema):
     base_uri = String()
 
 
@@ -38,7 +38,7 @@ class ManifestSchema(Schema):
 
 
 class RegisterDatasetSchema(Schema):
-    uuid = UUID()
+    uuid = String()
     base_uri = String()
     uri = String()
     # dtoolcore_version should be included when storing (based on current version) but not required on the request
@@ -65,9 +65,10 @@ class SearchDatasetSchema(Schema):
     free_text = String()
     creator_usernames = List(String)
     base_uris = List(String)
-    uuids = List(UUID)
+    uuids = List(String)
     tags = List(String)
 
+
 class SummarySchema(Schema):
     number_of_datasets = Integer()
     creator_usernames = List(String)
@@ -83,3 +84,11 @@ class UserResponseSchema(Schema):
     is_admin = Boolean()
     register_permissions_on_base_uris = List(String)
     search_permissions_on_base_uris = List(String)
+
+
+class AnnotationsSchema(Schema):
+    pass
+
+
+class ConfigSchema(Schema):
+    pass

dtool_lookup_server/sql_models.py

@@ -2,6 +2,8 @@
 from dtool_lookup_server import ma
 from dtool_lookup_server import sql_db as db
 
+from marshmallow.fields import Float
+
 search_permissions = db.Table(
     "search_permissions",
     db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True),
@@ -110,7 +112,7 @@ def as_dict(self):
         }
 
 
-class BaseURISchema(ma.SQLAlchemyAutoSchema):
+class BaseURISQLAlchemySchema(ma.SQLAlchemyAutoSchema):
     class Meta:
         model = BaseURI
         fields = ('base_uri',)
@@ -120,7 +122,12 @@ class UserSchema(ma.SQLAlchemyAutoSchema):
     class Meta:
         model = User
 
+
 class DatasetSchema(ma.SQLAlchemyAutoSchema):
+    created_at = Float()
+    frozen_at = Float()
+
     class Meta:
         model = Dataset
-        fields = ('base_uri', 'created_at', 'creator_username', 'frozen_at', 'created_at', 'name', 'uri', 'uuid')
+        additional = ('base_uri', 'creator_username', 'name', 'uri', 'uuid')
+

dtool_lookup_server/utils.py

@@ -415,8 +415,7 @@ def search_datasets_by_user(username, query):
         },
     )
     for ds in cx:
-
-        # Convert datetime object to float timestamp.
+        # convert datetime object to float timestamp.
         for key in ("created_at", "frozen_at"):
             datetime_obj = ds[key]
             ds[key] = dtoolcore.utils.timestamp(datetime_obj)
@@ -604,7 +603,7 @@ def _extract_created_at_as_datetime(admin_metadata):
     return datetime.utcfromtimestamp(created_at)
 
 
-def _extract_frozen_at_as_datatime(admin_metadata):
+def _extract_frozen_at_as_datetime(admin_metadata):
     frozen_at = admin_metadata["frozen_at"]
     frozen_at = float(frozen_at)
     return datetime.utcfromtimestamp(frozen_at)
@@ -614,7 +613,7 @@ def register_dataset_admin_metadata(admin_metadata):
     """Register the admin metadata in the dataset SQL table."""
     base_uri = get_base_uri_obj(admin_metadata["base_uri"])
 
-    frozen_at = _extract_frozen_at_as_datatime(admin_metadata)
+    frozen_at = _extract_frozen_at_as_datetime(admin_metadata)
     created_at = _extract_created_at_as_datetime(admin_metadata)
 
     try:
@@ -664,7 +663,7 @@ def _register_dataset_descriptive_metadata(collection, dataset_info):
     if not dataset_info_is_valid(dataset_info):
         return None
 
-    frozen_at = _extract_frozen_at_as_datatime(dataset_info)
+    frozen_at = _extract_frozen_at_as_datetime(dataset_info)
     created_at = _extract_created_at_as_datetime(dataset_info)
 
     dataset_info["frozen_at"] = frozen_at

setup.py

@@ -33,9 +33,6 @@
         "marshmallow-sqlalchemy",
         "flask-cors",
         "dtoolcore>=3.18.0",
-        "dtool_irods",
-        "dtool_s3",
-        "dtool_ecs",
         "flask-jwt-extended[asymmetric_crypto]>=4.0",
         "pyyaml",
     ],