diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ceb7c42a..f9cb4de8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -39,7 +39,7 @@ security-scan: APPSECHUB_PARENT_PIPELINE_ID: $CI_PIPELINE_ID APPSECHUB_SCA_SBOM_GENERATOR: custom APPSECHUB_SBOM_PATH: sbom.cyclonedx.json - APPSECHUB_SBOM_MASK: "*bom*.json" + APPSECHUB_SBOM_MASK: '*bom*.json' CUSTOM_SBOM_GENERATOR_JOB_NAME: sbom-creation rules: - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH diff --git a/mddocs/_static/custom.css b/mddocs/_static/custom.css new file mode 100644 index 00000000..ac17ca7e --- /dev/null +++ b/mddocs/_static/custom.css @@ -0,0 +1,3 @@ +.logo { + width: 200px !important; +} diff --git a/mddocs/_static/icon.svg b/mddocs/_static/icon.svg new file mode 100644 index 00000000..186f363e --- /dev/null +++ b/mddocs/_static/icon.svg @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/_static/logo.svg b/mddocs/_static/logo.svg new file mode 100644 index 00000000..ce803900 --- /dev/null +++ b/mddocs/_static/logo.svg @@ -0,0 +1,2731 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/_static/logo_no_title.svg b/mddocs/_static/logo_no_title.svg new file mode 100644 index 00000000..e2021113 --- /dev/null +++ b/mddocs/_static/logo_no_title.svg @@ -0,0 +1,2693 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mddocs/_static/metrics.prom b/mddocs/_static/metrics.prom new file mode 100644 index 00000000..0a831744 --- /dev/null +++ b/mddocs/_static/metrics.prom @@ -0,0 +1 @@ +# Generated in CI diff --git a/mddocs/_static/openapi.json b/mddocs/_static/openapi.json new file mode 100644 index 00000000..7b3bf552 --- /dev/null +++ b/mddocs/_static/openapi.json @@ -0,0 +1,9 @@ +{ + "openapi": "3.1.0", + "version": "unknown", + "info": { + "title": "Generated in CI", + "version": "unknown" + }, + "paths": {} +} \ No newline at end of file diff --git a/mddocs/_static/redoc.html b/mddocs/_static/redoc.html new file mode 100644 index 00000000..3c4246e5 --- /dev/null +++ b/mddocs/_static/redoc.html @@ -0,0 +1,28 @@ + + + + + Horizon - ReDoc + + + + + + + + + + + + + + diff --git a/mddocs/_static/stats.prom b/mddocs/_static/stats.prom new file mode 100644 index 00000000..0a831744 --- /dev/null +++ b/mddocs/_static/stats.prom @@ -0,0 +1 @@ +# Generated in CI diff --git a/mddocs/_static/swagger.html b/mddocs/_static/swagger.html new file mode 100644 index 00000000..0f891cd9 --- /dev/null +++ b/mddocs/_static/swagger.html @@ -0,0 +1,26 @@ + + + + + + + SwaggerUI + + + + +
+ + + + diff --git a/mddocs/backend/architecture.md b/mddocs/backend/architecture.md new file mode 100644 index 00000000..60216a3b --- /dev/null +++ b/mddocs/backend/architecture.md @@ -0,0 +1,8 @@ +# Architecture { #backend-architecture } + +```mermaid +stateDiagram-v2 +[User] --> [RESTAPI] +[RESTAPI] --> [Database] +[RESTAPI] --> [LDAP] +``` diff --git a/mddocs/backend/auth/cached_ldap.md b/mddocs/backend/auth/cached_ldap.md new file mode 100644 index 00000000..0e70ffc1 --- /dev/null +++ b/mddocs/backend/auth/cached_ldap.md @@ -0,0 +1,107 @@ +# LDAP Cached Auth provider { #backend-auth-ldap-cached } + +## Description { #cached_ldap-description } + +Same as [LDAP Auth provider][backend-auth-ldap-cached], but if LDAP request for checking user credentials was successful, +credentials are stored in local cache (table in internal database, in form `login` + `hash(password)` + `update timestamp`). + +Next auth requests for the same login are performed against this cache **first**. LDAP requests are send *only* if cache have been expired. + +This allows to: + +- Bypass errors with LDAP availability, e.g. network errors +- Reduce number of requests made to LDAP. + +Downsides: + +- If user changed password, and cache is not expired yet, user may still log in with old credentials. +- Same if user was blocked in LDAP. + +## Interaction schema { #cached_ldap-interaction-schema } + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" +participant "LDAP" + +activate "Client" +alt First time auth | Empty cache | Cache expired +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Search for credentials cache by login +"Backend" ->> "Backend" : No items found or item expired, using LDAP +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Successful +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +"Backend" ->> "Backend" : Create user if not exist +"Backend" ->> "Backend" : Save credentials to cache +"Backend" ->> "Client" : Generate and return access_token + +else Using cache, LDAP is totally ignored +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Search for credentials cache by login +"Backend" ->> "Backend" : Found credentials, check for expiration +"Backend" ->> "Backend" : Not expired, validate password is matching hash +"Backend" ->> "Backend" : Password match, not calling LDAP +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Create user if not exist +"Backend" ->> "Client" : Generate and return access_token + +else Password mismatch with cache, LDAP is totally ignored +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Search for credentials cache by login +"Backend" ->> "Backend" : Found credentials, check for expiration +"Backend" ->> "Backend" : Not expired, validate password is matching hash +"Backend" ->> "Backend" : Password do not match local cache +"Backend" --x "Client" : 401 Unauthorized + +else No cache or cache expired, LDAP is unavailable +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Search for credentials cache by login +"Backend" ->> "Backend" : No items found or item expired, using LDAP +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" --x "LDAP" : Call bind(DN, password) +"Backend" --x "Client" : 503 Service unavailable + +else +Note right of "Client" : Other cases are same as for LDAPAuthProvider,\nlike lookup, blocked/deleted users +end + +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +"Backend" ->> "Client" : Return data + +else Token is expired +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 401 Unauthorized + +else User is deleted +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 404 Not found +end + +deactivate "Client" +``` + +## Configuration { #cached_ldap-configuration } + +Other settings are just the same as for `LDAPAuthProvider` + +::: horizon.backend.settings.auth.cached_ldap.CachedLDAPAuthProviderSettings + +::: horizon.backend.settings.auth.cached_ldap.LDAPCacheSettings + +::: horizon.backend.settings.auth.cached_ldap.LDAPCachePasswordHashSettings diff --git a/mddocs/backend/auth/custom.md b/mddocs/backend/auth/custom.md new file mode 100644 index 00000000..37f38c0c --- /dev/null +++ b/mddocs/backend/auth/custom.md @@ -0,0 +1,5 @@ +# Custom Auth provider { #backend-auth-custom } + +You can implement custom auth provider by inheriting from class below and implementing necessary methods. + +::: horizon.backend.providers.auth.AuthProvider diff --git a/mddocs/backend/auth/dummy.md b/mddocs/backend/auth/dummy.md new file mode 100644 index 00000000..7ce26db1 --- /dev/null +++ b/mddocs/backend/auth/dummy.md @@ -0,0 +1,68 @@ +# Dummy Auth provider { #backend-auth-dummy } + +## Description { #dummy-description } + +This auth provider allows to sign-in with any username and password, and and then issues an access token. + +After successful auth, username is saved to backend database. It is then used for creating audit records for any object change, see `changed_by` field. + +## Interaction schema { #dummy-interaction-schema } + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" + +activate "Client" +alt Successful case +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Password is completely ignored +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Create user if not exist +"Backend" ->> "Client" : Generate and return access_token + +else User is blocked +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Password is completely ignored +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 401 Unauthorized + +else User is deleted +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : Password is completely ignored +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 404 Not found +end + +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +"Backend" ->> "Client" : Return data + +else Token is expired +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database + +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 404 Not found +end + +deactivate "Client" +``` + +## Configuration { #dummy-configuration } + +::: horizon.backend.settings.auth.dummy.DummyAuthProviderSettings + +::: horizon.backend.settings.auth.jwt.JWTSettings diff --git a/mddocs/backend/auth/index.md b/mddocs/backend/auth/index.md new file mode 100644 index 00000000..4c0526e3 --- /dev/null +++ b/mddocs/backend/auth/index.md @@ -0,0 +1,26 @@ +# Auth Providers { #backend-auth-providers } + +Horizon supports different auth provider implementations. You can change implementation via settings: + +::: horizon.backend.settings.auth.AuthSettings + +## Auth providers + +* [Dummy Auth provider][backend-auth-dummy] + * [Description][dummy-description] + * [Interaction schema][dummy-interaction-schema] + * [Configuration][dummy-configuration] +* [LDAP Auth provider][backend-auth-ldap] + * [Description][ldap-description] + * [Strategies][ldap-strategies] + * [Interaction schema][ldap-interaction-schema] + * [Basic configuration][ldap-basic-configuration] + * [Lookup-related configuration][ldap-lookup-related-configuration] +* [LDAP Cached Auth provider][backend-auth-ldap-cached] + * [Description][cached_ldap-description] + * [Interaction schema][cached_ldap-interaction-schema] + * [Configuration][cached_ldap-configuration] + +## For developers + +* [Custom Auth provider][backend-auth-custom] diff --git a/mddocs/backend/auth/ldap.md b/mddocs/backend/auth/ldap.md new file mode 100644 index 00000000..a0bd60b6 --- /dev/null +++ b/mddocs/backend/auth/ldap.md @@ -0,0 +1,219 @@ +# LDAP Auth provider { #backend-auth-ldap } + +## Description { #ldap-description } + +This auth provider checks for user credentials in LDAP, and and then issues an access token. + +All requests to backend should be made with passing this access token. If token is expired, then new auth token should be issued. + +After successful auth, username is saved to backend database. It is then used for creating audit records for any object change, see `changed_by` field. + +!!! warning + + Until token is valid, no requests will be made to LDAP to check if user exists and not locked. So do not set access token expiration time for too long (e.g. longer than a day). + +## Strategies { #ldap-strategies } + +!!! note + + Basic LDAP terminology is explained here: [LDAP Overview](https://www.zytrax.com/books/ldap/ch2/) + +There are 2 strategies to check for user in LDAP: + +- Try to call `bind` request in LDAP with `DN` (`DistinguishedName`) and user password. `DN` is generated using [bind_dn_template][horizon.backend.settings.auth.ldap.LDAPSettings.bind_dn_template] +- First try to *lookup* for user (`search` request) in LDAP to get user's `DN` using some query, and then try to call `bind` using this `DN`. See [lookup settings][horizon.backend.settings.auth.ldap.LDAPSettings.lookup] + +By default, **lookup strategy is used**, as it can find user in a complex LDAP/ActiveDirectory environment. For example: + +- search for user by `uid`, e.g. `(uid={login})` or `(sAMAccountName={login})` +- search for user by several attributes, e.g. `(|(uid={login})(mail={login}@domain.com))` +- filter for entries, like `(&(uid={login})(objectClass=person)` +- filter for users matching a specific group or some other condition, like `(&(uid={login})(memberOf=cn=MyPrettyGroup,ou=Groups,dc=mycompany,dc=com))` + +After user is found in LDAP, its [uid_attribute][horizon.backend.settings.auth.ldap.LDAPSettings.uid_attribute] is used for audit records. + +## Interaction schema { #ldap-interaction-schema } + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" +participant "LDAP" + +activate "Client" +alt Successful case +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Successful +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +"Backend" ->> "Backend" : Create user if not exist +"Backend" ->> "Client" : Generate and return access_token + +else Wrong credentials | User blocker in LDAP +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" --x "Backend" : Bind error +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked in internal backend database +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +"Backend" --x "Client" : 404 Not found + +else User is deleted in internal backend database +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Return user info +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = login +"Backend" --x "Client" : 404 Not found + +else LDAP is unavailable +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : DN = bind_dn_template(login) +"Backend" --x "LDAP" : Call bind(DN, password) +"Backend" --x "Client" : 503 Service unavailable +end + +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +"Backend" ->> "Client" : Return data + +else Token is expired +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 401 Unauthorized + +else +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 404 Not found +end + +deactivate "Client" +``` + +```mermaid +sequenceDiagram +participant "Client" +participant "Backend" +participant "LDAP" + +"Backend" -> "LDAP" : bind(lookup.username, lookup.password) +activate "LDAP" +Note right of "LDAP" : Open connection \npool for\nsearch queries\n(optional, recommended) + +activate "Client" +alt Successful case +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->> "LDAP" : Call search(query, base_dn, attributes=*) +activate "LDAP" +"LDAP" ->> "Backend" : Return user DN and uid_attribute +deactivate "LDAP" +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Successful +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +"Backend" ->> "Backend" : Create user if not exist +"Backend" ->> "Client" : Generate and return access_token + +else Wrong credentials | User blocker in LDAP +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->> "LDAP" : Call search(query, base_dn, attributes=*) +activate "LDAP" +"LDAP" ->> "Backend" : Return user DN and uid_attribute +deactivate "LDAP" +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" --x "Backend" : Bind error +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked in internal backend database +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->> "LDAP" : Call search(query, base_dn, attributes=*) +activate "LDAP" +"LDAP" ->> "Backend" : Return user DN and uid_attribute +deactivate "LDAP" +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Successful +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +"Backend" --x "Client" : 404 Not found + +else User is deleted in internal backend database +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" ->> "LDAP" : Call search(query, base_dn, attributes=*) +activate "LDAP" +"LDAP" ->> "Backend" : Return user DN and uid_attribute +deactivate "LDAP" +"Backend" ->> "LDAP" : Call bind(DN, password) +"LDAP" ->> "Backend" : Successful +"Backend" ->> "Backend" : Check user in internal backend database,\nusername = uid_attribute from LDAP response +"Backend" --x "Client" : 404 Not found + +else LDAP is unavailable +"Client" ->> "Backend" : login + password +"Backend" ->> "Backend" : query = query_template(login) +"Backend" --x "LDAP" : Call search(query, base_dn, attributes=*) +"Backend" --x "Client" : 503 Service unavailable +end + +alt Successful case +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" ->> "Backend" : Get data +"Backend" ->> "Client" : Return data + +else Token is expired +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" --x "Client" : 401 Unauthorized + +else User is blocked +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 401 Unauthorized + +else User is deleted +"Client" ->> "Backend" : access_token +"Backend" ->> "Backend" : Validate token +"Backend" ->> "Backend" : Check user in internal backend database +"Backend" --x "Client" : 404 Not found +end + +deactivate "LDAP" +deactivate "Client" +``` + +## Basic configuration { #ldap-basic-configuration } + +::: horizon.backend.settings.auth.ldap.LDAPAuthProviderSettings + +::: horizon.backend.settings.auth.ldap.LDAPSettings + +::: horizon.backend.settings.auth.jwt.JWTSettings + +::: horizon.backend.settings.auth.ldap.LDAPConnectionPoolSettings + +## Lookup-related configuration { #ldap-lookup-related-configuration } + +::: horizon.backend.settings.auth.ldap.LDAPLookupSettings + +::: horizon.backend.settings.auth.ldap.LDAPCredentials diff --git a/mddocs/backend/configuration/cors.md b/mddocs/backend/configuration/cors.md new file mode 100644 index 00000000..4c7fe860 --- /dev/null +++ b/mddocs/backend/configuration/cors.md @@ -0,0 +1,5 @@ +# CORS settings { #backend-configuration-cors } + +These settings used to control [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) options. + +::: horizon.backend.settings.server.cors.CORSSettings diff --git a/mddocs/backend/configuration/database.md b/mddocs/backend/configuration/database.md new file mode 100644 index 00000000..f77b8909 --- /dev/null +++ b/mddocs/backend/configuration/database.md @@ -0,0 +1,3 @@ +# Database settings { #backend-configuration-database } + +::: horizon.backend.settings.database.DatabaseSettings diff --git a/mddocs/backend/configuration/debug.md b/mddocs/backend/configuration/debug.md new file mode 100644 index 00000000..d27938e9 --- /dev/null +++ b/mddocs/backend/configuration/debug.md @@ -0,0 +1,103 @@ +# Enabling debug { #backend-configuration-debug } + +## Return debug info in backend responses + +By default, server does not add error details to response bodies, +to avoid exposing instance-specific information to end users. + +You can change this by setting: + +```console +$ export HORIZON__SERVER__DEBUG=False +$ # start backend +$ curl -XPOST http://localhost:8000/failing/endpoint ... +{ + "error": { + "code": "unknown", + "message": "Got unhandled exception. Please contact support", + "details": null, + }, +} +``` + +```console +$ export HORIZON__SERVER__DEBUG=True +$ # start backend +$ curl -XPOST http://localhost:8000/failing/endpoint ... +Traceback (most recent call last): +File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +File ".../site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ + return await self.app(scope, receive, send) +``` + +!!! WARNING + This is only for development environment only. Do **NOT** use on production! + +## Print debug logs on backend + +See [Logging settings][backend-configuration-logging], but replace log level `INFO` with `DEBUG`. + +## Fill up `X-Request-ID` header on backend + +Server can add `X-Request-ID` header to responses, which allows to match request on client with backend response. + +This is done by `request_id` middleware, which is enabled by default and can configured as described below: + +::: horizon.backend.settings.server.request_id.RequestIDSettings + +## Print request ID to backend logs + +This is done by adding a specific filter to logging handler: + +### `logging.yml` + +--8<-- +horizon/backend/settings/server/log/plain.yml:6:12,17:18,25 +--8<-- + +Resulting logs look like: + +```text +2023-12-18 17:14:11.711 uvicorn.access:498 [INFO] 018c15e97a068ae09484f8c25e2799dd 127.0.0.1:34884 - "GET /monitoring/ping HTTP/1.1" 200 +``` + +## Use `X-Request-ID` header on client + +If client got `X-Request-ID` header from backend, it is printed to logs with `DEBUG` level: + +```pycon +>>> import logging +>>> logging.basicConfig(level=logging.DEBUG) +>>> client.ping() +DEBUG:urllib3.connectionpool:http://localhost:8000 "GET /monitoring/ping HTTP/1.1" 200 15 +DEBUG:horizon.client.base:Request ID: '018c15e97a068ae09484f8c25e2799dd' +``` + +Also, if backend response was not successful, `Request ID` is added to exception message: + +```pycon +>>> client.get_namespace("unknown") +requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://localhost:8000/v1/namespaces/unknown +Request ID: '018c15eb80fa81a6b38c9eaa519cd322' +``` + +## Fill up `X-Application-Version` header on backend + +Server can add `X-Application-Version` header to responses, which allows to determine which version of backend is deployed. + +This is done by `application_version` middleware, which is enabled by default and can configured as described below: + +::: horizon.backend.settings.server.application_version.ApplicationVersionSettings + +## Use `X-Application-Version` header on client + +If client got `X-Application-Version` header from backend, it is compared with client version. + +If versions do not match, a warning is shown: + +```pycon +>>> client.ping() +UserWarning: Horizon client version '0.0.9' does not match backend version '1.0.0'. Please upgrade. +``` diff --git a/mddocs/backend/configuration/index.md b/mddocs/backend/configuration/index.md new file mode 100644 index 00000000..22a5a5a9 --- /dev/null +++ b/mddocs/backend/configuration/index.md @@ -0,0 +1,16 @@ +# Configuration { #backend-configuration } + +* [Database][backend-configuration-database] +* [Logging][backend-configuration-logging] +* [Monitoring][backend-configuration-monitoring] +* [CORS][backend-configuration-cors] +* [Static_files][backend-configuration-static-files] +* [Openapi][backend-configuration-openapi] +* [Debug][backend-configuration-debug] + +::: horizon.backend.settings + options: + members: + - Settings + - server + - ServerSettings diff --git a/mddocs/backend/configuration/logging.md b/mddocs/backend/configuration/logging.md new file mode 100644 index 00000000..25e12b0a --- /dev/null +++ b/mddocs/backend/configuration/logging.md @@ -0,0 +1,3 @@ +# Logging settings { #backend-configuration-logging } + +::: horizon.backend.settings.server.log.LoggingSettings diff --git a/mddocs/backend/configuration/monitoring.md b/mddocs/backend/configuration/monitoring.md new file mode 100644 index 00000000..8a0016ad --- /dev/null +++ b/mddocs/backend/configuration/monitoring.md @@ -0,0 +1,19 @@ +# Setup monitoring { #backend-configuration-monitoring } + +Backend provides 2 endpoints with Prometheus compatible metrics: + +- `GET /monitoring/metrics` - server metrics, like number of requests per path and response status, CPU and RAM usage, and so on. + +## Example metrics + +!include(../../_static/metrics.prom) + +- `GET /monitoring/stats` - usage statistics, like number of users, namespaces, HWMs. + +## Example stats + +!include(../../_static/stats.prom) + +These endpoints are enabled and configured using settings below: + +::: horizon.backend.settings.server.monitoring.MonitoringSettings diff --git a/mddocs/backend/configuration/openapi.md b/mddocs/backend/configuration/openapi.md new file mode 100644 index 00000000..7c28b95a --- /dev/null +++ b/mddocs/backend/configuration/openapi.md @@ -0,0 +1,12 @@ +# OpenAPI settings { #backend-configuration-openapi } + +These settings used to control exposing OpenAPI.json and SwaggerUI/ReDoc endpoints. + +::: horizon.backend.settings.server.openapi + options: + members: + - OpenAPISettings + - SwaggerSettings + - RedocSettings + - LogoSettings + - FaviconSettings diff --git a/mddocs/backend/configuration/static_files.md b/mddocs/backend/configuration/static_files.md new file mode 100644 index 00000000..340cc9c6 --- /dev/null +++ b/mddocs/backend/configuration/static_files.md @@ -0,0 +1,5 @@ +# Serving static files { #backend-configuration-static-files } + +These settings used to control serving static files by backend. + +::: horizon.backend.settings.server.static_files.StaticFilesSettings diff --git a/mddocs/backend/install.md b/mddocs/backend/install.md new file mode 100644 index 00000000..dbb9e159 --- /dev/null +++ b/mddocs/backend/install.md @@ -0,0 +1,105 @@ +# Install & run backend { #backend-install } + +## With docker + +### Requirements + +- [Docker](https://docs.docker.com/engine/install/) +- [docker-compose](https://github.com/docker/compose/releases/) + +### Installation process + +Docker will download backend image of Horizon & Postgres, and run them. +Options can be set via `.env` file or `environment` section in `docker-compose.yml` + +### `docker-compose.yml` + +!include(../../docker-compose.yml) + +### `.env.docker` + +!include(../../env.docker) + +After container is started and ready, open [http://localhost:8000/docs](http://localhost:8000/docs). + +Users listed in `HORIZON__ENTRYPOINT__ADMIN_USERS` env variable will be automatically promoted to `SUPERADMIN` role. + +## Without docker + +### Requirements without docker + +- Python 3.7 or above +- Pydantic 2.x +- `libldap2-dev`, `libsasl2-dev`, `libkrb5-dev` (for [LDAP Auth provider][backend-auth-ldap]) +- Some relation database instance, like [Postgres](https://www.postgresql.org/) + +### Installation process without docker + +Install `data-horizon` package with following *extra* dependencies: + +```console +$ pip install data-horizon[backend,postgres,ldap] +... +``` + +Available *extras* are: + +- `backend` - main backend requirements, like FastAPI, SQLAlchemy and so on. +- `postgres` - requirements required to use Postgres as backend data storage. +- `ldap` - requirements used by [LDAP Auth provider][backend-auth-ldap]. + +!!! note + For **macOS** users, an additional step is required. [You need to install the “bonsai” Python library from source code](https://bonsai.readthedocs.io/en/latest/install.html#install-from-source-on-macos). This installation is necessary to work with LDAP. + +### Run database + +Start Postgres instance somewhere, and set up database url using environment variables: + +```bash +HORIZON__DATABASE__URL=postgresql+asyncpg://user:password@postgres-host:5432/database_name +``` + +You can use virtually any database supported by [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls), +but the only one we really tested is Postgres. + +See [Database settings][backend-configuration-database] for more options. + +### Run migrations + +To apply migrations (database structure changes) you need to execute following command: + +```console +$ python -m horizon.backend.db.migrations upgrade head +... +``` + +This is a thin wrapper around [alembic](https://alembic.sqlalchemy.org/en/latest/tutorial.html#running-our-first-migration) cli, +options and commands are just the same. + +!!! note "run migrations" + This command should be executed after each upgrade to new Horizon version. + +### Run backend + +To start backend server you need to execute following command: + +```console +$ python -m horizon.backend --host 0.0.0.0 --port 8000 +... +``` + +This is a thin wrapper around [uvicorn](https://www.uvicorn.org/#command-line-options) cli, +options and commands are just the same. + +After server is started and ready, open [http://localhost:8000/docs](http://localhost:8000/docs). + +### Add admin users + +To promote specific users to `SUPERADMIN` role, run the following script: + +```console +$ python -m horizon.backend.scripts.manage_admins add admin1 admin2 +... +``` + +See [Scripts][scripts] documentation. diff --git a/mddocs/backend/openapi.md b/mddocs/backend/openapi.md new file mode 100644 index 00000000..d3f21083 --- /dev/null +++ b/mddocs/backend/openapi.md @@ -0,0 +1,5 @@ +# OpenAPI specification { #backend-openapi } + +--8<-- +mddocs/_static/swagger.html +--8<-- diff --git a/mddocs/backend/scripts/index.md b/mddocs/backend/scripts/index.md new file mode 100644 index 00000000..c842f892 --- /dev/null +++ b/mddocs/backend/scripts/index.md @@ -0,0 +1,5 @@ +# Scripts { #scripts } + +Horizon includes several utility scripts to manage the application at a low level. + +* [Manage Admins][manage-admins-script] diff --git a/mddocs/backend/scripts/manage_admins.md b/mddocs/backend/scripts/manage_admins.md new file mode 100644 index 00000000..c4ee0300 --- /dev/null +++ b/mddocs/backend/scripts/manage_admins.md @@ -0,0 +1,7 @@ +# Manage Admins { #manage-admins-script } + +```argparse + :module: horizon.backend.scripts.manage_admins + :func: create_parser + :prog: python -m horizon.backend.scripts.manage_admins +``` diff --git a/mddocs/changelog.md b/mddocs/changelog.md new file mode 100644 index 00000000..307e7141 --- /dev/null +++ b/mddocs/changelog.md @@ -0,0 +1,19 @@ +# Changelog { #Changelog } + +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.1 (2025-01-28)][1.1.1] +* [1.0.2 (2024-11-21)][1.0.2] +* [1.0.1 (2024-06-27)][1.0.1] +* [1.0.0 (2024-06-10)][1.0.0] +* [0.2.1 (2024-05-29)][0.2.1] +* [0.2.0 (2024-05-15)][0.2.0] +* [0.1.3 (2024-05-02)][0.1.3] +* [0.1.2 (2024-04-02)][0.1.2] +* [0.1.1 (2024-03-27)][0.1.1] +* [0.0.13 (2024-02-13)][0.0.13] +* [0.0.12 (2024-01-24)][0.0.12] +* [0.0.11 (2024-01-22)][0.0.11] +* [0.0.10 (2024-01-22)][0.0.10] +* [0.0.9 (2024-01-19)][0.0.9] +* [0.0.8 (2024-01-18)][0.0.8] diff --git a/mddocs/changelog/0.0.10.md b/mddocs/changelog/0.0.10.md new file mode 100644 index 00000000..d2bc8d98 --- /dev/null +++ b/mddocs/changelog/0.0.10.md @@ -0,0 +1,5 @@ +# 0.0.10 (2024-01-22) { #0.0.10 } + +## Bug Fixes + +- Update `starlette-exporter` and `uvicorn` to latest versions. diff --git a/mddocs/changelog/0.0.11.md b/mddocs/changelog/0.0.11.md new file mode 100644 index 00000000..4d84db4a --- /dev/null +++ b/mddocs/changelog/0.0.11.md @@ -0,0 +1,5 @@ +# 0.0.11 (2024-01-22) { #0.0.11 } + +## Bug Fixes + +- Remove `starlette` from list of packages installed with `client-sync` extras. diff --git a/mddocs/changelog/0.0.12.md b/mddocs/changelog/0.0.12.md new file mode 100644 index 00000000..eccb99ea --- /dev/null +++ b/mddocs/changelog/0.0.12.md @@ -0,0 +1,5 @@ +# 0.0.12 (2024-01-24) { #0.0.12 } + +## Bug Fixes + +- Fix client compatibility with `urllib` 1.x . diff --git a/mddocs/changelog/0.0.13.md b/mddocs/changelog/0.0.13.md new file mode 100644 index 00000000..5edcd885 --- /dev/null +++ b/mddocs/changelog/0.0.13.md @@ -0,0 +1,5 @@ +# 0.0.13 (2024-02-13) { #0.0.13 } + +## Features + +- `Horizon` is open-source now. diff --git a/mddocs/changelog/0.0.8.md b/mddocs/changelog/0.0.8.md new file mode 100644 index 00000000..f310586a --- /dev/null +++ b/mddocs/changelog/0.0.8.md @@ -0,0 +1,6 @@ +# 0.0.8 (2024-01-18) { #0.0.8 } + +## Features + +- Added retry configuration support to `HorizonClientSync`. ([#DOP-12454](https://github.com/MobileTeleSystems/horizon/issues/DOP-12454)) +- Added `timeout` config parameter support to `HorizonClientSync`. ([#DOP-12545](https://github.com/MobileTeleSystems/horizon/issues/DOP-12545)) diff --git a/mddocs/changelog/0.0.9.md b/mddocs/changelog/0.0.9.md new file mode 100644 index 00000000..c3fb9c93 --- /dev/null +++ b/mddocs/changelog/0.0.9.md @@ -0,0 +1,5 @@ +# 0.0.9 (2024-01-19) { #0.0.9 } + +## Bug Fixes + +- Fix empty `/monitoring/metrics` data. diff --git a/mddocs/changelog/0.1.1.md b/mddocs/changelog/0.1.1.md new file mode 100644 index 00000000..43857e89 --- /dev/null +++ b/mddocs/changelog/0.1.1.md @@ -0,0 +1,27 @@ +# 0.1.1 (2024-03-27) { #0.1.1 } + +## Breaking Changes + +Users now required to explicitly have a role assigned within a namespace to manipulate HWMs as they could before. These changes enforce stricter access control and better management of user permissions within the system. + +- Add role model to Horizon, documentation available at [Role Permissions](../design/permissions.md#role-permissions). ([#27](https://github.com/MobileTeleSystems/horizon/issues/27), [#31](https://github.com/MobileTeleSystems/horizon/issues/31)) +- Restrict deletion of `Namespace` if there are any `hwms` related to it. ([#25](https://github.com/MobileTeleSystems/horizon/issues/25)) + +## Features + +- Add `Namespace History`. Now it is possible to view paginated history of actions for specific namespace. ([#24](https://github.com/MobileTeleSystems/horizon/issues/24)) +- Add `owner_id` field to `Namespace` model to keep track of the owner of the namespace. ([#26](https://github.com/MobileTeleSystems/horizon/issues/26)) +- Add support for managing `SUPERADMIN` roles. ([#36](https://github.com/MobileTeleSystems/horizon/issues/36)) +- Permissions Management: + - Add new API endpoint `PATCH /namespace/:id/permissions` for updating the permissions of users within a namespace. + - Add new API endpoint `GET /namespace/:id/permissions` for fetching the permissions of users within a specific namespace. + - Extend the Python client library with methods `get_namespace_permissions` and `update_namespace_permissions` to interact with the new API endpoints. ([#29](https://github.com/MobileTeleSystems/horizon/issues/29)) +- High Water Marks (HWMs) Management: + - Add new API endpoint `DELETE /hwm/` for bulk deletion of High Water Marks (HWMs) by namespace_id and a list of hwm_ids. + - Extend the Python client library with the method `bulk_delete_hwm` to interact with the new bulk delete HWM API endpoint. ([#37](https://github.com/MobileTeleSystems/horizon/issues/37)) + - Add new API endpoint `POST /hwm/copy` endpoint for copying HWMs between namespaces, with optional history copying. + - Extend the Python client library with the method `copy_hwms` to support the new HWM copy functionality. ([#42](https://github.com/MobileTeleSystems/horizon/issues/42)) + +## Improvements + +- Fix documentation examples. Make documentation more user-friendly. ([#20](https://github.com/MobileTeleSystems/horizon/issues/20)) diff --git a/mddocs/changelog/0.1.2.md b/mddocs/changelog/0.1.2.md new file mode 100644 index 00000000..2c2e7db2 --- /dev/null +++ b/mddocs/changelog/0.1.2.md @@ -0,0 +1,11 @@ +# 0.1.2 (2024-04-02) { #0.1.2 } + +## Features + +- Add new environment variable `HORIZON__ENTRYPOINT__ADMIN_USERS` to Docker image entrypoint. + Here you can pass of usernames which should be automatically promoted to `SUPERADMIN` role during backend startup. ([#45](https://github.com/MobileTeleSystems/horizon/issues/45)) + +## Improvements + +- Improve logging in `manage_admins` script. ([#46](https://github.com/MobileTeleSystems/horizon/issues/46)) +- Fix Pydantic v2 model warnings while starting backend. ([#47](https://github.com/MobileTeleSystems/horizon/issues/47)) diff --git a/mddocs/changelog/0.1.3.md b/mddocs/changelog/0.1.3.md new file mode 100644 index 00000000..2aa05fe1 --- /dev/null +++ b/mddocs/changelog/0.1.3.md @@ -0,0 +1,6 @@ +# 0.1.3 (2024-05-02) { #0.1.3 } + +## Improvements + +- Properly handle `SIGTERM` signals in Docker image entrypoint. +- Update dependencies diff --git a/mddocs/changelog/0.2.0.md b/mddocs/changelog/0.2.0.md new file mode 100644 index 00000000..aa891201 --- /dev/null +++ b/mddocs/changelog/0.2.0.md @@ -0,0 +1,16 @@ +# 0.2.0 (2024-05-15) { #0.2.0 } + +## Breaking Changes + +- Rename `/v1/namespace/:id/permissions` endpoint to `/v1/namespaces/:id/permissions`. ([#61](https://github.com/MobileTeleSystems/horizon/issues/61)) + +## Features + +- Allow using Horizon with multiple uvicorn workers ([#60](https://github.com/MobileTeleSystems/horizon/issues/60)): + - Add `pid` to log formatters + - Add `PROMETHEUS_MULTIPROC_DIR` to `docker-compose.yml` example + +## Bug Fixes + +- Use connection timeout while creating LDAP connections in the pool. ([#58](https://github.com/MobileTeleSystems/horizon/issues/58)) +- Fix response schema for invalid JSON input. diff --git a/mddocs/changelog/0.2.1.md b/mddocs/changelog/0.2.1.md new file mode 100644 index 00000000..debc8aae --- /dev/null +++ b/mddocs/changelog/0.2.1.md @@ -0,0 +1,7 @@ +# 0.2.1 (2024-05-29) { #0.2.1 } + +## Improvements + +- Fix LDAP connection pool configuration example. +- Update uvicorn to 0.30.0, including new multiprocessing workers manager. +- Update dependencies. diff --git a/mddocs/changelog/1.0.0.md b/mddocs/changelog/1.0.0.md new file mode 100644 index 00000000..115bab51 --- /dev/null +++ b/mddocs/changelog/1.0.0.md @@ -0,0 +1,7 @@ +# 1.0.0 (2024-06-10) { #1.0.0 } + +First production-ready release! + +## Improvements + +- Update dependencies diff --git a/mddocs/changelog/1.0.1.md b/mddocs/changelog/1.0.1.md new file mode 100644 index 00000000..53622747 --- /dev/null +++ b/mddocs/changelog/1.0.1.md @@ -0,0 +1,10 @@ +# 1.0.1 (2024-06-27) { #1.0.1 } + +## Dependencies + +- Bump minimal `urllib3` version to `1.26.0`, to avoid exceptions like: + +``` +ValidationError: 1 validation error for HorizonClientSync__root__ + __init__() got an unexpected keyword argument 'allowed_methods' (type=type_error) +``` diff --git a/mddocs/changelog/1.0.2.md b/mddocs/changelog/1.0.2.md new file mode 100644 index 00000000..71b47d12 --- /dev/null +++ b/mddocs/changelog/1.0.2.md @@ -0,0 +1,34 @@ +# 1.0.2 (2024-11-21) { #1.0.2 } + +## Bug fixes + +- Previously client after receiving 4xx responses from the server, raised `requests.exceptions.HTTPError` like: + + ```pycon + >>> client.update_namespace_permissions(namespace_id=234, changes=to_update) + Traceback (most recent call last): + File "horizon/horizon/client/base.py", line 135, in _handle_response + response.raise_for_status() + File "horizon/.venv/lib/python3.12/site-packages/requests/models.py", line 1021, in raise_for_status + raise HTTPError(http_error_msg, response=self) + requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http://localhost:8000/v1/namespaces/234/permissions + ``` + + Now it wraps all these exceptions with `horizon.commons.exceptions` classes, like: + + ```pycon + >>> client.update_namespace_permissions(namespace_id=234, changes=to_update) + Traceback (most recent call last): + File "", line 1, in + File "horizon/horizon/client/sync.py", line 914, in update_namespace_permissions + return self._request( # type: ignore[return-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "horizon/horizon/client/sync.py", line 1031, in _request + return self._handle_response(response, response_class) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "horizon/horizon/client/base.py", line 170, in _handle_response + raise get_exception() from http_exception + horizon.commons.exceptions.entity.EntityNotFoundError: Namespace with id=234 not found + ``` + + `horizon.commons.exceptions` exception types were documented long time ago, so this is not a breaking change, but a bug fix. diff --git a/mddocs/changelog/1.1.1.md b/mddocs/changelog/1.1.1.md new file mode 100644 index 00000000..643b3c2a --- /dev/null +++ b/mddocs/changelog/1.1.1.md @@ -0,0 +1,9 @@ +# 1.1.1 (2025-01-28) { #1.1.1 } + +## Improvements + +- Add compatibility with `Python 3.13` ([#94](https://github.com/MobileTeleSystems/horizon/issues/94)) +- Replace outdated `python-jose` dependency with `authlib.jose`, to fix security issues. ([#97](https://github.com/MobileTeleSystems/horizon/issues/97)) + +Note: preliminary release 1.1.0 was yanked from PyPI because it has wrong `horizon.__version__` value. +The source code of 1.1.1 and dependencies are just the same as 1.1.0. diff --git a/mddocs/changelog/1.1.2.md b/mddocs/changelog/1.1.2.md new file mode 100644 index 00000000..b1c3cbcb --- /dev/null +++ b/mddocs/changelog/1.1.2.md @@ -0,0 +1,7 @@ +# 1.1.2 (2025-04-07) { #1.1.2 } + +## Improvements + +- Reduce image size down to x3. +- Change docker image user from `root` to `horizon`, to improve security. +- SBOM file is generated on release. diff --git a/mddocs/changelog/DRAFT.md b/mddocs/changelog/DRAFT.md new file mode 100644 index 00000000..e69de29b diff --git a/mddocs/changelog/NEXT_RELEASE.md b/mddocs/changelog/NEXT_RELEASE.md new file mode 100644 index 00000000..a9831f9d --- /dev/null +++ b/mddocs/changelog/NEXT_RELEASE.md @@ -0,0 +1 @@ +% towncrier release notes start diff --git a/mddocs/changelog/index.md b/mddocs/changelog/index.md new file mode 100644 index 00000000..be216b27 --- /dev/null +++ b/mddocs/changelog/index.md @@ -0,0 +1,18 @@ +# Changelog + +* [1.1.2 (2025-04-07)][1.1.2] +* [1.1.1 (2025-01-28)][1.1.1] +* [1.0.2 (2024-11-21)][1.0.2] +* [1.0.1 (2024-06-27)][1.0.1] +* [1.0.0 (2024-06-10)][1.0.0] +* [0.2.1 (2024-05-29)][0.2.1] +* [0.2.0 (2024-05-15)][0.2.0] +* [0.1.3 (2024-05-02)][0.1.3] +* [0.1.2 (2024-04-02)][0.1.2] +* [0.1.1 (2024-03-27)][0.1.1] +* [0.0.13 (2024-02-13)][0.0.13] +* [0.0.12 (2024-01-24)][0.0.12] +* [0.0.11 (2024-01-22)][0.0.11] +* [0.0.10 (2024-01-22)][0.0.10] +* [0.0.9 (2024-01-19)][0.0.9] +* [0.0.8 (2024-01-18)][0.0.8] diff --git a/mddocs/changelog/next_release/.keep b/mddocs/changelog/next_release/.keep new file mode 100644 index 00000000..e69de29b diff --git a/mddocs/client/auth.md b/mddocs/client/auth.md new file mode 100644 index 00000000..587997bc --- /dev/null +++ b/mddocs/client/auth.md @@ -0,0 +1,14 @@ +# Auth { #client-auth } + +These classes are used for adding auth information to requests send from client. + +::: horizon.client.auth.LoginPassword + options: + members: + - login + - password + +::: horizon.client.auth.AccessToken + options: + members: + - token diff --git a/mddocs/client/exceptions.md b/mddocs/client/exceptions.md new file mode 100644 index 00000000..b628a075 --- /dev/null +++ b/mddocs/client/exceptions.md @@ -0,0 +1,61 @@ +# Exceptions { #client-exceptions } + +These exception classes are used in client implementations. + +## Base + +::: horizon.commons.exceptions.base.ApplicationError + options: + members: + - message + - details + +## Authorization + +::: horizon.commons.exceptions.auth.ApplicationError + options: + members: + - message + - details + +## Permissions + +::: horizon.commons.exceptions.permission.PermissionDeniedError + options: + members: + - message + - details + - required_role + - actual_role + +::: horizon.commons.exceptions.bad_request.BadRequestError + options: + members: + - reason + +## Entity + +::: horizon.commons.exceptions.entity.EntityNotFoundError + options: + members: + - message + - details + - entity_type + - field + - value + +::: horizon.commons.exceptions.entity.EntityAlreadyExistsError + options: + members: + - message + - details + - entity_type + - field + - value + +## Service + +::: horizon.commons.exceptions.service.ServiceError + options: + members: + - message diff --git a/mddocs/client/install.md b/mddocs/client/install.md new file mode 100644 index 00000000..84fdf1dc --- /dev/null +++ b/mddocs/client/install.md @@ -0,0 +1,19 @@ +# Install client { #client-install } + +## Requirements + +- Python 3.7 or above +- Pydantic 1.x or 2.x + +## Installation process + +Install `data-horizon` package with following *extra* dependencies: + +```console +$ pip install data-horizon[client-sync] +... +``` + +Available *extras* are: + +- `client-sync` - [Sync client][client-sync], based on [authlib](https://docs.authlib.org) and [requests](https://requests.readthedocs.io) diff --git a/mddocs/client/schemas/hwm.md b/mddocs/client/schemas/hwm.md new file mode 100644 index 00000000..600f2afb --- /dev/null +++ b/mddocs/client/schemas/hwm.md @@ -0,0 +1,12 @@ +# HWM-related schemas { #client-schemas-hwm } + +::: horizon.commons.schemas.v1.hwm + options: + members: + - HWMResponseV1 + - HWMListResponseV1 + - HWMPaginateQueryV1 + - HWMCreateRequestV1 + - HWMUpdateRequestV1 + - HWMBulkCopyRequestV1 + - HWMBulkDeleteRequestV1 diff --git a/mddocs/client/schemas/hwm_history.md b/mddocs/client/schemas/hwm_history.md new file mode 100644 index 00000000..f4d3b264 --- /dev/null +++ b/mddocs/client/schemas/hwm_history.md @@ -0,0 +1,7 @@ +# HWM history-related schemas { #client-schemas-hwm-history } + +::: horizon.commons.schemas.v1.hwm_history + options: + members: + - HWMHistoryResponseV1 + - HWMHistoryPaginateQueryV1 diff --git a/mddocs/client/schemas/index.md b/mddocs/client/schemas/index.md new file mode 100644 index 00000000..54c5eb1b --- /dev/null +++ b/mddocs/client/schemas/index.md @@ -0,0 +1,16 @@ +# Schemas { #client-schemas-root } + +These classes are used for sending requests to backend and parsing responses. + +All of then are based on Pydantic models. + +## Horizon schemas + +* [Namespace-related schemas][client-schemas-namespace] +* [Namespace history-related schemas][client-schemas-namespace-history] +* [HWM-related schemas][client-schemas-hwm] +* [HWM history-related schemas][client-schemas-hwm-history] +* [Permissions-related schemas][client-schemas-permissions] +* [User-related schemas][client-schemas-user] +* [Ping-related schemas][client-schemas-ping] +* [Pagination-related schemas][client-schemas-pagination] diff --git a/mddocs/client/schemas/namespace.md b/mddocs/client/schemas/namespace.md new file mode 100644 index 00000000..81b4b052 --- /dev/null +++ b/mddocs/client/schemas/namespace.md @@ -0,0 +1,9 @@ +# Namespace-related schemas { #client-schemas-namespace } + +::: horizon.commons.schemas.v1.namespace + options: + members: + - NamespaceResponseV1 + - NamespacePaginateQueryV1 + - NamespaceCreateRequestV1 + - NamespaceUpdateRequestV1 diff --git a/mddocs/client/schemas/namespace_history.md b/mddocs/client/schemas/namespace_history.md new file mode 100644 index 00000000..67643b6d --- /dev/null +++ b/mddocs/client/schemas/namespace_history.md @@ -0,0 +1,7 @@ +# Namespace history-related schemas { #client-schemas-namespace-history } + +::: horizon.commons.schemas.v1.namespace_history + options: + members: + - NamespaceHistoryResponseV1 + - NamespaceHistoryPaginateQueryV1 diff --git a/mddocs/client/schemas/pagination.md b/mddocs/client/schemas/pagination.md new file mode 100644 index 00000000..480e88d0 --- /dev/null +++ b/mddocs/client/schemas/pagination.md @@ -0,0 +1,8 @@ +# Pagination-related schemas { #client-schemas-pagination } + +::: horizon.commons.schemas.v1.pagination + options: + members: + - PaginateQueryV1 + - PageResponseV1 + - PageMetaResponseV1 diff --git a/mddocs/client/schemas/permissions.md b/mddocs/client/schemas/permissions.md new file mode 100644 index 00000000..dce29ad9 --- /dev/null +++ b/mddocs/client/schemas/permissions.md @@ -0,0 +1,9 @@ +# Permissions-related schemas { #client-schemas-permissions } + +::: horizon.commons.schemas.v1.permission + options: + members: + - PermissionResponseItemV1 + - PermissionsResponseV1 + - PermissionUpdateRequestItemV1 + - PermissionsUpdateRequestV1 diff --git a/mddocs/client/schemas/ping.md b/mddocs/client/schemas/ping.md new file mode 100644 index 00000000..85e9efa7 --- /dev/null +++ b/mddocs/client/schemas/ping.md @@ -0,0 +1,6 @@ +# Ping-related schemas { #client-schemas-ping } + +::: horizon.commons.schemas + options: + members: + - PingResponse diff --git a/mddocs/client/schemas/user.md b/mddocs/client/schemas/user.md new file mode 100644 index 00000000..b12c7fd2 --- /dev/null +++ b/mddocs/client/schemas/user.md @@ -0,0 +1,30 @@ +# User-related schemas { #client-schemas-user } + + + + + + + + + + diff --git a/mddocs/client/sync.md b/mddocs/client/sync.md new file mode 100644 index 00000000..012f0300 --- /dev/null +++ b/mddocs/client/sync.md @@ -0,0 +1,108 @@ +# Sync client { #client-sync } + +## Quickstart + +Here is a short example of using sync client to interact with backend. + +Create client object: + +```pycon +>>> from horizon.client.sync import HorizonClientSync +>>> from horizon.client.auth import LoginPassword +>>> client = HorizonClientSync( +... base_url="http://some.domain.com/api", +... auth=LoginPassword(login="me", password="12345"), +... ) +``` + +Check for credentials and issue access token: + +```pycon +>>> client.authorize() +``` + +Create namespace with name "my_namespace": + +```pycon +>>> from horizon.commons.schemas.v1 import NamespaceCreateRequestV1 +>>> created_namespace = client.create_namespace(NamespaceCreateRequestV1(name="my_namespace")) +>>> created_namespace +NamespaceResponseV1( + id=1, + name="my_namespace", + description="", +) +``` + +Create HWM with name "my_hwm" in this namespace: + +```pycon +>>> from horizon.commons.schemas.v1 import HWMCreateRequestV1 +>>> hwm = HWMCreateRequestV1( +... namespace_id=created_namespace.id, +... name="my_hwm", +... type="column_int", +... value=123, +... ) +>>> created_hwm = client.create_hwm(hwm) +>>> created_hwm +HWMResponseV1( + id=1, + namespace_id=1, + name="my_hwm", + description="", + type="column_int", + value=123, + entity="", + expression="", +) +``` + +Update HWM with name "my_hwm" in this namespace: + +```pycon +>>> from horizon.commons.schemas.v1 import HWMUpdateRequestV1 +>>> hwm_change = HWMUpdateRequestV1(value=234) +>>> updated_hwm = client.update_hwm(created_hwm.id, hwm_change) +>>> updated_hwm +HWMResponseV1( + id=1, + namespace_id=1, + name="my_hwm", + description="", + type="column_int", + value=234, + entity="", + expression="", +) +``` + +## Reference + +::: horizon.client.sync.HorizonClientSync + options: + members: + - authorize + - ping + - whoami + - paginate_namespaces + - get_namespace + - create_namespace + - update_namespace + - delete_namespace + - paginate_hwm + - get_hwm + - create_hwm + - update_hwm + - delete_hwm + - bulk_delete_hwm + - get_namespace_permissions + - update_namespace_permissions + - paginate_hwm_history + - paginate_namespace_history + - retry + - bulk_copy_hwm + +::: horizon.client.sync.RetryConfig + +::: horizon.client.sync.TimeoutConfig diff --git a/mddocs/contributing.md b/mddocs/contributing.md new file mode 100644 index 00000000..177f6924 --- /dev/null +++ b/mddocs/contributing.md @@ -0,0 +1,383 @@ +# Contributing Guide { #contributing } + +Welcome! There are many ways to contribute, including submitting bug +reports, improving documentation, submitting feature requests, reviewing +new submissions, or contributing code that can be incorporated into the +project. + +## Limitations + +We should keep close to these items during development: + +* Some companies still use Python 3.7. So it is required to keep compatibility if possible, at least for *client* part of package. +* Different users uses Horizon in different ways - someone store data in Postgres, someone in MySQL, some users need LDAP. Such dependencies should be optional. + +## Initial setup for local development + +### Install Git + +Please follow [instruction](https://docs.github.com/en/get-started/quickstart/set-up-git). + +### Create a fork + +If you are not a member of a development team building horizon, you should create a fork before making any changes. + +Please follow [instruction](https://docs.github.com/en/get-started/quickstart/fork-a-repo). + +### Clone the repo + +Open terminal and run these commands: + +```bash +git clone https://github.com/MobileTeleSystems/horizon -b develop + +cd horizon +``` + +### Setup environment + +Firstly, install [make](https://www.gnu.org/software/make/manual/make.html). It is used for running complex commands in local environment. + +Secondly, create virtualenv and install dependencies: + +```bash +make venv-init +``` + +If you already have venv, but need to install dependencies required for development: + +```bash +make venv-install +``` + +We are using [poetry](https://python-poetry.org/docs/managing-dependencies/) for managing dependencies and building the package. +It allows to keep development environment the same for all developers due to using lock file with fixed dependency versions. + +There are *extra* dependencies (included into package as optional): + +* `backend` +* `client-sync` +* `postgres` +* `ldap` + +And *groups* (not included into package, used locally and in CI): + +* `test` - for running tests +* `dev` - for development, like linters, formatters, mypy, pre-commit and so on +* `docs` - for building documentation + +### Enable pre-commit hooks + +[pre-commit](https://pre-commit.com/) hooks allows to validate & fix repository content before making new commit. +It allows to run linters, formatters, fix file permissions and so on. If something is wrong, changes cannot be committed. + +Firstly, install pre-commit hooks: + +```bash +pre-commit install --install-hooks +``` + +Ant then test hooks run: + +```bash +pre-commit run +``` + +## How to + +### Run development instance locally + +Start DB container: + +```bash +make db +``` + +Then start development server: + +```bash +make dev +``` + +And open [http://localhost:8000/docs](http://localhost:8000/docs) + +Settings are stored in `.env.local` file. + +### Working with migrations + +Start database: + +```bash +make db-start +``` + +Generate revision: + +```bash +make db-revision +``` + +Upgrade db to `head` migration: + +```bash +make db-upgrade +``` + +Downgrade db to `head-1` migration: + +```bash +make db-downgrade +``` + +### Run tests locally + +Start all containers with dependencies: + +```bash +make db # for backend & client tests +make ldap-start # for backend tests +make dev # for client test, run in separate terminal tab +``` + +Run tests: + +```bash +make test +``` + +You can pass additional arguments, they will be passed to pytest: + +```bash +make test PYTEST_ARGS="-m client-sync -lsx -vvvv --log-cli-level=INFO" +``` + +Stop all containers and remove created volumes: + +```bash +make cleanup ARGS="-v" +``` + +Get fixtures not used by any test: + +```bash +make check-fixtures +``` + +### Build CI image locally + +This image is build in CI for testing purposes, but you can do that locally as well: + +```bash +make test-build +``` + +### Run production instance locally + +Firstly, build production image: + +```bash +make prod-build +``` + +And then start it: + +```bash +make prod +``` + +Then open [http://localhost:8000/docs](http://localhost:8000/docs) + +Settings are stored in `.env.docker` file. + +### Build documentation + +Build documentation using Sphinx & open it: + +```bash +make docs +``` + +If documentation should be build cleanly instead of reusing existing build result: + +```bash +make docs-fresh +``` + +## Review process + +Please create a new GitHub issue for any significant changes and +enhancements that you wish to make. Provide the feature you would like +to see, why you need it, and how it will work. Discuss your ideas +transparently and get community feedback before proceeding. + +Significant Changes that you wish to contribute to the project should be +discussed first in a GitHub issue that clearly outlines the changes and +benefits of the feature. + +Small Changes can directly be crafted and submitted to the GitHub +Repository as a Pull Request. + +### Create pull request + +Commit your changes: + +```bash +git commit -m "Commit message" +git push +``` + +Then open Github interface and [create pull request](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#making-a-pull-request). +Please follow guide from PR body template. + +After pull request is created, it get a corresponding number, e.g. 123 (`pr_number`). + +### Write release notes + +`horizon` uses [towncrier](https://pypi.org/project/towncrier/) +for changelog management. + +To submit a change note about your PR, add a text file into the +[docs/changelog/next_release](./next_release) folder. It should contain an +explanation of what applying this PR will change in the way +end-users interact with the project. One sentence is usually +enough but feel free to add as many details as you feel necessary +for the users to understand what it means. + +**Use the past tense** for the text in your fragment because, +combined with others, it will be a part of the “news digest” +telling the readers **what changed** in a specific version of +the library *since the previous version*. + +reStructuredText syntax for highlighting code (inline or block), +linking parts of the docs or external sites. +If you wish to sign your change, feel free to add `-- by +:user:`github-username`` at the end (replace `github-username` +with your own!). + +Finally, name your file following the convention that Towncrier +understands: it should start with the number of an issue or a +PR followed by a dot, then add a patch type, like `feature`, +`doc`, `misc` etc., and add `.rst` as a suffix. If you +need to add more than one fragment, you may add an optional +sequence number (delimited with another period) between the type +and the suffix. + +In general the name will follow `..rst` pattern, +where the categories are: + +* `feature`: Any new feature +* `bugfix`: A bug fix +* `improvement`: An improvement +* `doc`: A change to the documentation +* `dependency`: Dependency-related changes +* `misc`: Changes internal to the repo like CI, test and build changes + +A pull request may have more than one of these components, for example +a code change may introduce a new feature that deprecates an old +feature, in which case two fragments should be added. It is not +necessary to make a separate documentation fragment for documentation +changes accompanying the relevant code changes. + +#### Examples for adding changelog entries to your Pull Requests + +```rst +Added a ``:github:user:`` role to Sphinx config -- by :github:user:`someuser` +``` + +```rst +Fixed behavior of ``backend`` -- by :github:user:`someuser` +``` + +```rst +Added support of ``timeout`` in ``LDAP`` +-- by :github:user:`someuser`, :github:user:`anotheruser` and :github:user:`otheruser` +``` + +#### How to skip change notes check? + +Just add `ci:skip-changelog` label to pull request. + +#### Release Process + +Before making a release from the `develop` branch, follow these steps: + +1. Checkout to `develop` branch and update it to the actual state + +```bash +git checkout develop +git pull -p +``` + +1. Backup `NEXT_RELEASE.rst` + +```bash +cp "docs/changelog/NEXT_RELEASE.rst" "docs/changelog/temp_NEXT_RELEASE.rst" +``` + +1. Build the Release notes with Towncrier + +```bash +VERSION=$(poetry version -s) +towncrier build "--version=${VERSION}" --yes +``` + +1. Change file with changelog to release version number + +```bash +mv docs/changelog/NEXT_RELEASE.rst "docs/changelog/${VERSION}.rst" +``` + +1. Remove content above the version number heading in the `${VERSION}.rst` file + +```bash +awk '!/^.*towncrier release notes start/' "docs/changelog/${VERSION}.rst" > temp && mv temp "docs/changelog/${VERSION}.rst" +``` + +1. Update Changelog Index + +```bash +awk -v version=${VERSION} '/DRAFT/{print;print " " version;next}1' docs/changelog/index.rst > temp && mv temp docs/changelog/index.rst +``` + +1. Restore `NEXT_RELEASE.rst` file from backup + +```bash +mv "docs/changelog/temp_NEXT_RELEASE.rst" "docs/changelog/NEXT_RELEASE.rst" +``` + +1. Commit and push changes to `develop` branch + +```bash +git add . +git commit -m "Prepare for release ${VERSION}" +git push +``` + +1. Merge `develop` branch to `master`, **WITHOUT** squashing + +```bash +git checkout master +git pull +git merge develop +git push +``` + +1. Add git tag to the latest commit in `master` branch + +```bash +git tag "$VERSION" +git push origin "$VERSION" +``` + +1. Update version in `develop` branch **after release**: + +```bash +git checkout develop + +NEXT_VERSION=$(echo "$VERSION" | awk -F. '/[0-9]+\./{$NF++;print}' OFS=.) +poetry version "$NEXT_VERSION" + +git add . +git commit -m "Bump version" +git push +``` diff --git a/mddocs/design/entities.md b/mddocs/design/entities.md new file mode 100644 index 00000000..053aeee6 --- /dev/null +++ b/mddocs/design/entities.md @@ -0,0 +1,224 @@ +# Entities { #entities } + +## HWM + +### Description + +High Water Mark (or *HWM* for short) is a record which allows tracking state of operations (usually read). For example: + +- Save max value of a column read from a table (e.g. `updated_at`), and then use it exclude already read rows on the next read. +- Save list of files handled by a process, and then use it to exclude these files on the next read. +- Same max modification time of files handled by a process, and then use it to exclude these files on the next read. + +Each HWM record is bound to a specific Namespace. HWM `name` is unique within this Namespace. + +Users may create unlimited number of HWMs within a namespace. + +### Fields + +HWM record has following fields: + +- `id: integer` - internal HWM identifier, generated using sequence, read only. +- `namespace_id: integer` - bound to a Namespace, mandatory. +- `name: string` - unique name within a Namespace, mandatory. +- `value: json` - value associated with HWM, mandatory. +- `type: string` - any string describing type of `value` field content, mandatory. +- `description: string` - human readable description. +- `entity: string | null` - name of entity (e.g. table name, folder) which value is bound to. +- `expression: string | null` - expression (e.g. column name) used to get value from `entity`. +- `changed_at: datetime` - filled up automatically on each item change, read only. +- `changed_by: User | null` - filled up automatically on each item change, read only. + +`type` and `value` can contain any value, and are not parsed or checked by backend. This allows users to create HWMs of any type in any time, +without patching backend. But it is up to user to keep consistency of these fields. + +### Limitations + +HWM cannot be moved between namespaces. Users have to explicitly create a copy of existing HWM in new namespace, +and delete old HWM. + +## Namespace + +### Description namespace + +Namespace is a container for HWM records. + +Each namespace has an owner which can alter namespace properties, see [Role Permissions][role-permissions]. + +Users may create unlimited number of namespaces. If user created a namespace, it is automatically set as the namespace owner. + +### Fields namespace + +Namespace record has following fields: + +- `id: integer` - internal namespace identifier, generated using sequence, read only. +- `name: string` - unique per Horizon instance, mandatory +- `description: string` - human readable description. +- `owned_by: User` - is set automatically while namespace is created, mandatory. +- `changed_at: datetime` - filled up automatically on each item change, read only. +- `changed_by: User | null` - filled up automatically on each item change, read only. + +## User + +### Description user + +Users are used for: + +- Authentication +- RBAC permissions model +- Keeping track of changes made on Namespace or HWM. + +User records are created automatically after successful authentication. + +### Fields user + +User record has following fields: + +- `id: integer` - internal user identifier, generated using sequence, read only +- `username: string` - unique per Horizon instance +- `is_active: boolean` - flag if user is allowed to log in. +- `is_admin: boolean` - flag for SUPERUSER role. + +### Limitations user + +For now it is not possible to remove user after creation. + +## Permission + +### Description permission + +User can be assigned a Role within a Namespace, which can allow or disallow performing specific operation within this Namespace. +User can have different roles in different namespaces. See [Role Permissions][role-permissions]. + +### Fields permission + +Permission record has following fields: + +- `user: User` +- `namespace: Namespace` +- `role: enum`. + +### Limitations permission + +User can be assigned only one Role within a Namespace. + +## NamespaceHistory + +### Description namespacehistory + +Change of each Namespace value produces a HWMHistory item, which can be used for audit purpose. +History is append-only, items cannot be changed or deleted using API. + +### Fields namespacehistory + +NamespaceHistory record has following fields (all read-only): + +- `id: integer` - internal history item identifier, generated using sequence. +- `namespace_id: integer` - bound to Namespace item. +- `name: string`. +- `description: string`. +- `owned_by: User`. +- `changed_at: datetime` - filled up automatically on each item change. +- `changed_by: User | null` - filled up automatically on each item change. +- `action: string` - change description, e.g. `Created`, `Updated`, `Deleted`. + +## HWMHistory + +### Description HWMhistory + +Change of each HWM value produces a HWMHistory item, which can be used for audit purpose. +History is append-only, items cannot be changed or deleted using API. + +### Fields HWMhistory + +HWMHistory record has following fields (all read-only): + +- `id: integer` - internal history item identifier, generated using sequence. +- `hwm_id: integer` - bound to HWM item. +- `name: string`. +- `value: any | null`. +- `type: string`. +- `description: string`. +- `entity: string | null`. +- `expression: string | null`. +- `changed_at: datetime` - filled up automatically on each item change. +- `changed_by: User | null` - filled up automatically on each item change. +- `action: string` - change description, e.g. `Created`, `Updated`, `Deleted`. + +## Entity Diagram + +```mermaid +--- +title: Entity Diagram +--- +erDiagram + direction LR + User { + number id + string username + bool is_active + bool is_admin + } + + Namespace { + number id + number namespace_id + string name + string owned_by + string description + string changed_at + string changed_by + } + + HWM { + number id + string name + string type + string value + string description + string entity + string expression + string changed_at + string changed_by + } + + NamespaceHistory { + number id + number namespace_id + string name + string owned_by + string description + string changed_at + string changed_by + string action + } + + + HWMHistory { + number id + number hwm_id + number namespace_id + string name + string type + string value + string description + string entity + string expression + string changed_at + string changed_by + string action + } + + Permission { + number user_id + number namespace_id + string role + } + + HWM ||--o{ Namespace: contains + Namespace }o--o| NamespaceHistory: contains + HWM }o--o| HWMHistory: contains + Namespace ||--o{ User: contains + Namespace }o--|| Permission: contains + Permission ||--o{ User: contains +``` diff --git a/mddocs/design/permissions.md b/mddocs/design/permissions.md new file mode 100644 index 00000000..fc3427a6 --- /dev/null +++ b/mddocs/design/permissions.md @@ -0,0 +1,40 @@ +# Role Permissions { #role-permissions } + +Horizon implements a role-based access control model to manage permissions across different entities in the service. + +## Role Model Overview + +Roles are defined within the context of namespaces, with the exception of the superadmin role. A user can be associated with one, several, or no namespaces at all. + +- **GUEST**: User without a specific namespace assignment, having limited access rights. This is a default role. +- **DEVELOPER**: Users with development-related permissions. +- **MAINTAINER**: Users with permissions similar to developers but with additional rights in certain areas. +- **OWNER**: Users with full permissions within their owned namespaces and associated HWMs. +- **SUPERADMIN**: Users with full system-wide permissions. + +## Namespace Permissions + +| Role | Create | Read | Update | Delete | Manage Users | +|------------|----------|--------|----------|----------|----------------| +| GUEST | `+` | `+` | `-` | `-` | `-` | +| DEVELOPER | `+` | `+` | `-` | `-` | `-` | +| MAINTAINER | `+` | `+` | `-` | `-` | `-` | +| OWNER | `+` | `+` | `+` | `+` | `+` | +| SUPERADMIN | `+` | `+` | `+` | `+` | `+` | + +## HWM Permissions + +| Role | Create | Read | Update | Delete | +|------------|----------|--------|----------|----------| +| GUEST | `-` | `+` | `-` | `-` | +| DEVELOPER | `+` | `+` | `+` | `-` | +| MAINTAINER | `+` | `+` | `+` | `+` | +| OWNER | `+` | `+` | `+` | `+` | +| SUPERADMIN | `+` | `+` | `+` | `+` | + +## Superadmin Role + +The `SUPERADMIN` role grants a user unrestricted access across all entities and operations within the Horizon service. +Users with the `SUPERADMIN` role can create, read, update, delete, and manage users across all `namespaces` and `HWMs` without any restrictions. + +For details on how to update `SUPERADMIN` roles via the command-line script, see the [Manage Admins][manage-admins-script]. diff --git a/mddocs/index.md b/mddocs/index.md new file mode 100644 index 00000000..4b38c4f9 --- /dev/null +++ b/mddocs/index.md @@ -0,0 +1,55 @@ +# Data.Horizon { #readme } + +[![Repo status - Active](https://www.repostatus.org/badges/latest/active.svg)](https://github.com/MobileTeleSystems/horizon) [![DockerHub - Latest release](https://img.shields.io/docker/v/mtsrus/horizon-backend?sort=semver&label=docker)](https://hub.docker.com/r/mtsrus/horizon-backend) [![PyPI - Latest Release](https://img.shields.io/pypi/v/data-horizon)](https://pypi.org/project/data-horizon/) [![PyPI - License](https://img.shields.io/pypi/l/data-horizon.svg)](https://github.com/MobileTeleSystems/horizon/blob/develop/LICENSE.txt) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/data-horizon.svg)](https://pypi.org/project/data-horizon/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/data-horizon)](https://pypi.org/project/data-horizon/) +[![Documentation - ReadTheDocs](https://readthedocs.org/projects/data-horizon/badge/?version=stable)](https://data-horizon.readthedocs.io/) [![Github Actions - latest CI build status](https://github.com/MobileTeleSystems/horizon/workflows/Tests/badge.svg)](https://github.com/MobileTeleSystems/horizon/actions) [![Test coverage - percent](https://codecov.io/gh/MobileTeleSystems/horizon/branch/develop/graph/badge.svg?token=BIRWPTWEE0)](https://codecov.io/gh/MobileTeleSystems/horizon) [![pre-commit.ci - status](https://results.pre-commit.ci/badge/github/MobileTeleSystems/horizon/develop.svg)](https://results.pre-commit.ci/latest/github/MobileTeleSystems/horizon/develop) + +![Horizon logo](_static/logo.svg) + +## What is Data.Horizon? + +Data.Horizon is an application that implements simple HWM Store. Right now it includes: + +* REST API +* Python client + +## Goals + +* Allow users to save and fetch High Water Mark (*HWM*) items. These are `name+type+value` triples with few optional fields. +* Avoid confusion between different user’s data by separating HWMs to different *namespaces*. Each HWM is bound to namespace. +* Allow users to get HWM change history, to determine who and when changed a specific HWM value and other fields. +* Provide RBAC model to ensure that interaction with `HWMs` and `Namespaces` are governed by role assigned to each user. Roles are assigned per namespace. + +## Non-goals + +* This is not a *data* storage, it is not designed to store raw table rows. It is designed to store only HWM values. +* Attaching machine-readable metadata for HWMs (like `process`, `origin`) is not supported. This should be stored somewhere else. + +## Horizon + +High-level design + +* [entities][entities] +* [permissions][permissions] + +Backend + +* [install][backend-install] +* [architecture][backend-architecture] +* [configuration][backend-configuration] +* [auth][backend-auth-providers] +* [openapi][backend-openapi] +* [scripts][scripts] + +Client + +* [install][client-install] +* [sync][client-sync] +* [auth][client-auth] +* [schemas][client-schemas-root] +* [exceptions][client-exceptions] + +Development + +* [changelog][changelog] +* [contributing][contributing] +* [security][security] diff --git a/mddocs/robots.txt b/mddocs/robots.txt new file mode 100644 index 00000000..46f730d1 --- /dev/null +++ b/mddocs/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Allow: /*/stable/ +Allow: /en/stable/ # Fallback for bots that don't understand wildcards +Disallow: / +Sitemap: https://data-horizon.readthedocs.io/sitemap.xml \ No newline at end of file diff --git a/mddocs/security.md b/mddocs/security.md new file mode 100644 index 00000000..b5880bb8 --- /dev/null +++ b/mddocs/security.md @@ -0,0 +1,26 @@ +# Security { #security } + +## Supported Python versions + +* Client: 3.7 or above +* Backend: 3.11 or above + +## Product development security recommendations + +1. Update dependencies to last stable version +2. Build SBOM for the project +3. Perform SAST (Static Application Security Testing) where possible + +## Product development security requirements + +1. No binaries in repository +2. No passwords, keys, access tokens in source code +3. No “Critical” and/or “High” vulnerabilities in contributed source code + +## Vulnerability reports + +Please, use email [mailto:onetools@mts.ru](mailto:onetools@mts.ru) for reporting security issues or anything that can cause any consequences for security. + +Please avoid any public disclosure (including registering issues) at least until it is fixed. + +Thank you in advance for understanding.