From 90f80c4280b17d5bfa292c78ec1460ff03d67894 Mon Sep 17 00:00:00 2001 From: dswbx Date: Sat, 16 Nov 2024 12:01:47 +0100 Subject: [PATCH] public commit --- .gitignore | 24 + LICENSE.md | 110 ++ README.md | 58 + app/.ncurc.json | 3 + app/__test__/App.spec.ts | 15 + app/__test__/Module.spec.ts | 38 + app/__test__/ModuleManager.spec.ts | 197 +++ app/__test__/api/api.spec.ts | 15 + app/__test__/auth/Authenticator.spec.ts | 41 + app/__test__/auth/authorize/authorize.spec.ts | 89 ++ .../auth/strategies/OAuthStrategy.spec.ts | 46 + app/__test__/core/Endpoint.spec.ts | 54 + app/__test__/core/EventManager.spec.ts | 46 + app/__test__/core/Registry.spec.ts | 56 + app/__test__/core/benchmarks/crypto.bm.ts | 31 + .../cache/CloudflareKvCache.native-spec.ts | 57 + app/__test__/core/cache/MemoryCache.spec.ts | 15 + app/__test__/core/cache/cache-test-suite.ts | 84 ++ app/__test__/core/crypto.spec.ts | 14 + app/__test__/core/helper.ts | 18 + app/__test__/core/object/SchemaObject.spec.ts | 332 +++++ app/__test__/core/object/object-query.spec.ts | 83 ++ app/__test__/core/utils.spec.ts | 111 ++ app/__test__/data/DataController.spec.ts | 235 ++++ app/__test__/data/data-query-impl.spec.ts | 92 ++ app/__test__/data/data.test.ts | 113 ++ app/__test__/data/helper.ts | 35 + app/__test__/data/mutation.relation.test.ts | 50 + app/__test__/data/mutation.simple.test.ts | 145 ++ app/__test__/data/polymorphic.test.ts | 96 ++ app/__test__/data/prototype.test.ts | 267 ++++ app/__test__/data/relations.test.ts | 368 +++++ app/__test__/data/specs/Entity.spec.ts | 60 + app/__test__/data/specs/EntityManager.spec.ts | 106 ++ app/__test__/data/specs/JoinBuilder.spec.ts | 43 + app/__test__/data/specs/Mutator.spec.ts | 302 ++++ app/__test__/data/specs/Repository.spec.ts | 222 +++ app/__test__/data/specs/SchemaManager.spec.ts | 269 ++++ app/__test__/data/specs/WithBuilder.spec.ts | 195 +++ .../data/specs/connection/Connection.spec.ts | 92 ++ .../data/specs/fields/BooleanField.spec.ts | 29 + .../data/specs/fields/DateField.spec.ts | 13 + .../data/specs/fields/EnumField.spec.ts | 44 + app/__test__/data/specs/fields/Field.spec.ts | 45 + .../data/specs/fields/FieldIndex.spec.ts | 38 + .../data/specs/fields/JsonField.spec.ts | 47 + .../data/specs/fields/JsonSchemaField.spec.ts | 9 + .../data/specs/fields/NumberField.spec.ts | 19 + .../data/specs/fields/PrimaryField.spec.ts | 37 + .../data/specs/fields/TextField.spec.ts | 15 + app/__test__/data/specs/fields/inc.ts | 162 +++ .../specs/relations/EntityRelation.spec.ts | 78 ++ app/__test__/flows/FetchTask.spec.ts | 114 ++ app/__test__/flows/SubWorkflowTask.spec.ts | 91 ++ app/__test__/flows/Task.spec.ts | 112 ++ app/__test__/flows/inc/back.ts | 24 + app/__test__/flows/inc/fanout-condition.ts | 23 + app/__test__/flows/inc/helper.tsx | 61 + app/__test__/flows/inc/parallel.ts | 15 + app/__test__/flows/inc/simple-fetch.ts | 18 + app/__test__/flows/inputs.test.ts | 175 +++ app/__test__/flows/render.tsx | 186 +++ app/__test__/flows/trigger.test.ts | 175 +++ app/__test__/flows/workflow-basic.test.ts | 449 ++++++ app/__test__/helper.ts | 53 + app/__test__/media/MediaController.spec.ts | 56 + app/__test__/media/Storage.spec.ts | 81 ++ .../media/StorageR2Adapter.native-spec.ts | 34 + .../adapters/StorageCloudinaryAdapter.spec.ts | 61 + .../adapters/StorageLocalAdapter.spec.ts | 46 + .../media/adapters/StorageS3Adapter.spec.ts | 96 ++ app/__test__/media/adapters/icon.png | Bin 0 -> 2789 bytes app/__test__/modules/AppAuth.spec.ts | 60 + app/__test__/modules/AppData.spec.ts | 13 + app/__test__/modules/AppMedia.spec.ts | 7 + app/__test__/modules/module-test-suite.ts | 43 + app/bknd.config.js | 12 + app/build-cf.ts | 42 + app/bunfig.toml | 2 + app/env.d.ts | 56 + app/index.html | 13 + app/package.json | 175 +++ app/postcss.config.js | 18 + app/src/Api.ts | 95 ++ app/src/App.ts | 142 ++ app/src/adapter/bun/bun.adapter.ts | 33 + app/src/adapter/bun/index.ts | 1 + .../cloudflare/cloudflare-workers.adapter.ts | 267 ++++ app/src/adapter/cloudflare/index.ts | 1 + app/src/adapter/index.ts | 36 + app/src/adapter/nextjs/index.ts | 1 + app/src/adapter/nextjs/nextjs.adapter.ts | 25 + app/src/adapter/remix/index.ts | 1 + app/src/adapter/remix/remix.adapter.ts | 12 + app/src/adapter/vite/index.ts | 1 + app/src/adapter/vite/vite.adapter.ts | 82 ++ app/src/auth/AppAuth.ts | 269 ++++ app/src/auth/api/AuthApi.ts | 41 + app/src/auth/api/AuthController.ts | 57 + app/src/auth/auth-schema.ts | 85 ++ app/src/auth/authenticate/Authenticator.ts | 190 +++ .../strategies/PasswordStrategy.ts | 98 ++ app/src/auth/authenticate/strategies/index.ts | 13 + .../strategies/oauth/CustomOAuthStrategy.ts | 77 ++ .../strategies/oauth/OAuthStrategy.ts | 431 ++++++ .../strategies/oauth/issuers/github.ts | 63 + .../strategies/oauth/issuers/google.ts | 29 + .../strategies/oauth/issuers/index.ts | 2 + app/src/auth/authorize/Guard.ts | 160 +++ app/src/auth/authorize/Role.ts | 45 + app/src/auth/errors.ts | 28 + app/src/auth/index.ts | 21 + app/src/auth/utils/hash.ts | 13 + app/src/cli/commands/config.ts | 12 + app/src/cli/commands/debug.ts | 20 + app/src/cli/commands/index.ts | 5 + app/src/cli/commands/run/index.ts | 1 + app/src/cli/commands/run/platform.ts | 96 ++ app/src/cli/commands/run/run.ts | 115 ++ app/src/cli/commands/schema.ts | 12 + app/src/cli/commands/user.ts | 144 ++ app/src/cli/index.ts | 22 + app/src/cli/types.d.ts | 3 + app/src/cli/utils/sys.ts | 40 + .../core/cache/adapters/CloudflareKvCache.ts | 127 ++ app/src/core/cache/adapters/MemoryCache.ts | 139 ++ app/src/core/cache/cache-interface.ts | 178 +++ app/src/core/clients/aws/AwsClient.ts | 96 ++ app/src/core/config.ts | 12 + app/src/core/env.ts | 27 + app/src/core/errors.ts | 37 + app/src/core/events/Event.ts | 21 + app/src/core/events/EventListener.ts | 22 + app/src/core/events/EventManager.ts | 151 ++ app/src/core/events/index.ts | 8 + app/src/core/index.ts | 28 + app/src/core/object/SchemaObject.ts | 199 +++ app/src/core/object/query/object-query.ts | 96 ++ app/src/core/object/query/query.ts | 209 +++ app/src/core/registry/Registry.ts | 30 + app/src/core/security/Permission.ts | 11 + app/src/core/server/ContextHelper.ts | 29 + app/src/core/server/Controller.ts | 155 +++ app/src/core/server/Endpoint.ts | 147 ++ app/src/core/server/lib/tbValidator.ts | 37 + app/src/core/server/lib/zValidator.ts | 75 + app/src/core/template/SimpleRenderer.ts | 96 ++ app/src/core/types.ts | 4 + app/src/core/utils/DebugLogger.ts | 36 + app/src/core/utils/browser.ts | 20 + app/src/core/utils/crypto.ts | 29 + app/src/core/utils/dates.ts | 14 + app/src/core/utils/index.ts | 13 + app/src/core/utils/objects.ts | 198 +++ app/src/core/utils/perf.ts | 60 + app/src/core/utils/reqres.ts | 84 ++ app/src/core/utils/sql.ts | 9 + app/src/core/utils/strings.ts | 62 + app/src/core/utils/test.ts | 18 + app/src/core/utils/typebox/from-schema.ts | 268 ++++ app/src/core/utils/typebox/index.ts | 206 +++ app/src/core/utils/types.d.ts | 8 + app/src/core/utils/uuid.ts | 4 + app/src/core/utils/xml.ts | 6 + app/src/data/AppData.ts | 122 ++ app/src/data/api/DataApi.ts | 63 + app/src/data/api/DataController.ts | 384 ++++++ app/src/data/connection/Connection.ts | 97 ++ app/src/data/connection/LibsqlConnection.ts | 100 ++ app/src/data/connection/SqliteConnection.ts | 22 + app/src/data/connection/SqliteIntrospector.ts | 164 +++ .../data/connection/SqliteLocalConnection.ts | 31 + app/src/data/data-schema.ts | 83 ++ app/src/data/entities/Entity.ts | 238 ++++ app/src/data/entities/EntityManager.ts | 266 ++++ app/src/data/entities/Mutator.ts | 270 ++++ app/src/data/entities/index.ts | 6 + app/src/data/entities/query/JoinBuilder.ts | 51 + app/src/data/entities/query/Repository.ts | 407 ++++++ app/src/data/entities/query/WhereBuilder.ts | 132 ++ app/src/data/entities/query/WithBuilder.ts | 42 + app/src/data/errors.ts | 77 ++ app/src/data/events/index.ts | 74 + app/src/data/fields/BooleanField.ts | 88 ++ app/src/data/fields/DateField.ts | 151 ++ app/src/data/fields/EnumField.ts | 153 +++ app/src/data/fields/Field.ts | 244 ++++ app/src/data/fields/JsonField.ts | 104 ++ app/src/data/fields/JsonSchemaField.ts | 132 ++ app/src/data/fields/NumberField.ts | 100 ++ app/src/data/fields/PrimaryField.ts | 46 + app/src/data/fields/TextField.ts | 120 ++ app/src/data/fields/VirtualField.ts | 32 + app/src/data/fields/index.ts | 55 + app/src/data/fields/indices/EntityIndex.ts | 46 + app/src/data/helper.ts | 48 + app/src/data/index.ts | 28 + app/src/data/permissions/index.ts | 9 + .../plugins/DeserializeJsonValuesPlugin.ts | 36 + .../data/plugins/FilterNumericKeysPlugin.ts | 31 + app/src/data/plugins/KyselyPluginRunner.ts | 23 + app/src/data/prototype/index.ts | 295 ++++ app/src/data/relations/EntityRelation.ts | 231 ++++ .../data/relations/EntityRelationAnchor.ts | 25 + app/src/data/relations/ManyToManyRelation.ts | 189 +++ app/src/data/relations/ManyToOneRelation.ts | 228 ++++ app/src/data/relations/OneToOneRelation.ts | 77 ++ app/src/data/relations/PolymorphicRelation.ts | 130 ++ app/src/data/relations/RelationAccessor.ts | 74 + app/src/data/relations/RelationField.ts | 101 ++ app/src/data/relations/RelationHelper.ts | 86 ++ app/src/data/relations/RelationMutator.ts | 121 ++ app/src/data/relations/index.ts | 50 + app/src/data/relations/relation-types.ts | 7 + app/src/data/schema/SchemaManager.ts | 349 +++++ app/src/data/server/data-query-impl.ts | 77 ++ app/src/data/server/query.ts | 112 ++ app/src/data/test-types.ts | 78 ++ app/src/flows/AppFlows.ts | 81 ++ app/src/flows/examples/simple-fetch.ts | 20 + app/src/flows/flows-schema.ts | 84 ++ app/src/flows/flows/Execution.ts | 248 ++++ app/src/flows/flows/Flow.ts | 214 +++ app/src/flows/flows/FlowTaskConnector.ts | 118 ++ .../flows/flows/executors/RuntimeExecutor.ts | 28 + app/src/flows/flows/triggers/EventTrigger.ts | 41 + app/src/flows/flows/triggers/HttpTrigger.ts | 49 + app/src/flows/flows/triggers/Trigger.ts | 35 + app/src/flows/flows/triggers/index.ts | 13 + app/src/flows/index.ts | 41 + app/src/flows/tasks/Task.tsx | 235 ++++ app/src/flows/tasks/TaskConnection.ts | 106 ++ app/src/flows/tasks/presets/FetchTask.ts | 81 ++ app/src/flows/tasks/presets/LogTask.ts | 16 + app/src/flows/tasks/presets/RenderTask.ts | 17 + app/src/flows/tasks/presets/SubFlowTask.ts | 40 + app/src/index.ts | 12 + app/src/media/AppMedia.ts | 171 +++ app/src/media/MediaField.ts | 71 + app/src/media/api/MediaApi.ts | 44 + app/src/media/api/MediaController.ts | 193 +++ app/src/media/index.ts | 54 + app/src/media/media-schema.ts | 49 + app/src/media/storage/Storage.ts | 228 ++++ .../adapters/StorageCloudinaryAdapter.ts | 256 ++++ .../StorageLocalAdapter.ts | 125 ++ .../adapters/StorageLocalAdapter/index.ts | 5 + .../storage/adapters/StorageR2Adapter.ts | 137 ++ .../storage/adapters/StorageS3Adapter.ts | 213 +++ app/src/media/storage/events/index.ts | 17 + app/src/media/storage/mime-types.ts | 1214 +++++++++++++++++ app/src/media/utils/index.ts | 21 + app/src/modules/Module.ts | 112 ++ app/src/modules/ModuleApi.ts | 149 ++ app/src/modules/ModuleManager.ts | 443 ++++++ app/src/modules/SystemApi.ts | 24 + app/src/modules/index.ts | 21 + app/src/modules/migrations.ts | 143 ++ app/src/modules/permissions/index.ts | 7 + app/src/modules/registries.ts | 7 + app/src/modules/server/AppController.ts | 110 ++ app/src/modules/server/AppServer.ts | 143 ++ app/src/modules/server/SystemController.ts | 311 +++++ app/src/modules/server/openapi.ts | 312 +++++ .../cloudflare/image-optimization-plugin.ts | 83 ++ app/src/ui/Admin.tsx | 36 + app/src/ui/assets/favicon.ico | Bin 0 -> 15086 bytes app/src/ui/client/BkndProvider.tsx | 83 ++ app/src/ui/client/ClientProvider.tsx | 82 ++ app/src/ui/client/index.ts | 4 + app/src/ui/client/schema/actions.ts | 190 +++ app/src/ui/client/schema/auth/use-auth.ts | 107 ++ .../ui/client/schema/auth/use-bknd-auth.ts | 33 + .../ui/client/schema/data/use-bknd-data.ts | 115 ++ app/src/ui/client/schema/flows/use-flows.ts | 23 + .../client/schema/system/use-bknd-system.ts | 40 + app/src/ui/client/use-theme.ts | 8 + app/src/ui/client/utils/AppQueryClient.ts | 211 +++ app/src/ui/client/utils/AppReduced.ts | 83 ++ app/src/ui/client/utils/theme-switcher.ts | 28 + app/src/ui/components/Context.tsx | 17 + app/src/ui/components/buttons/Button.tsx | 75 + app/src/ui/components/buttons/IconButton.tsx | 42 + app/src/ui/components/canvas/Canvas.tsx | 203 +++ .../canvas/components/nodes/DefaultNode.tsx | 52 + app/src/ui/components/canvas/layouts/index.ts | 55 + app/src/ui/components/canvas/panels/Panel.tsx | 78 ++ app/src/ui/components/canvas/panels/index.tsx | 65 + app/src/ui/components/code/CodeEditor.tsx | 27 + app/src/ui/components/code/JsonEditor.tsx | 22 + app/src/ui/components/code/JsonViewer.tsx | 71 + app/src/ui/components/code/LiquidJsEditor.tsx | 123 ++ app/src/ui/components/display/Empty.tsx | 33 + app/src/ui/components/display/Logo.tsx | 31 + .../form/FloatingSelect/FloatingSelect.tsx | 55 + app/src/ui/components/form/Formy.tsx | 176 +++ app/src/ui/components/form/SearchInput.tsx | 16 + .../ui/components/form/SegmentedControl.tsx | 24 + .../hook-form-mantine/MantineNumberInput.tsx | 42 + .../form/hook-form-mantine/MantineRadio.tsx | 82 ++ .../MantineSegmentedControl.tsx | 58 + .../form/hook-form-mantine/MantineSelect.tsx | 48 + .../form/hook-form-mantine/MantineSwitch.tsx | 40 + .../form/json-schema/JsonSchemaForm.tsx | 159 +++ .../form/json-schema/JsonSchemaValidator.ts | 121 ++ .../form/json-schema/fields/JsonField.tsx | 32 + .../form/json-schema/fields/LiquidJsField.tsx | 26 + .../json-schema/fields/MultiSchemaField.tsx | 307 +++++ .../form/json-schema/fields/index.ts | 10 + .../ui/components/form/json-schema/styles.css | 264 ++++ .../templates/ArrayFieldItemTemplate.tsx | 80 ++ .../templates/ArrayFieldTemplate.tsx | 100 ++ .../templates/BaseInputTemplate.tsx | 120 ++ .../json-schema/templates/ButtonTemplates.tsx | 29 + .../json-schema/templates/FieldTemplate.tsx | 95 ++ .../templates/ObjectFieldTemplate.tsx | 101 ++ .../templates/TitleFieldTemplate.tsx | 22 + .../templates/WrapIfAdditionalTemplate.tsx | 84 ++ .../form/json-schema/templates/index.ts | 19 + .../typebox/RJSFTypeboxValidator.ts | 76 ++ .../form/json-schema/typebox/from-schema.ts | 299 ++++ .../json-schema/widgets/CheckboxWidget.tsx | 48 + .../json-schema/widgets/CheckboxesWidget.tsx | 98 ++ .../form/json-schema/widgets/JsonWidget.tsx | 19 + .../form/json-schema/widgets/RadioWidget.tsx | 97 ++ .../form/json-schema/widgets/SelectWidget.tsx | 105 ++ .../form/json-schema/widgets/index.tsx | 30 + app/src/ui/components/list/SortableList.tsx | 107 ++ app/src/ui/components/menu/Dropdown.tsx | 3 + app/src/ui/components/modal/Modal.tsx | 0 app/src/ui/components/modal/Modal2.tsx | 154 +++ app/src/ui/components/overlay/Dropdown.tsx | 111 ++ app/src/ui/components/overlay/Modal.tsx | 48 + app/src/ui/components/overlay/Popover.tsx | 57 + app/src/ui/components/radix/ScrollArea.tsx | 22 + app/src/ui/components/radix/extend.tsx | 86 ++ app/src/ui/components/steps/Steps.tsx | 67 + app/src/ui/components/table/DataTable.tsx | 297 ++++ app/src/ui/components/wouter/Link.tsx | 79 ++ app/src/ui/container/EntitiesContainer.tsx | 95 ++ app/src/ui/container/EntityContainer.tsx | 136 ++ app/src/ui/container/index.ts | 2 + app/src/ui/container/use-flows.ts | 22 + app/src/ui/hooks/use-browser-title.ts | 8 + app/src/ui/hooks/use-event.ts | 18 + app/src/ui/hooks/use-search.ts | 47 + app/src/ui/index.ts | 21 + app/src/ui/layouts/AppShell/AppShell.tsx | 362 +++++ app/src/ui/layouts/AppShell/Breadcrumbs.tsx | 116 ++ app/src/ui/layouts/AppShell/Breadcrumbs2.tsx | 120 ++ app/src/ui/layouts/AppShell/Header.tsx | 203 +++ app/src/ui/layouts/AppShell/index.ts | 1 + app/src/ui/layouts/AppShell/use-appshell.tsx | 28 + app/src/ui/lib/mantine/theme.ts | 130 ++ app/src/ui/lib/routes.ts | 128 ++ app/src/ui/lib/utils.ts | 5 + app/src/ui/main.tsx | 38 + app/src/ui/modals/debug/DebugModal.tsx | 74 + app/src/ui/modals/debug/SchemaFormModal.tsx | 69 + app/src/ui/modals/debug/TestModal.tsx | 22 + app/src/ui/modals/index.tsx | 57 + app/src/ui/modules/auth/LoginForm.tsx | 117 ++ .../ui/modules/data/components/EntityForm.tsx | 329 +++++ .../modules/data/components/EntityTable.tsx | 242 ++++ .../modules/data/components/EntityTable2.tsx | 51 + .../components/canvas/DataSchemaCanvas.tsx | 114 ++ .../components/canvas/EntityTableNode.tsx | 124 ++ .../modules/data/components/fields-specs.ts | 80 ++ .../fields/EntityJsonSchemaFormField.tsx | 56 + .../fields/EntityRelationalFormField.tsx | 258 ++++ .../schema/create-modal/CreateModal.tsx | 114 ++ .../schema/create-modal/step.create.tsx | 187 +++ .../create-modal/step.entity.fields.tsx | 122 ++ .../schema/create-modal/step.entity.tsx | 95 ++ .../schema/create-modal/step.relation.tsx | 377 +++++ .../schema/create-modal/step.select.tsx | 116 ++ .../create-modal/templates/media/index.ts | 2 + .../media/template.media.component.tsx | 183 +++ .../templates/media/template.media.meta.ts | 9 + .../schema/create-modal/templates/register.ts | 15 + .../ui/modules/data/hooks/useEntityForm.tsx | 67 + .../modules/flows/components/FlowCanvas.tsx | 158 +++ .../flows/components/TriggerComponent.tsx | 71 + .../flows/components/form/TaskForm.tsx | 41 + .../flows/components/form/TemplateField.tsx | 94 ++ .../components/tasks/FetchTaskComponent.tsx | 17 + .../components/tasks/RenderTaskComponent.tsx | 19 + .../flows/components/tasks/TaskComponent.tsx | 52 + .../modules/flows/components/tasks/index.ts | 2 + .../flows/components2/form/KeyValueInput.tsx | 109 ++ .../flows/components2/hud/FlowPanel.tsx | 79 ++ .../flows/components2/nodes/BaseNode.tsx | 148 ++ .../flows/components2/nodes/Handle.tsx | 31 + .../flows/components2/nodes/SelectNode.tsx | 107 ++ .../modules/flows/components2/nodes/index.ts | 9 + .../components2/nodes/tasks/FetchTaskNode.tsx | 140 ++ .../components2/nodes/tasks/RenderNode.tsx | 13 + .../components2/nodes/tasks/TaskNode.tsx | 51 + .../nodes/triggers/TriggerNode.tsx | 172 +++ .../ui/modules/flows/hooks/use-flow/index.tsx | 209 +++ .../ui/modules/flows/hooks/use-flow/state.ts | 6 + app/src/ui/modules/flows/utils/index.ts | 89 ++ .../media/components/dropzone/Dropzone.tsx | 413 ++++++ .../components/dropzone/file-selector.ts | 262 ++++ .../media/components/dropzone/use-dropzone.ts | 79 ++ app/src/ui/modules/media/helper.ts | 31 + app/src/ui/routes/auth/_auth.root.tsx | 60 + app/src/ui/routes/auth/auth.index.tsx | 92 ++ app/src/ui/routes/auth/auth.login.tsx | 121 ++ .../ui/routes/auth/auth.roles.edit.$role.tsx | 90 ++ app/src/ui/routes/auth/auth.roles.tsx | 128 ++ app/src/ui/routes/auth/auth.settings.tsx | 79 ++ app/src/ui/routes/auth/auth.strategies.tsx | 57 + app/src/ui/routes/auth/auth.users.tsx | 3 + app/src/ui/routes/auth/forms/role.form.tsx | 152 +++ app/src/ui/routes/auth/index.tsx | 21 + app/src/ui/routes/data/_data.root.tsx | 136 ++ app/src/ui/routes/data/data.$entity.$id.tsx | 296 ++++ .../ui/routes/data/data.$entity.create.tsx | 103 ++ app/src/ui/routes/data/data.$entity.index.tsx | 145 ++ .../ui/routes/data/data.schema.$entity.tsx | 233 ++++ app/src/ui/routes/data/data.schema.index.tsx | 41 + .../routes/data/forms/entity.fields.form.tsx | 454 ++++++ app/src/ui/routes/data/index.tsx | 25 + app/src/ui/routes/flows/_flows.root.tsx | 67 + .../flows/components/FlowCreateModal.tsx | 136 ++ app/src/ui/routes/flows/flows.edit.$name.tsx | 241 ++++ app/src/ui/routes/flows/flows.list.tsx | 72 + app/src/ui/routes/flows/index.tsx | 15 + app/src/ui/routes/flows_old/_flows.root.tsx | 64 + app/src/ui/routes/flows_old/flow.$key.tsx | 202 +++ app/src/ui/routes/flows_old/index.tsx | 12 + app/src/ui/routes/index.tsx | 81 ++ app/src/ui/routes/media/_media.root.tsx | 99 ++ app/src/ui/routes/media/index.tsx | 10 + app/src/ui/routes/root.tsx | 52 + .../ui/routes/settings/components/Setting.tsx | 358 +++++ .../settings/components/SettingNewModal.tsx | 170 +++ .../components/SettingSchemaModal.tsx | 73 + app/src/ui/routes/settings/index.tsx | 186 +++ .../routes/settings/routes/auth.settings.tsx | 156 +++ .../routes/settings/routes/data.settings.tsx | 347 +++++ .../routes/settings/routes/flows.settings.tsx | 212 +++ app/src/ui/routes/settings/utils/schema.ts | 50 + app/src/ui/routes/test/index.tsx | 81 ++ .../test/tests/appshell-accordions-test.tsx | 124 ++ .../ui/routes/test/tests/dropdown-test.tsx | 17 + .../routes/test/tests/entity-fields-form.tsx | 296 ++++ .../test/tests/flow-create-schema-test.tsx | 25 + .../ui/routes/test/tests/flow-form-test.tsx | 18 + app/src/ui/routes/test/tests/flows-test.tsx | 243 ++++ .../routes/test/tests/jsonform-test/index.tsx | 65 + .../ui/routes/test/tests/liquid-js-test.tsx | 15 + app/src/ui/routes/test/tests/mantine-test.tsx | 42 + app/src/ui/routes/test/tests/modal-test.tsx | 17 + .../ui/routes/test/tests/query-jsonform.tsx | 158 +++ .../routes/test/tests/react-hook-errors.tsx | 48 + .../ui/routes/test/tests/reactflow-test.tsx | 15 + app/src/ui/routes/test/tests/schema-test.tsx | 113 ++ .../ui/routes/test/tests/sortable-test.tsx | 107 ++ app/src/ui/routes/test/tests/sql-ai-test.tsx | 37 + app/src/ui/routes/test/tests/swagger-test.tsx | 42 + app/src/ui/styles.css | 207 +++ app/tailwind.config.js | 17 + app/tsconfig.json | 39 + app/tsup.adapters.ts | 48 + app/vite.config.ts | 62 + app/vite.dev.ts | 25 + biome.json | 74 + bun.lockb | Bin 0 -> 762496 bytes docs/_assets/Favicon.svg | 12 + docs/_assets/favicon.ico | 0 docs/_assets/images/checks-passed.png | Bin 0 -> 160724 bytes docs/_assets/images/hero-dark.svg | 161 +++ docs/_assets/images/hero-light.svg | 155 +++ docs/_assets/logo/bknd_logo_black.svg | 4 + docs/_assets/logo/bknd_logo_white.svg | 4 + docs/_assets/logo/bknd_logo_white_test.svg | 4 + docs/_assets/logo/dark.svg | 55 + docs/_assets/logo/light.svg | 51 + docs/api-reference/auth/login.mdx | 4 + docs/api-reference/auth/me.mdx | 4 + docs/api-reference/auth/register.mdx | 4 + docs/api-reference/auth/strategies.mdx | 4 + docs/api-reference/data/create.mdx | 4 + docs/api-reference/data/delete.mdx | 4 + docs/api-reference/data/get.mdx | 4 + docs/api-reference/data/list.mdx | 4 + docs/api-reference/data/update.mdx | 4 + docs/api-reference/endpoint/create.mdx | 4 + docs/api-reference/endpoint/delete.mdx | 4 + docs/api-reference/endpoint/get.mdx | 4 + docs/api-reference/introduction.mdx | 23 + docs/api-reference/openapi.json | 391 ++++++ docs/api-reference/system/config.mdx | 4 + docs/api-reference/system/ping.mdx | 4 + docs/api-reference/system/schema.mdx | 4 + docs/bun.lockb | Bin 0 -> 297390 bytes docs/concepts.mdx | 23 + docs/development.mdx | 106 ++ docs/guide/introduction.mdx | 8 + docs/integration/bun.mdx | 40 + docs/integration/cloudflare.mdx | 96 ++ docs/integration/nextjs.mdx | 57 + docs/integration/remix.mdx | 58 + docs/introduction.mdx | 66 + docs/mint.json | 211 +++ docs/modules/auth.mdx | 20 + docs/modules/data.mdx | 37 + docs/modules/flows.mdx | 36 + docs/modules/media.mdx | 18 + docs/modules/overview.mdx | 49 + docs/package.json | 10 + docs/snippets/install-bknd.mdx | 14 + examples/bun/.gitignore | 175 +++ examples/bun/README.md | 15 + examples/bun/index.ts | 23 + examples/bun/package.json | 18 + examples/bun/tsconfig.json | 23 + examples/cloudflare-worker/.gitignore | 172 +++ examples/cloudflare-worker/build.ts | 24 + examples/cloudflare-worker/package.json | 20 + examples/cloudflare-worker/src/index.ts | 22 + examples/cloudflare-worker/tsconfig.json | 22 + .../worker-configuration.d.ts | 12 + examples/cloudflare-worker/wrangler.toml | 13 + examples/nextjs/.gitignore | 40 + examples/nextjs/README.md | 40 + examples/nextjs/next.config.ts | 18 + examples/nextjs/package.json | 27 + examples/nextjs/postcss.config.mjs | 8 + examples/nextjs/public/favicon.ico | Bin 0 -> 25931 bytes examples/nextjs/public/file.svg | 1 + examples/nextjs/public/globe.svg | 1 + examples/nextjs/public/next.svg | 1 + examples/nextjs/public/vercel.svg | 1 + examples/nextjs/public/window.svg | 1 + examples/nextjs/src/components/BkndAdmin.tsx | 11 + examples/nextjs/src/pages/_app.tsx | 6 + examples/nextjs/src/pages/_document.tsx | 13 + .../nextjs/src/pages/admin/[[...admin]].tsx | 14 + examples/nextjs/src/pages/api/[...route].ts | 59 + examples/nextjs/src/pages/api/hello.ts | 13 + .../nextjs/src/pages/fonts/GeistMonoVF.woff | Bin 0 -> 67864 bytes examples/nextjs/src/pages/fonts/GeistVF.woff | Bin 0 -> 66268 bytes examples/nextjs/src/pages/index.tsx | 115 ++ examples/nextjs/src/styles/globals.css | 21 + examples/nextjs/tailwind.config.ts | 19 + examples/nextjs/tsconfig.json | 22 + examples/remix/.gitignore | 5 + examples/remix/README.md | 1 + examples/remix/app/root.tsx | 37 + examples/remix/app/routes/_index.tsx | 30 + examples/remix/app/routes/admin.$.tsx | 18 + examples/remix/app/routes/api.$.ts | 13 + examples/remix/package.json | 33 + examples/remix/public/favicon.ico | 0 examples/remix/tsconfig.json | 30 + examples/remix/vite.config.ts | 24 + examples/sw/index.html | 63 + examples/sw/main.ts | 19 + examples/sw/package.json | 19 + examples/sw/sw.ts | 35 + examples/sw/tsconfig.json | 10 + package.json | 50 + packages/cli/package.json | 33 + packages/plasmic/.gitignore | 175 +++ packages/plasmic/README.md | 15 + packages/plasmic/components/Image.tsx | 243 ++++ packages/plasmic/components/LazyRender.tsx | 111 ++ packages/plasmic/components/WouterLink.tsx | 23 + packages/plasmic/components/data/BkndData.tsx | 385 ++++++ packages/plasmic/components/index.ts | 4 + packages/plasmic/contexts/BkndContext.tsx | 143 ++ packages/plasmic/contexts/index.ts | 1 + packages/plasmic/index.ts | 4 + packages/plasmic/loader.tsx | 109 ++ packages/plasmic/package.json | 48 + packages/plasmic/tsconfig.json | 22 + tmp/event_manager_returning_test.patch | 150 ++ tsconfig.json | 20 + verdaccio.yml | 14 + 582 files changed, 49291 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 app/.ncurc.json create mode 100644 app/__test__/App.spec.ts create mode 100644 app/__test__/Module.spec.ts create mode 100644 app/__test__/ModuleManager.spec.ts create mode 100644 app/__test__/api/api.spec.ts create mode 100644 app/__test__/auth/Authenticator.spec.ts create mode 100644 app/__test__/auth/authorize/authorize.spec.ts create mode 100644 app/__test__/auth/strategies/OAuthStrategy.spec.ts create mode 100644 app/__test__/core/Endpoint.spec.ts create mode 100644 app/__test__/core/EventManager.spec.ts create mode 100644 app/__test__/core/Registry.spec.ts create mode 100644 app/__test__/core/benchmarks/crypto.bm.ts create mode 100644 app/__test__/core/cache/CloudflareKvCache.native-spec.ts create mode 100644 app/__test__/core/cache/MemoryCache.spec.ts create mode 100644 app/__test__/core/cache/cache-test-suite.ts create mode 100644 app/__test__/core/crypto.spec.ts create mode 100644 app/__test__/core/helper.ts create mode 100644 app/__test__/core/object/SchemaObject.spec.ts create mode 100644 app/__test__/core/object/object-query.spec.ts create mode 100644 app/__test__/core/utils.spec.ts create mode 100644 app/__test__/data/DataController.spec.ts create mode 100644 app/__test__/data/data-query-impl.spec.ts create mode 100644 app/__test__/data/data.test.ts create mode 100644 app/__test__/data/helper.ts create mode 100644 app/__test__/data/mutation.relation.test.ts create mode 100644 app/__test__/data/mutation.simple.test.ts create mode 100644 app/__test__/data/polymorphic.test.ts create mode 100644 app/__test__/data/prototype.test.ts create mode 100644 app/__test__/data/relations.test.ts create mode 100644 app/__test__/data/specs/Entity.spec.ts create mode 100644 app/__test__/data/specs/EntityManager.spec.ts create mode 100644 app/__test__/data/specs/JoinBuilder.spec.ts create mode 100644 app/__test__/data/specs/Mutator.spec.ts create mode 100644 app/__test__/data/specs/Repository.spec.ts create mode 100644 app/__test__/data/specs/SchemaManager.spec.ts create mode 100644 app/__test__/data/specs/WithBuilder.spec.ts create mode 100644 app/__test__/data/specs/connection/Connection.spec.ts create mode 100644 app/__test__/data/specs/fields/BooleanField.spec.ts create mode 100644 app/__test__/data/specs/fields/DateField.spec.ts create mode 100644 app/__test__/data/specs/fields/EnumField.spec.ts create mode 100644 app/__test__/data/specs/fields/Field.spec.ts create mode 100644 app/__test__/data/specs/fields/FieldIndex.spec.ts create mode 100644 app/__test__/data/specs/fields/JsonField.spec.ts create mode 100644 app/__test__/data/specs/fields/JsonSchemaField.spec.ts create mode 100644 app/__test__/data/specs/fields/NumberField.spec.ts create mode 100644 app/__test__/data/specs/fields/PrimaryField.spec.ts create mode 100644 app/__test__/data/specs/fields/TextField.spec.ts create mode 100644 app/__test__/data/specs/fields/inc.ts create mode 100644 app/__test__/data/specs/relations/EntityRelation.spec.ts create mode 100644 app/__test__/flows/FetchTask.spec.ts create mode 100644 app/__test__/flows/SubWorkflowTask.spec.ts create mode 100644 app/__test__/flows/Task.spec.ts create mode 100644 app/__test__/flows/inc/back.ts create mode 100644 app/__test__/flows/inc/fanout-condition.ts create mode 100644 app/__test__/flows/inc/helper.tsx create mode 100644 app/__test__/flows/inc/parallel.ts create mode 100644 app/__test__/flows/inc/simple-fetch.ts create mode 100644 app/__test__/flows/inputs.test.ts create mode 100644 app/__test__/flows/render.tsx create mode 100644 app/__test__/flows/trigger.test.ts create mode 100644 app/__test__/flows/workflow-basic.test.ts create mode 100644 app/__test__/helper.ts create mode 100644 app/__test__/media/MediaController.spec.ts create mode 100644 app/__test__/media/Storage.spec.ts create mode 100644 app/__test__/media/StorageR2Adapter.native-spec.ts create mode 100644 app/__test__/media/adapters/StorageCloudinaryAdapter.spec.ts create mode 100644 app/__test__/media/adapters/StorageLocalAdapter.spec.ts create mode 100644 app/__test__/media/adapters/StorageS3Adapter.spec.ts create mode 100644 app/__test__/media/adapters/icon.png create mode 100644 app/__test__/modules/AppAuth.spec.ts create mode 100644 app/__test__/modules/AppData.spec.ts create mode 100644 app/__test__/modules/AppMedia.spec.ts create mode 100644 app/__test__/modules/module-test-suite.ts create mode 100644 app/bknd.config.js create mode 100644 app/build-cf.ts create mode 100644 app/bunfig.toml create mode 100644 app/env.d.ts create mode 100644 app/index.html create mode 100644 app/package.json create mode 100644 app/postcss.config.js create mode 100644 app/src/Api.ts create mode 100644 app/src/App.ts create mode 100644 app/src/adapter/bun/bun.adapter.ts create mode 100644 app/src/adapter/bun/index.ts create mode 100644 app/src/adapter/cloudflare/cloudflare-workers.adapter.ts create mode 100644 app/src/adapter/cloudflare/index.ts create mode 100644 app/src/adapter/index.ts create mode 100644 app/src/adapter/nextjs/index.ts create mode 100644 app/src/adapter/nextjs/nextjs.adapter.ts create mode 100644 app/src/adapter/remix/index.ts create mode 100644 app/src/adapter/remix/remix.adapter.ts create mode 100644 app/src/adapter/vite/index.ts create mode 100644 app/src/adapter/vite/vite.adapter.ts create mode 100644 app/src/auth/AppAuth.ts create mode 100644 app/src/auth/api/AuthApi.ts create mode 100644 app/src/auth/api/AuthController.ts create mode 100644 app/src/auth/auth-schema.ts create mode 100644 app/src/auth/authenticate/Authenticator.ts create mode 100644 app/src/auth/authenticate/strategies/PasswordStrategy.ts create mode 100644 app/src/auth/authenticate/strategies/index.ts create mode 100644 app/src/auth/authenticate/strategies/oauth/CustomOAuthStrategy.ts create mode 100644 app/src/auth/authenticate/strategies/oauth/OAuthStrategy.ts create mode 100644 app/src/auth/authenticate/strategies/oauth/issuers/github.ts create mode 100644 app/src/auth/authenticate/strategies/oauth/issuers/google.ts create mode 100644 app/src/auth/authenticate/strategies/oauth/issuers/index.ts create mode 100644 app/src/auth/authorize/Guard.ts create mode 100644 app/src/auth/authorize/Role.ts create mode 100644 app/src/auth/errors.ts create mode 100644 app/src/auth/index.ts create mode 100644 app/src/auth/utils/hash.ts create mode 100644 app/src/cli/commands/config.ts create mode 100644 app/src/cli/commands/debug.ts create mode 100644 app/src/cli/commands/index.ts create mode 100644 app/src/cli/commands/run/index.ts create mode 100644 app/src/cli/commands/run/platform.ts create mode 100644 app/src/cli/commands/run/run.ts create mode 100644 app/src/cli/commands/schema.ts create mode 100644 app/src/cli/commands/user.ts create mode 100644 app/src/cli/index.ts create mode 100644 app/src/cli/types.d.ts create mode 100644 app/src/cli/utils/sys.ts create mode 100644 app/src/core/cache/adapters/CloudflareKvCache.ts create mode 100644 app/src/core/cache/adapters/MemoryCache.ts create mode 100644 app/src/core/cache/cache-interface.ts create mode 100644 app/src/core/clients/aws/AwsClient.ts create mode 100644 app/src/core/config.ts create mode 100644 app/src/core/env.ts create mode 100644 app/src/core/errors.ts create mode 100644 app/src/core/events/Event.ts create mode 100644 app/src/core/events/EventListener.ts create mode 100644 app/src/core/events/EventManager.ts create mode 100644 app/src/core/events/index.ts create mode 100644 app/src/core/index.ts create mode 100644 app/src/core/object/SchemaObject.ts create mode 100644 app/src/core/object/query/object-query.ts create mode 100644 app/src/core/object/query/query.ts create mode 100644 app/src/core/registry/Registry.ts create mode 100644 app/src/core/security/Permission.ts create mode 100644 app/src/core/server/ContextHelper.ts create mode 100644 app/src/core/server/Controller.ts create mode 100644 app/src/core/server/Endpoint.ts create mode 100644 app/src/core/server/lib/tbValidator.ts create mode 100644 app/src/core/server/lib/zValidator.ts create mode 100644 app/src/core/template/SimpleRenderer.ts create mode 100644 app/src/core/types.ts create mode 100644 app/src/core/utils/DebugLogger.ts create mode 100644 app/src/core/utils/browser.ts create mode 100644 app/src/core/utils/crypto.ts create mode 100644 app/src/core/utils/dates.ts create mode 100644 app/src/core/utils/index.ts create mode 100644 app/src/core/utils/objects.ts create mode 100644 app/src/core/utils/perf.ts create mode 100644 app/src/core/utils/reqres.ts create mode 100644 app/src/core/utils/sql.ts create mode 100644 app/src/core/utils/strings.ts create mode 100644 app/src/core/utils/test.ts create mode 100644 app/src/core/utils/typebox/from-schema.ts create mode 100644 app/src/core/utils/typebox/index.ts create mode 100644 app/src/core/utils/types.d.ts create mode 100644 app/src/core/utils/uuid.ts create mode 100644 app/src/core/utils/xml.ts create mode 100644 app/src/data/AppData.ts create mode 100644 app/src/data/api/DataApi.ts create mode 100644 app/src/data/api/DataController.ts create mode 100644 app/src/data/connection/Connection.ts create mode 100644 app/src/data/connection/LibsqlConnection.ts create mode 100644 app/src/data/connection/SqliteConnection.ts create mode 100644 app/src/data/connection/SqliteIntrospector.ts create mode 100644 app/src/data/connection/SqliteLocalConnection.ts create mode 100644 app/src/data/data-schema.ts create mode 100644 app/src/data/entities/Entity.ts create mode 100644 app/src/data/entities/EntityManager.ts create mode 100644 app/src/data/entities/Mutator.ts create mode 100644 app/src/data/entities/index.ts create mode 100644 app/src/data/entities/query/JoinBuilder.ts create mode 100644 app/src/data/entities/query/Repository.ts create mode 100644 app/src/data/entities/query/WhereBuilder.ts create mode 100644 app/src/data/entities/query/WithBuilder.ts create mode 100644 app/src/data/errors.ts create mode 100644 app/src/data/events/index.ts create mode 100644 app/src/data/fields/BooleanField.ts create mode 100644 app/src/data/fields/DateField.ts create mode 100644 app/src/data/fields/EnumField.ts create mode 100644 app/src/data/fields/Field.ts create mode 100644 app/src/data/fields/JsonField.ts create mode 100644 app/src/data/fields/JsonSchemaField.ts create mode 100644 app/src/data/fields/NumberField.ts create mode 100644 app/src/data/fields/PrimaryField.ts create mode 100644 app/src/data/fields/TextField.ts create mode 100644 app/src/data/fields/VirtualField.ts create mode 100644 app/src/data/fields/index.ts create mode 100644 app/src/data/fields/indices/EntityIndex.ts create mode 100644 app/src/data/helper.ts create mode 100644 app/src/data/index.ts create mode 100644 app/src/data/permissions/index.ts create mode 100644 app/src/data/plugins/DeserializeJsonValuesPlugin.ts create mode 100644 app/src/data/plugins/FilterNumericKeysPlugin.ts create mode 100644 app/src/data/plugins/KyselyPluginRunner.ts create mode 100644 app/src/data/prototype/index.ts create mode 100644 app/src/data/relations/EntityRelation.ts create mode 100644 app/src/data/relations/EntityRelationAnchor.ts create mode 100644 app/src/data/relations/ManyToManyRelation.ts create mode 100644 app/src/data/relations/ManyToOneRelation.ts create mode 100644 app/src/data/relations/OneToOneRelation.ts create mode 100644 app/src/data/relations/PolymorphicRelation.ts create mode 100644 app/src/data/relations/RelationAccessor.ts create mode 100644 app/src/data/relations/RelationField.ts create mode 100644 app/src/data/relations/RelationHelper.ts create mode 100644 app/src/data/relations/RelationMutator.ts create mode 100644 app/src/data/relations/index.ts create mode 100644 app/src/data/relations/relation-types.ts create mode 100644 app/src/data/schema/SchemaManager.ts create mode 100644 app/src/data/server/data-query-impl.ts create mode 100644 app/src/data/server/query.ts create mode 100644 app/src/data/test-types.ts create mode 100644 app/src/flows/AppFlows.ts create mode 100644 app/src/flows/examples/simple-fetch.ts create mode 100644 app/src/flows/flows-schema.ts create mode 100644 app/src/flows/flows/Execution.ts create mode 100644 app/src/flows/flows/Flow.ts create mode 100644 app/src/flows/flows/FlowTaskConnector.ts create mode 100644 app/src/flows/flows/executors/RuntimeExecutor.ts create mode 100644 app/src/flows/flows/triggers/EventTrigger.ts create mode 100644 app/src/flows/flows/triggers/HttpTrigger.ts create mode 100644 app/src/flows/flows/triggers/Trigger.ts create mode 100644 app/src/flows/flows/triggers/index.ts create mode 100644 app/src/flows/index.ts create mode 100644 app/src/flows/tasks/Task.tsx create mode 100644 app/src/flows/tasks/TaskConnection.ts create mode 100644 app/src/flows/tasks/presets/FetchTask.ts create mode 100644 app/src/flows/tasks/presets/LogTask.ts create mode 100644 app/src/flows/tasks/presets/RenderTask.ts create mode 100644 app/src/flows/tasks/presets/SubFlowTask.ts create mode 100644 app/src/index.ts create mode 100644 app/src/media/AppMedia.ts create mode 100644 app/src/media/MediaField.ts create mode 100644 app/src/media/api/MediaApi.ts create mode 100644 app/src/media/api/MediaController.ts create mode 100644 app/src/media/index.ts create mode 100644 app/src/media/media-schema.ts create mode 100644 app/src/media/storage/Storage.ts create mode 100644 app/src/media/storage/adapters/StorageCloudinaryAdapter.ts create mode 100644 app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts create mode 100644 app/src/media/storage/adapters/StorageLocalAdapter/index.ts create mode 100644 app/src/media/storage/adapters/StorageR2Adapter.ts create mode 100644 app/src/media/storage/adapters/StorageS3Adapter.ts create mode 100644 app/src/media/storage/events/index.ts create mode 100644 app/src/media/storage/mime-types.ts create mode 100644 app/src/media/utils/index.ts create mode 100644 app/src/modules/Module.ts create mode 100644 app/src/modules/ModuleApi.ts create mode 100644 app/src/modules/ModuleManager.ts create mode 100644 app/src/modules/SystemApi.ts create mode 100644 app/src/modules/index.ts create mode 100644 app/src/modules/migrations.ts create mode 100644 app/src/modules/permissions/index.ts create mode 100644 app/src/modules/registries.ts create mode 100644 app/src/modules/server/AppController.ts create mode 100644 app/src/modules/server/AppServer.ts create mode 100644 app/src/modules/server/SystemController.ts create mode 100644 app/src/modules/server/openapi.ts create mode 100644 app/src/plugins/cloudflare/image-optimization-plugin.ts create mode 100644 app/src/ui/Admin.tsx create mode 100644 app/src/ui/assets/favicon.ico create mode 100644 app/src/ui/client/BkndProvider.tsx create mode 100644 app/src/ui/client/ClientProvider.tsx create mode 100644 app/src/ui/client/index.ts create mode 100644 app/src/ui/client/schema/actions.ts create mode 100644 app/src/ui/client/schema/auth/use-auth.ts create mode 100644 app/src/ui/client/schema/auth/use-bknd-auth.ts create mode 100644 app/src/ui/client/schema/data/use-bknd-data.ts create mode 100644 app/src/ui/client/schema/flows/use-flows.ts create mode 100644 app/src/ui/client/schema/system/use-bknd-system.ts create mode 100644 app/src/ui/client/use-theme.ts create mode 100644 app/src/ui/client/utils/AppQueryClient.ts create mode 100644 app/src/ui/client/utils/AppReduced.ts create mode 100644 app/src/ui/client/utils/theme-switcher.ts create mode 100644 app/src/ui/components/Context.tsx create mode 100644 app/src/ui/components/buttons/Button.tsx create mode 100644 app/src/ui/components/buttons/IconButton.tsx create mode 100644 app/src/ui/components/canvas/Canvas.tsx create mode 100644 app/src/ui/components/canvas/components/nodes/DefaultNode.tsx create mode 100644 app/src/ui/components/canvas/layouts/index.ts create mode 100644 app/src/ui/components/canvas/panels/Panel.tsx create mode 100644 app/src/ui/components/canvas/panels/index.tsx create mode 100644 app/src/ui/components/code/CodeEditor.tsx create mode 100644 app/src/ui/components/code/JsonEditor.tsx create mode 100644 app/src/ui/components/code/JsonViewer.tsx create mode 100644 app/src/ui/components/code/LiquidJsEditor.tsx create mode 100644 app/src/ui/components/display/Empty.tsx create mode 100644 app/src/ui/components/display/Logo.tsx create mode 100644 app/src/ui/components/form/FloatingSelect/FloatingSelect.tsx create mode 100644 app/src/ui/components/form/Formy.tsx create mode 100644 app/src/ui/components/form/SearchInput.tsx create mode 100644 app/src/ui/components/form/SegmentedControl.tsx create mode 100644 app/src/ui/components/form/hook-form-mantine/MantineNumberInput.tsx create mode 100644 app/src/ui/components/form/hook-form-mantine/MantineRadio.tsx create mode 100644 app/src/ui/components/form/hook-form-mantine/MantineSegmentedControl.tsx create mode 100644 app/src/ui/components/form/hook-form-mantine/MantineSelect.tsx create mode 100644 app/src/ui/components/form/hook-form-mantine/MantineSwitch.tsx create mode 100644 app/src/ui/components/form/json-schema/JsonSchemaForm.tsx create mode 100644 app/src/ui/components/form/json-schema/JsonSchemaValidator.ts create mode 100644 app/src/ui/components/form/json-schema/fields/JsonField.tsx create mode 100644 app/src/ui/components/form/json-schema/fields/LiquidJsField.tsx create mode 100644 app/src/ui/components/form/json-schema/fields/MultiSchemaField.tsx create mode 100644 app/src/ui/components/form/json-schema/fields/index.ts create mode 100644 app/src/ui/components/form/json-schema/styles.css create mode 100644 app/src/ui/components/form/json-schema/templates/ArrayFieldItemTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/ArrayFieldTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/BaseInputTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/ButtonTemplates.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/FieldTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/ObjectFieldTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/TitleFieldTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/WrapIfAdditionalTemplate.tsx create mode 100644 app/src/ui/components/form/json-schema/templates/index.ts create mode 100644 app/src/ui/components/form/json-schema/typebox/RJSFTypeboxValidator.ts create mode 100644 app/src/ui/components/form/json-schema/typebox/from-schema.ts create mode 100644 app/src/ui/components/form/json-schema/widgets/CheckboxWidget.tsx create mode 100644 app/src/ui/components/form/json-schema/widgets/CheckboxesWidget.tsx create mode 100644 app/src/ui/components/form/json-schema/widgets/JsonWidget.tsx create mode 100644 app/src/ui/components/form/json-schema/widgets/RadioWidget.tsx create mode 100644 app/src/ui/components/form/json-schema/widgets/SelectWidget.tsx create mode 100644 app/src/ui/components/form/json-schema/widgets/index.tsx create mode 100644 app/src/ui/components/list/SortableList.tsx create mode 100644 app/src/ui/components/menu/Dropdown.tsx create mode 100644 app/src/ui/components/modal/Modal.tsx create mode 100644 app/src/ui/components/modal/Modal2.tsx create mode 100644 app/src/ui/components/overlay/Dropdown.tsx create mode 100644 app/src/ui/components/overlay/Modal.tsx create mode 100644 app/src/ui/components/overlay/Popover.tsx create mode 100644 app/src/ui/components/radix/ScrollArea.tsx create mode 100644 app/src/ui/components/radix/extend.tsx create mode 100644 app/src/ui/components/steps/Steps.tsx create mode 100644 app/src/ui/components/table/DataTable.tsx create mode 100644 app/src/ui/components/wouter/Link.tsx create mode 100644 app/src/ui/container/EntitiesContainer.tsx create mode 100644 app/src/ui/container/EntityContainer.tsx create mode 100644 app/src/ui/container/index.ts create mode 100644 app/src/ui/container/use-flows.ts create mode 100644 app/src/ui/hooks/use-browser-title.ts create mode 100644 app/src/ui/hooks/use-event.ts create mode 100644 app/src/ui/hooks/use-search.ts create mode 100644 app/src/ui/index.ts create mode 100644 app/src/ui/layouts/AppShell/AppShell.tsx create mode 100644 app/src/ui/layouts/AppShell/Breadcrumbs.tsx create mode 100644 app/src/ui/layouts/AppShell/Breadcrumbs2.tsx create mode 100644 app/src/ui/layouts/AppShell/Header.tsx create mode 100644 app/src/ui/layouts/AppShell/index.ts create mode 100644 app/src/ui/layouts/AppShell/use-appshell.tsx create mode 100644 app/src/ui/lib/mantine/theme.ts create mode 100644 app/src/ui/lib/routes.ts create mode 100644 app/src/ui/lib/utils.ts create mode 100644 app/src/ui/main.tsx create mode 100644 app/src/ui/modals/debug/DebugModal.tsx create mode 100644 app/src/ui/modals/debug/SchemaFormModal.tsx create mode 100644 app/src/ui/modals/debug/TestModal.tsx create mode 100644 app/src/ui/modals/index.tsx create mode 100644 app/src/ui/modules/auth/LoginForm.tsx create mode 100644 app/src/ui/modules/data/components/EntityForm.tsx create mode 100644 app/src/ui/modules/data/components/EntityTable.tsx create mode 100644 app/src/ui/modules/data/components/EntityTable2.tsx create mode 100644 app/src/ui/modules/data/components/canvas/DataSchemaCanvas.tsx create mode 100644 app/src/ui/modules/data/components/canvas/EntityTableNode.tsx create mode 100644 app/src/ui/modules/data/components/fields-specs.ts create mode 100644 app/src/ui/modules/data/components/fields/EntityJsonSchemaFormField.tsx create mode 100644 app/src/ui/modules/data/components/fields/EntityRelationalFormField.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/CreateModal.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/step.create.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/step.entity.fields.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/step.entity.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/step.relation.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/step.select.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/templates/media/index.ts create mode 100644 app/src/ui/modules/data/components/schema/create-modal/templates/media/template.media.component.tsx create mode 100644 app/src/ui/modules/data/components/schema/create-modal/templates/media/template.media.meta.ts create mode 100644 app/src/ui/modules/data/components/schema/create-modal/templates/register.ts create mode 100644 app/src/ui/modules/data/hooks/useEntityForm.tsx create mode 100644 app/src/ui/modules/flows/components/FlowCanvas.tsx create mode 100644 app/src/ui/modules/flows/components/TriggerComponent.tsx create mode 100644 app/src/ui/modules/flows/components/form/TaskForm.tsx create mode 100644 app/src/ui/modules/flows/components/form/TemplateField.tsx create mode 100644 app/src/ui/modules/flows/components/tasks/FetchTaskComponent.tsx create mode 100644 app/src/ui/modules/flows/components/tasks/RenderTaskComponent.tsx create mode 100644 app/src/ui/modules/flows/components/tasks/TaskComponent.tsx create mode 100644 app/src/ui/modules/flows/components/tasks/index.ts create mode 100644 app/src/ui/modules/flows/components2/form/KeyValueInput.tsx create mode 100644 app/src/ui/modules/flows/components2/hud/FlowPanel.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/BaseNode.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/Handle.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/SelectNode.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/index.ts create mode 100644 app/src/ui/modules/flows/components2/nodes/tasks/FetchTaskNode.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/tasks/RenderNode.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/tasks/TaskNode.tsx create mode 100644 app/src/ui/modules/flows/components2/nodes/triggers/TriggerNode.tsx create mode 100644 app/src/ui/modules/flows/hooks/use-flow/index.tsx create mode 100644 app/src/ui/modules/flows/hooks/use-flow/state.ts create mode 100644 app/src/ui/modules/flows/utils/index.ts create mode 100644 app/src/ui/modules/media/components/dropzone/Dropzone.tsx create mode 100644 app/src/ui/modules/media/components/dropzone/file-selector.ts create mode 100644 app/src/ui/modules/media/components/dropzone/use-dropzone.ts create mode 100644 app/src/ui/modules/media/helper.ts create mode 100644 app/src/ui/routes/auth/_auth.root.tsx create mode 100644 app/src/ui/routes/auth/auth.index.tsx create mode 100644 app/src/ui/routes/auth/auth.login.tsx create mode 100644 app/src/ui/routes/auth/auth.roles.edit.$role.tsx create mode 100644 app/src/ui/routes/auth/auth.roles.tsx create mode 100644 app/src/ui/routes/auth/auth.settings.tsx create mode 100644 app/src/ui/routes/auth/auth.strategies.tsx create mode 100644 app/src/ui/routes/auth/auth.users.tsx create mode 100644 app/src/ui/routes/auth/forms/role.form.tsx create mode 100644 app/src/ui/routes/auth/index.tsx create mode 100644 app/src/ui/routes/data/_data.root.tsx create mode 100644 app/src/ui/routes/data/data.$entity.$id.tsx create mode 100644 app/src/ui/routes/data/data.$entity.create.tsx create mode 100644 app/src/ui/routes/data/data.$entity.index.tsx create mode 100644 app/src/ui/routes/data/data.schema.$entity.tsx create mode 100644 app/src/ui/routes/data/data.schema.index.tsx create mode 100644 app/src/ui/routes/data/forms/entity.fields.form.tsx create mode 100644 app/src/ui/routes/data/index.tsx create mode 100644 app/src/ui/routes/flows/_flows.root.tsx create mode 100644 app/src/ui/routes/flows/components/FlowCreateModal.tsx create mode 100644 app/src/ui/routes/flows/flows.edit.$name.tsx create mode 100644 app/src/ui/routes/flows/flows.list.tsx create mode 100644 app/src/ui/routes/flows/index.tsx create mode 100644 app/src/ui/routes/flows_old/_flows.root.tsx create mode 100644 app/src/ui/routes/flows_old/flow.$key.tsx create mode 100644 app/src/ui/routes/flows_old/index.tsx create mode 100644 app/src/ui/routes/index.tsx create mode 100644 app/src/ui/routes/media/_media.root.tsx create mode 100644 app/src/ui/routes/media/index.tsx create mode 100644 app/src/ui/routes/root.tsx create mode 100644 app/src/ui/routes/settings/components/Setting.tsx create mode 100644 app/src/ui/routes/settings/components/SettingNewModal.tsx create mode 100644 app/src/ui/routes/settings/components/SettingSchemaModal.tsx create mode 100644 app/src/ui/routes/settings/index.tsx create mode 100644 app/src/ui/routes/settings/routes/auth.settings.tsx create mode 100644 app/src/ui/routes/settings/routes/data.settings.tsx create mode 100644 app/src/ui/routes/settings/routes/flows.settings.tsx create mode 100644 app/src/ui/routes/settings/utils/schema.ts create mode 100644 app/src/ui/routes/test/index.tsx create mode 100644 app/src/ui/routes/test/tests/appshell-accordions-test.tsx create mode 100644 app/src/ui/routes/test/tests/dropdown-test.tsx create mode 100644 app/src/ui/routes/test/tests/entity-fields-form.tsx create mode 100644 app/src/ui/routes/test/tests/flow-create-schema-test.tsx create mode 100644 app/src/ui/routes/test/tests/flow-form-test.tsx create mode 100644 app/src/ui/routes/test/tests/flows-test.tsx create mode 100644 app/src/ui/routes/test/tests/jsonform-test/index.tsx create mode 100644 app/src/ui/routes/test/tests/liquid-js-test.tsx create mode 100644 app/src/ui/routes/test/tests/mantine-test.tsx create mode 100644 app/src/ui/routes/test/tests/modal-test.tsx create mode 100644 app/src/ui/routes/test/tests/query-jsonform.tsx create mode 100644 app/src/ui/routes/test/tests/react-hook-errors.tsx create mode 100644 app/src/ui/routes/test/tests/reactflow-test.tsx create mode 100644 app/src/ui/routes/test/tests/schema-test.tsx create mode 100644 app/src/ui/routes/test/tests/sortable-test.tsx create mode 100644 app/src/ui/routes/test/tests/sql-ai-test.tsx create mode 100644 app/src/ui/routes/test/tests/swagger-test.tsx create mode 100644 app/src/ui/styles.css create mode 100644 app/tailwind.config.js create mode 100644 app/tsconfig.json create mode 100644 app/tsup.adapters.ts create mode 100644 app/vite.config.ts create mode 100644 app/vite.dev.ts create mode 100644 biome.json create mode 100755 bun.lockb create mode 100644 docs/_assets/Favicon.svg create mode 100644 docs/_assets/favicon.ico create mode 100644 docs/_assets/images/checks-passed.png create mode 100644 docs/_assets/images/hero-dark.svg create mode 100644 docs/_assets/images/hero-light.svg create mode 100644 docs/_assets/logo/bknd_logo_black.svg create mode 100644 docs/_assets/logo/bknd_logo_white.svg create mode 100644 docs/_assets/logo/bknd_logo_white_test.svg create mode 100644 docs/_assets/logo/dark.svg create mode 100644 docs/_assets/logo/light.svg create mode 100644 docs/api-reference/auth/login.mdx create mode 100644 docs/api-reference/auth/me.mdx create mode 100644 docs/api-reference/auth/register.mdx create mode 100644 docs/api-reference/auth/strategies.mdx create mode 100644 docs/api-reference/data/create.mdx create mode 100644 docs/api-reference/data/delete.mdx create mode 100644 docs/api-reference/data/get.mdx create mode 100644 docs/api-reference/data/list.mdx create mode 100644 docs/api-reference/data/update.mdx create mode 100644 docs/api-reference/endpoint/create.mdx create mode 100644 docs/api-reference/endpoint/delete.mdx create mode 100644 docs/api-reference/endpoint/get.mdx create mode 100644 docs/api-reference/introduction.mdx create mode 100644 docs/api-reference/openapi.json create mode 100644 docs/api-reference/system/config.mdx create mode 100644 docs/api-reference/system/ping.mdx create mode 100644 docs/api-reference/system/schema.mdx create mode 100755 docs/bun.lockb create mode 100644 docs/concepts.mdx create mode 100644 docs/development.mdx create mode 100644 docs/guide/introduction.mdx create mode 100644 docs/integration/bun.mdx create mode 100644 docs/integration/cloudflare.mdx create mode 100644 docs/integration/nextjs.mdx create mode 100644 docs/integration/remix.mdx create mode 100644 docs/introduction.mdx create mode 100644 docs/mint.json create mode 100644 docs/modules/auth.mdx create mode 100644 docs/modules/data.mdx create mode 100644 docs/modules/flows.mdx create mode 100644 docs/modules/media.mdx create mode 100644 docs/modules/overview.mdx create mode 100644 docs/package.json create mode 100644 docs/snippets/install-bknd.mdx create mode 100644 examples/bun/.gitignore create mode 100644 examples/bun/README.md create mode 100644 examples/bun/index.ts create mode 100644 examples/bun/package.json create mode 100644 examples/bun/tsconfig.json create mode 100644 examples/cloudflare-worker/.gitignore create mode 100644 examples/cloudflare-worker/build.ts create mode 100644 examples/cloudflare-worker/package.json create mode 100644 examples/cloudflare-worker/src/index.ts create mode 100644 examples/cloudflare-worker/tsconfig.json create mode 100644 examples/cloudflare-worker/worker-configuration.d.ts create mode 100644 examples/cloudflare-worker/wrangler.toml create mode 100644 examples/nextjs/.gitignore create mode 100644 examples/nextjs/README.md create mode 100644 examples/nextjs/next.config.ts create mode 100644 examples/nextjs/package.json create mode 100644 examples/nextjs/postcss.config.mjs create mode 100644 examples/nextjs/public/favicon.ico create mode 100644 examples/nextjs/public/file.svg create mode 100644 examples/nextjs/public/globe.svg create mode 100644 examples/nextjs/public/next.svg create mode 100644 examples/nextjs/public/vercel.svg create mode 100644 examples/nextjs/public/window.svg create mode 100644 examples/nextjs/src/components/BkndAdmin.tsx create mode 100644 examples/nextjs/src/pages/_app.tsx create mode 100644 examples/nextjs/src/pages/_document.tsx create mode 100644 examples/nextjs/src/pages/admin/[[...admin]].tsx create mode 100644 examples/nextjs/src/pages/api/[...route].ts create mode 100644 examples/nextjs/src/pages/api/hello.ts create mode 100644 examples/nextjs/src/pages/fonts/GeistMonoVF.woff create mode 100644 examples/nextjs/src/pages/fonts/GeistVF.woff create mode 100644 examples/nextjs/src/pages/index.tsx create mode 100644 examples/nextjs/src/styles/globals.css create mode 100644 examples/nextjs/tailwind.config.ts create mode 100644 examples/nextjs/tsconfig.json create mode 100644 examples/remix/.gitignore create mode 100644 examples/remix/README.md create mode 100644 examples/remix/app/root.tsx create mode 100644 examples/remix/app/routes/_index.tsx create mode 100644 examples/remix/app/routes/admin.$.tsx create mode 100644 examples/remix/app/routes/api.$.ts create mode 100644 examples/remix/package.json create mode 100644 examples/remix/public/favicon.ico create mode 100644 examples/remix/tsconfig.json create mode 100644 examples/remix/vite.config.ts create mode 100644 examples/sw/index.html create mode 100644 examples/sw/main.ts create mode 100644 examples/sw/package.json create mode 100644 examples/sw/sw.ts create mode 100644 examples/sw/tsconfig.json create mode 100644 package.json create mode 100644 packages/cli/package.json create mode 100644 packages/plasmic/.gitignore create mode 100644 packages/plasmic/README.md create mode 100644 packages/plasmic/components/Image.tsx create mode 100644 packages/plasmic/components/LazyRender.tsx create mode 100644 packages/plasmic/components/WouterLink.tsx create mode 100644 packages/plasmic/components/data/BkndData.tsx create mode 100644 packages/plasmic/components/index.ts create mode 100644 packages/plasmic/contexts/BkndContext.tsx create mode 100644 packages/plasmic/contexts/index.ts create mode 100644 packages/plasmic/index.ts create mode 100644 packages/plasmic/loader.tsx create mode 100644 packages/plasmic/package.json create mode 100644 packages/plasmic/tsconfig.json create mode 100644 tmp/event_manager_returning_test.patch create mode 100644 tsconfig.json create mode 100644 verdaccio.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3afed12 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +.DS_Store +node_modules +/.cache +/.wrangler +/build +/public/build +packages/media/.env +*.sqlite +/data.sqld/ +/dist +**/*/dist +**/*/build +**/*/.cache +**/*/.env +**/*/.dev.vars +**/*/.wrangler +**/*/vite.config.ts.timestamp* +.history +**/*/.db/* +.npmrc +/.verdaccio +.idea +.vscode +.git_old \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..485865e --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,110 @@ +# Functional Source License, Version 1.1, MIT Future License + +## Abbreviation + +FSL-1.1-MIT + +## Notice + +Copyright 2024 Webintex GmbH + +## Terms and Conditions + +### Licensor ("We") + +The party offering the Software under these Terms and Conditions. + +### The Software + +The "Software" is each version of the software that we make available under +these Terms and Conditions, as indicated by our inclusion of these Terms and +Conditions with the Software. + +### License Grant + +Subject to your compliance with this License Grant and the Patents, +Redistribution and Trademark clauses below, we hereby grant you the right to +use, copy, modify, create derivative works, publicly perform, publicly display +and redistribute the Software for any Permitted Purpose identified below. + +### Permitted Purpose + +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use +means making the Software available to others in a commercial product or +service that: + +1. substitutes for the Software; + +2. substitutes for any other product or service we offer using the Software + that exists as of the date we make the Software available; or + +3. offers the same or substantially similar functionality as the Software. + +Permitted Purposes specifically include using the Software: + +1. for your internal use and access; + +2. for non-commercial education; + +3. for non-commercial research; and + +4. in connection with professional services that you provide to a licensee + using the Software in accordance with these Terms and Conditions. + +### Patents + +To the extent your use for a Permitted Purpose would necessarily infringe our +patents, the license grant above includes a license under our patents. If you +make a claim against any party that the Software infringes or contributes to +the infringement of any patent, then your patent license to the Software ends +immediately. + +### Redistribution + +The Terms and Conditions apply to all copies, modifications and derivatives of +the Software. + +If you redistribute any copies, modifications or derivatives of the Software, +you must include a copy of or a link to these Terms and Conditions and not +remove any copyright notices provided in or with the Software. + +### Disclaimer + +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. + +IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE +SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, +EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE. + +### Trademarks + +Except for displaying the License Details and identifying us as the origin of +the Software, you have no right under these Terms and Conditions to use our +trademarks, trade names, service marks or product names. + +## Grant of Future License + +We hereby irrevocably grant you an additional license to use the Software under +the MIT license that is effective on the second anniversary of the date we make +the Software available. On or after that date, you may use the Software under +the MIT license, in which case the following will apply: + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f108472 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# bknd +**Feature-rich backend built to run anywhere.** + +bknd simplifies backend development by providing powerful tools for data management, workflows, authentication, and media handling—all seamlessly integrated into a developer-friendly platform. + +**For documentation and examples, please visit https://docs.bknd.io.** + +> [!WARNING] +> Please keep in mind that **bknd** is still under active development +> and therefore full backward compatibility is not guaranteed before reaching v1.0.0. + + +## Why bknd? +**Developer-Centric**: Focus on building your app—bknd handles the heavy lifting. +**Scalable**: Designed to run in any JavaScript environment (cloud or edge) +databases. +**Integrated**: Everything from data to workflows, auth, and media, in one cohesive platform. + + +## ✨ Features +- **📊 Data**: Define, query, and control your data with ease. + - Define entities with fields and relationships, synced directly to your database. + - Supported field types: `primary`, `text`, `number`, `date`, `boolean`, `enum`, `json`, `jsonschema`. + - Relationship types: `one-to-one`, `many-to-one`, `many-to-many`, and `polymorphic`. + - Advanced querying with the **Repository**: filtering, sorting, pagination, and relational data handling. + - Seamlessly manage data with mutators and a robust event system. + - Extend database functionality with batching, introspection, and support for multiple SQL dialects. + +- **🔐 Auth**: Easily implement reliable authentication strategies. + - Built-in `user` entity with customizable fields. + - Supports multiple authentication strategies: + - Email/password (with hashed storage). + - OAuth/OIDC (Google, GitHub, and more). + - Secure JWT generation and session management. + +- **🖼️ Media**: Effortlessly manage and serve all your media files. + - Upload files with ease. + - Adapter-based support for S3, S3-compatible storage (e.g., R2, Tigris), and Cloudinary. + +- **🔄 Flows**: Design and run workflows with seamless automation. + - Create and run workflows with trigger-based automation: + - Manual triggers or events from data, auth, media, or server actions. + - HTTP triggers for external integrations. + - Define tasks in sequence, parallel, or loops, with conditional execution. + - Use reusable sub-workflows to organize complex processes. + - Leverage OpenAPI specifications for API-based tasks. + + +## 🚀 Quick start +To quickly spin up an instance, run: +```bash +npx bknd run +``` + +### Installation +```bash +npm install bknd +``` diff --git a/app/.ncurc.json b/app/.ncurc.json new file mode 100644 index 0000000..6e7be09 --- /dev/null +++ b/app/.ncurc.json @@ -0,0 +1,3 @@ +{ + "reject": ["react-icons", "@tabler/icons-react", "@tanstack/react-form"] +} diff --git a/app/__test__/App.spec.ts b/app/__test__/App.spec.ts new file mode 100644 index 0000000..395aec0 --- /dev/null +++ b/app/__test__/App.spec.ts @@ -0,0 +1,15 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { App } from "../src"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("App tests", async () => { + test("boots and pongs", async () => { + const app = new App(dummyConnection); + await app.build(); + + //expect(await app.data?.em.ping()).toBeTrue(); + }); +}); diff --git a/app/__test__/Module.spec.ts b/app/__test__/Module.spec.ts new file mode 100644 index 0000000..4089ab1 --- /dev/null +++ b/app/__test__/Module.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; +import { type TSchema, Type, stripMark } from "../src/core/utils"; +import { Module } from "../src/modules/Module"; + +function createModule(schema: Schema) { + class TestModule extends Module { + getSchema() { + return schema; + } + toJSON() { + return this.config; + } + useForceParse() { + return true; + } + } + + return TestModule; +} + +describe("Module", async () => { + test("basic", async () => {}); + + test("listener", async () => { + let result: any; + + const module = createModule(Type.Object({ a: Type.String() })); + const m = new module({ a: "test" }); + + await m.schema().set({ a: "test2" }); + m.setListener(async (c) => { + await new Promise((r) => setTimeout(r, 10)); + result = stripMark(c); + }); + await m.schema().set({ a: "test3" }); + expect(result).toEqual({ a: "test3" }); + }); +}); diff --git a/app/__test__/ModuleManager.spec.ts b/app/__test__/ModuleManager.spec.ts new file mode 100644 index 0000000..d58b98a --- /dev/null +++ b/app/__test__/ModuleManager.spec.ts @@ -0,0 +1,197 @@ +import { describe, expect, test } from "bun:test"; +import { mark, stripMark } from "../src/core/utils"; +import { ModuleManager } from "../src/modules/ModuleManager"; +import { CURRENT_VERSION, TABLE_NAME, migrateSchema } from "../src/modules/migrations"; +import { getDummyConnection } from "./helper"; + +describe("ModuleManager", async () => { + test("s1: no config, no build", async () => { + const { dummyConnection } = getDummyConnection(); + + const mm = new ModuleManager(dummyConnection); + + // that is because no module is built + expect(mm.toJSON()).toEqual({ version: 0 } as any); + }); + + test("s2: no config, build", async () => { + const { dummyConnection } = getDummyConnection(); + + const mm = new ModuleManager(dummyConnection); + await mm.build(); + + expect(mm.version()).toBe(CURRENT_VERSION); + expect(mm.built()).toBe(true); + }); + + test("s3: config given, table exists, version matches", async () => { + const c = getDummyConnection(); + const mm = new ModuleManager(c.dummyConnection); + await mm.build(); + const version = mm.version(); + const json = mm.configs(); + //const { version, ...json } = mm.toJSON() as any; + + const c2 = getDummyConnection(); + const db = c2.dummyConnection.kysely; + await migrateSchema(CURRENT_VERSION, { db }); + await db + .updateTable(TABLE_NAME) + .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) + .execute(); + + const mm2 = new ModuleManager(c2.dummyConnection, { initial: { version, ...json } }); + await mm2.build(); + + expect(json).toEqual(mm2.configs()); + }); + + test("s4: config given, table exists, version outdated, migrate", async () => { + const c = getDummyConnection(); + const mm = new ModuleManager(c.dummyConnection); + await mm.build(); + const version = mm.version(); + const json = mm.configs(); + //const { version, ...json } = mm.toJSON() as any; + + const c2 = getDummyConnection(); + const db = c2.dummyConnection.kysely; + console.log("here2"); + await migrateSchema(CURRENT_VERSION, { db }); + await db + .updateTable(TABLE_NAME) + .set({ json: JSON.stringify(json), version: CURRENT_VERSION - 1 }) + .execute(); + + const mm2 = new ModuleManager(c2.dummyConnection, { + initial: { version: version - 1, ...json } + }); + console.log("here3"); + await mm2.build(); + }); + + test("s5: config given, table exists, version mismatch", async () => { + const c = getDummyConnection(); + const mm = new ModuleManager(c.dummyConnection); + await mm.build(); + const version = mm.version(); + const json = mm.configs(); + //const { version, ...json } = mm.toJSON() as any; + + const c2 = getDummyConnection(); + const db = c2.dummyConnection.kysely; + await migrateSchema(CURRENT_VERSION, { db }); + await db + .updateTable(TABLE_NAME) + .set({ json: JSON.stringify(json), version: CURRENT_VERSION }) + .execute(); + + const mm2 = new ModuleManager(c2.dummyConnection, { + initial: { version: version - 1, ...json } + }); + + expect(mm2.build()).rejects.toThrow(/version.*do not match/); + }); + + test("s6: no config given, table exists, fetch", async () => { + const c = getDummyConnection(); + const mm = new ModuleManager(c.dummyConnection); + await mm.build(); + const json = mm.configs(); + //const { version, ...json } = mm.toJSON() as any; + + const c2 = getDummyConnection(); + const db = c2.dummyConnection.kysely; + await migrateSchema(CURRENT_VERSION, { db }); + + const config = { + ...json, + data: { + ...json.data, + basepath: "/api/data2" + } + }; + await db + .updateTable(TABLE_NAME) + .set({ json: JSON.stringify(config), version: CURRENT_VERSION }) + .execute(); + + // run without config given + const mm2 = new ModuleManager(c2.dummyConnection); + await mm2.build(); + + expect(mm2.configs().data.basepath).toBe("/api/data2"); + }); + + test("blank app, modify config", async () => { + const { dummyConnection } = getDummyConnection(); + + const mm = new ModuleManager(dummyConnection); + await mm.build(); + const configs = stripMark(mm.configs()); + + expect(mm.configs().server.admin.color_scheme).toBe("light"); + expect(() => mm.get("server").schema().patch("admin", { color_scheme: "violet" })).toThrow(); + await mm.get("server").schema().patch("admin", { color_scheme: "dark" }); + await mm.save(); + + expect(mm.configs().server.admin.color_scheme).toBe("dark"); + expect(stripMark(mm.configs())).toEqual({ + ...configs, + server: { + ...configs.server, + admin: { + ...configs.server.admin, + color_scheme: "dark" + } + } + }); + }); + + // @todo: check what happens here + /*test("blank app, modify deep config", async () => { + const { dummyConnection } = getDummyConnection(); + + const mm = new ModuleManager(dummyConnection); + await mm.build(); + + /!* await mm + .get("data") + .schema() + .patch("entities.test", { + fields: { + content: { + type: "text" + } + } + }); + await mm.build(); + + expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text"); + + expect( + mm.get("data").schema().patch("desc", "entities.users.config.sort_dir") + ).rejects.toThrow(); + await mm.build();*!/ + expect(mm.configs().data.entities?.users?.fields?.email.type).toBe("text"); + console.log("here", mm.configs()); + await mm + .get("data") + .schema() + .patch("entities.users", { config: { sort_dir: "desc" } }); + await mm.build(); + expect(mm.toJSON()); + + //console.log(_jsonp(mm.toJSON().data)); + /!*expect(mm.configs().data.entities!.test!.fields!.content.type).toBe("text"); + expect(mm.configs().data.entities!.users!.config!.sort_dir).toBe("desc");*!/ + });*/ + + /*test("accessing modules", async () => { + const { dummyConnection } = getDummyConnection(); + + const mm = new ModuleManager(dummyConnection); + + //mm.get("auth").mutate().set({}); + });*/ +}); diff --git a/app/__test__/api/api.spec.ts b/app/__test__/api/api.spec.ts new file mode 100644 index 0000000..9c2a61b --- /dev/null +++ b/app/__test__/api/api.spec.ts @@ -0,0 +1,15 @@ +import { describe, test } from "bun:test"; +import { DataApi } from "../../src/modules/data/api/DataApi"; + +describe("Api", async () => { + test("...", async () => { + /*const dataApi = new DataApi({ + host: "https://dev-config-soma.bknd.run" + }); + + const one = await dataApi.readOne("users", 1); + const many = await dataApi.readMany("users", { limit: 2 }); + console.log("one", one); + console.log("many", many);*/ + }); +}); diff --git a/app/__test__/auth/Authenticator.spec.ts b/app/__test__/auth/Authenticator.spec.ts new file mode 100644 index 0000000..93af826 --- /dev/null +++ b/app/__test__/auth/Authenticator.spec.ts @@ -0,0 +1,41 @@ +/*import { describe, expect, test } from "bun:test"; +import { decodeJwt, jwtVerify } from "jose"; +import { Authenticator, type User, type UserPool } from "../authenticate/Authenticator"; +import { PasswordStrategy } from "../authenticate/strategies/PasswordStrategy"; +import * as hash from "../utils/hash";*/ + +/*class MemoryUserPool implements UserPool { + constructor(private users: User[] = []) {} + + async findBy(prop: "id" | "email" | "username", value: string | number) { + return this.users.find((user) => user[prop] === value); + } + + async create(user: Pick) { + const id = this.users.length + 1; + const newUser = { ...user, id, username: user.email }; + this.users.push(newUser); + return newUser; + } +} + +describe("Authenticator", async () => { + const userpool = new MemoryUserPool([ + { id: 1, email: "d", username: "test", password: await hash.sha256("test") }, + ]); + + test("sha256 login", async () => { + const auth = new Authenticator(userpool, { + password: new PasswordStrategy({ + hashing: "sha256", + }), + }); + + const { token } = await auth.login("password", { email: "d", password: "test" }); + expect(token).toBeDefined(); + + const { iat, ...decoded } = decodeJwt(token); + expect(decoded).toEqual({ id: 1, email: "d", username: "test" }); + expect(await auth.verify(token)).toBe(true); + }); +});*/ diff --git a/app/__test__/auth/authorize/authorize.spec.ts b/app/__test__/auth/authorize/authorize.spec.ts new file mode 100644 index 0000000..b6318d6 --- /dev/null +++ b/app/__test__/auth/authorize/authorize.spec.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from "bun:test"; +import { Guard } from "../../../src/auth"; + +describe("authorize", () => { + test("basic", async () => { + const guard = Guard.create( + ["read", "write"], + { + admin: { + permissions: ["read", "write"] + } + }, + { enabled: true } + ); + const user = { + role: "admin" + }; + + guard.setUserContext(user); + + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(true); + + expect(() => guard.granted("something")).toThrow(); + }); + + test("with default", async () => { + const guard = Guard.create( + ["read", "write"], + { + admin: { + permissions: ["read", "write"] + }, + guest: { + permissions: ["read"], + is_default: true + } + }, + { enabled: true } + ); + + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(false); + + const user = { + role: "admin" + }; + + guard.setUserContext(user); + + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(true); + }); + + test("guard implicit allow", async () => { + const guard = Guard.create([], {}, { enabled: false }); + + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(true); + }); + + test("role implicit allow", async () => { + const guard = Guard.create(["read", "write"], { + admin: { + implicit_allow: true + } + }); + + guard.setUserContext({ + role: "admin" + }); + + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(true); + }); + + test("guard with guest role implicit allow", async () => { + const guard = Guard.create(["read", "write"], { + guest: { + implicit_allow: true, + is_default: true + } + }); + + expect(guard.getUserRole()?.name).toBe("guest"); + expect(guard.granted("read")).toBe(true); + expect(guard.granted("write")).toBe(true); + }); +}); diff --git a/app/__test__/auth/strategies/OAuthStrategy.spec.ts b/app/__test__/auth/strategies/OAuthStrategy.spec.ts new file mode 100644 index 0000000..a41f189 --- /dev/null +++ b/app/__test__/auth/strategies/OAuthStrategy.spec.ts @@ -0,0 +1,46 @@ +import { describe, test } from "bun:test"; +import { OAuthStrategy } from "../../../src/auth/authenticate/strategies"; + +const ALL_TESTS = !!process.env.ALL_TESTS; + +describe("OAuthStrategy", async () => { + const strategy = new OAuthStrategy({ + type: "oidc", + client: { + client_id: process.env.OAUTH_CLIENT_ID, + client_secret: process.env.OAUTH_CLIENT_SECRET + }, + name: "google" + }); + const state = "---"; + const redirect_uri = "http://localhost:3000/auth/google/callback"; + + test.skipIf(ALL_TESTS)("...", async () => { + const config = await strategy.getConfig(); + console.log("config", JSON.stringify(config, null, 2)); + + const request = await strategy.request({ + redirect_uri, + state + }); + + const server = Bun.serve({ + fetch: async (req) => { + const url = new URL(req.url); + if (url.pathname === "/auth/google/callback") { + console.log("req", req); + const user = await strategy.callback(url, { + redirect_uri, + state + }); + + console.log("---user", user); + } + return new Response("Bun!"); + } + }); + console.log("request", request); + + await new Promise((resolve) => setTimeout(resolve, 100000)); + }); +}); diff --git a/app/__test__/core/Endpoint.spec.ts b/app/__test__/core/Endpoint.spec.ts new file mode 100644 index 0000000..8dd9c10 --- /dev/null +++ b/app/__test__/core/Endpoint.spec.ts @@ -0,0 +1,54 @@ +import { describe, expect, it, test } from "bun:test"; +import { Endpoint } from "../../src/core"; +import { mockFetch2, unmockFetch } from "./helper"; + +const testC: any = { + json: (res: any) => Response.json(res) +}; +const testNext = async () => {}; + +describe("Endpoint", async () => { + it("behaves as expected", async () => { + const endpoint = new Endpoint("GET", "/test", async () => { + return { hello: "test" }; + }); + + expect(endpoint.method).toBe("GET"); + expect(endpoint.path).toBe("/test"); + + const handler = endpoint.toHandler(); + const response = await handler(testC, testNext); + + expect(response.ok).toBe(true); + expect(await response.json()).toEqual({ hello: "test" }); + }); + + it("can be $request(ed)", async () => { + const obj = { hello: "test" }; + const baseUrl = "https://local.com:123"; + const endpoint = Endpoint.get("/test", async () => obj); + + mockFetch2(async (input: RequestInfo, init: RequestInit) => { + expect(input).toBe(`${baseUrl}/test`); + return new Response(JSON.stringify(obj), { status: 200 }); + }); + const response = await endpoint.$request({}, baseUrl); + + expect(response).toEqual({ + status: 200, + ok: true, + response: obj + }); + unmockFetch(); + }); + + it("resolves helper functions", async () => { + const params = ["/test", () => ({ hello: "test" })]; + + ["get", "post", "patch", "put", "delete"].forEach((method) => { + const endpoint = Endpoint[method](...params); + expect(endpoint.method).toBe(method.toUpperCase()); + expect(endpoint.path).toBe(params[0]); + }); + }); +}); diff --git a/app/__test__/core/EventManager.spec.ts b/app/__test__/core/EventManager.spec.ts new file mode 100644 index 0000000..3327449 --- /dev/null +++ b/app/__test__/core/EventManager.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { Event, EventManager, NoParamEvent } from "../../src/core/events"; + +class SpecialEvent extends Event<{ foo: string }> { + static slug = "special-event"; + + isBar() { + return this.params.foo === "bar"; + } +} + +class InformationalEvent extends NoParamEvent { + static slug = "informational-event"; +} + +describe("EventManager", async () => { + test("test", async () => { + const emgr = new EventManager(); + emgr.registerEvents([SpecialEvent, InformationalEvent]); + + emgr.onEvent( + SpecialEvent, + async (event, name) => { + console.log("Event: ", name, event.params.foo, event.isBar()); + console.log("wait..."); + + await new Promise((resolve) => setTimeout(resolve, 100)); + console.log("done waiting"); + }, + "sync" + ); + + emgr.onEvent(InformationalEvent, async (event, name) => { + console.log("Event: ", name, event.params); + }); + + await emgr.emit(new SpecialEvent({ foo: "bar" })); + console.log("done"); + + // expect construct signatures to not cause ts errors + new SpecialEvent({ foo: "bar" }); + new InformationalEvent(); + + expect(true).toBe(true); + }); +}); diff --git a/app/__test__/core/Registry.spec.ts b/app/__test__/core/Registry.spec.ts new file mode 100644 index 0000000..1be9310 --- /dev/null +++ b/app/__test__/core/Registry.spec.ts @@ -0,0 +1,56 @@ +import { describe, test } from "bun:test"; +import type { TObject, TString } from "@sinclair/typebox"; +import { Registry } from "../../src/core/registry/Registry"; +import { type TSchema, Type } from "../../src/core/utils"; + +type Constructor = new (...args: any[]) => T; + +type ClassRef = Constructor & (new (...args: any[]) => T); + +class What { + method() { + return null; + } +} +class What2 extends What {} +class NotAllowed {} + +type Test1 = { + cls: new (...args: any[]) => What; + schema: TObject<{ type: TString }>; + enabled: boolean; +}; + +describe("Registry", () => { + test("adds an item", async () => { + const registry = new Registry().set({ + first: { + cls: What, + schema: Type.Object({ type: Type.String(), what: Type.String() }), + enabled: true + } + } satisfies Record); + + const item = registry.get("first"); + + registry.add("second", { + cls: What2, + schema: Type.Object({ type: Type.String(), what: Type.String() }), + enabled: true + }); + registry.add("third", { + // @ts-expect-error + cls: NotAllowed, + schema: Type.Object({ type: Type.String({ default: "1" }), what22: Type.String() }), + enabled: true + }); + registry.add("fourth", { + cls: What, + // @ts-expect-error + schema: Type.Object({ type: Type.Number(), what22: Type.String() }), + enabled: true + }); + + console.log("list", registry.all()); + }); +}); diff --git a/app/__test__/core/benchmarks/crypto.bm.ts b/app/__test__/core/benchmarks/crypto.bm.ts new file mode 100644 index 0000000..f4c70c0 --- /dev/null +++ b/app/__test__/core/benchmarks/crypto.bm.ts @@ -0,0 +1,31 @@ +import { baseline, bench, group, run } from "mitata"; +import * as crypt from "../../../src/core/utils/crypto"; + +// deno +// import { ... } from 'npm:mitata'; + +// d8/jsc +// import { ... } from '/src/cli.mjs'; + +const small = "hello"; +const big = "hello".repeat(1000); + +group("hashing (small)", () => { + baseline("baseline", () => JSON.parse(JSON.stringify({ small }))); + bench("sha-1", async () => await crypt.hash.sha256(small)); + bench("sha-256", async () => await crypt.hash.sha256(small)); +}); + +group("hashing (big)", () => { + baseline("baseline", () => JSON.parse(JSON.stringify({ big }))); + bench("sha-1", async () => await crypt.hash.sha256(big)); + bench("sha-256", async () => await crypt.hash.sha256(big)); +}); + +/*group({ name: 'group2', summary: false }, () => { + bench('new Array(0)', () => new Array(0)); + bench('new Array(1024)', () => new Array(1024)); +});*/ + +// @ts-ignore +await run(); diff --git a/app/__test__/core/cache/CloudflareKvCache.native-spec.ts b/app/__test__/core/cache/CloudflareKvCache.native-spec.ts new file mode 100644 index 0000000..360bc7d --- /dev/null +++ b/app/__test__/core/cache/CloudflareKvCache.native-spec.ts @@ -0,0 +1,57 @@ +import * as assert from "node:assert/strict"; +import { createWriteStream } from "node:fs"; +import { after, beforeEach, describe, test } from "node:test"; +import { Miniflare } from "miniflare"; +import { + CloudflareKVCacheItem, + CloudflareKVCachePool +} from "../../../src/core/cache/adapters/CloudflareKvCache"; +import { runTests } from "./cache-test-suite"; + +// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480 +console.log = async (message: any) => { + const tty = createWriteStream("/dev/tty"); + const msg = typeof message === "string" ? message : JSON.stringify(message, null, 2); + return tty.write(`${msg}\n`); +}; + +describe("CloudflareKv", async () => { + let mf: Miniflare; + runTests({ + createCache: async () => { + if (mf) { + await mf.dispose(); + } + + mf = new Miniflare({ + modules: true, + script: "export default { async fetch() { return new Response(null); } }", + kvNamespaces: ["TEST"] + }); + const kv = await mf.getKVNamespace("TEST"); + return new CloudflareKVCachePool(kv as any); + }, + createItem: (key, value) => new CloudflareKVCacheItem(key, value), + tester: { + test, + beforeEach, + expect: (actual?: any) => { + return { + toBe(expected: any) { + assert.equal(actual, expected); + }, + toEqual(expected: any) { + assert.deepEqual(actual, expected); + }, + toBeUndefined() { + assert.equal(actual, undefined); + } + }; + } + } + }); + + after(async () => { + await mf?.dispose(); + }); +}); diff --git a/app/__test__/core/cache/MemoryCache.spec.ts b/app/__test__/core/cache/MemoryCache.spec.ts new file mode 100644 index 0000000..051663a --- /dev/null +++ b/app/__test__/core/cache/MemoryCache.spec.ts @@ -0,0 +1,15 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { MemoryCache, MemoryCacheItem } from "../../../src/core/cache/adapters/MemoryCache"; +import { runTests } from "./cache-test-suite"; + +describe("MemoryCache", () => { + runTests({ + createCache: async () => new MemoryCache(), + createItem: (key, value) => new MemoryCacheItem(key, value), + tester: { + test, + beforeEach, + expect + } + }); +}); diff --git a/app/__test__/core/cache/cache-test-suite.ts b/app/__test__/core/cache/cache-test-suite.ts new file mode 100644 index 0000000..251dfde --- /dev/null +++ b/app/__test__/core/cache/cache-test-suite.ts @@ -0,0 +1,84 @@ +//import { beforeEach as bunBeforeEach, expect as bunExpect, test as bunTest } from "bun:test"; +import type { ICacheItem, ICachePool } from "../../../src/core/cache/cache-interface"; + +export type TestOptions = { + createCache: () => Promise; + createItem: (key: string, value: any) => ICacheItem; + tester: { + test: (name: string, fn: () => Promise) => void; + beforeEach: (fn: () => Promise) => void; + expect: (actual?: any) => { + toBe(expected: any): void; + toEqual(expected: any): void; + toBeUndefined(): void; + }; + }; +}; + +export function runTests({ createCache, createItem, tester }: TestOptions) { + let cache: ICachePool; + const { test, beforeEach, expect } = tester; + + beforeEach(async () => { + cache = await createCache(); + }); + + test("getItem returns correct item", async () => { + const item = createItem("key1", "value1"); + await cache.save(item); + const retrievedItem = await cache.get("key1"); + expect(retrievedItem.value()).toEqual(item.value()); + }); + + test("getItem returns new item when key does not exist", async () => { + const retrievedItem = await cache.get("key1"); + expect(retrievedItem.key()).toEqual("key1"); + expect(retrievedItem.value()).toBeUndefined(); + }); + + test("getItems returns correct items", async () => { + const item1 = createItem("key1", "value1"); + const item2 = createItem("key2", "value2"); + await cache.save(item1); + await cache.save(item2); + const retrievedItems = await cache.getMany(["key1", "key2"]); + expect(retrievedItems.get("key1")?.value()).toEqual(item1.value()); + expect(retrievedItems.get("key2")?.value()).toEqual(item2.value()); + }); + + test("hasItem returns true when item exists and is a hit", async () => { + const item = createItem("key1", "value1"); + await cache.save(item); + expect(await cache.has("key1")).toBe(true); + }); + + test("clear and deleteItem correctly clear the cache and delete items", async () => { + const item = createItem("key1", "value1"); + await cache.save(item); + + if (cache.supports().clear) { + await cache.clear(); + } else { + await cache.delete("key1"); + } + + expect(await cache.has("key1")).toBe(false); + }); + + test("save correctly saves items to the cache", async () => { + const item = createItem("key1", "value1"); + await cache.save(item); + expect(await cache.has("key1")).toBe(true); + }); + + test("putItem correctly puts items in the cache ", async () => { + await cache.put("key1", "value1", { ttl: 60 }); + const item = await cache.get("key1"); + expect(item.value()).toEqual("value1"); + expect(item.hit()).toBe(true); + }); + + /*test("commit returns true", async () => { + expect(await cache.commit()).toBe(true); + });*/ +} diff --git a/app/__test__/core/crypto.spec.ts b/app/__test__/core/crypto.spec.ts new file mode 100644 index 0000000..38178af --- /dev/null +++ b/app/__test__/core/crypto.spec.ts @@ -0,0 +1,14 @@ +import { describe, test } from "bun:test"; +import { checksum, hash } from "../../src/core/utils"; + +describe("crypto", async () => { + test("sha256", async () => { + console.log(await hash.sha256("test")); + }); + test("sha1", async () => { + console.log(await hash.sha1("test")); + }); + test("checksum", async () => { + console.log(checksum("hello world")); + }); +}); diff --git a/app/__test__/core/helper.ts b/app/__test__/core/helper.ts new file mode 100644 index 0000000..85de321 --- /dev/null +++ b/app/__test__/core/helper.ts @@ -0,0 +1,18 @@ +import { jest } from "bun:test"; + +let _oldFetch: typeof fetch; +export function mockFetch(responseMethods: Partial) { + _oldFetch = global.fetch; + // @ts-ignore + global.fetch = jest.fn(() => Promise.resolve(responseMethods)); +} + +export function mockFetch2(newFetch: (input: RequestInfo, init: RequestInit) => Promise) { + _oldFetch = global.fetch; + // @ts-ignore + global.fetch = jest.fn(newFetch); +} + +export function unmockFetch() { + global.fetch = _oldFetch; +} diff --git a/app/__test__/core/object/SchemaObject.spec.ts b/app/__test__/core/object/SchemaObject.spec.ts new file mode 100644 index 0000000..f0d8434 --- /dev/null +++ b/app/__test__/core/object/SchemaObject.spec.ts @@ -0,0 +1,332 @@ +import { describe, expect, test } from "bun:test"; +import { SchemaObject } from "../../../src/core"; +import { Type } from "../../../src/core/utils"; + +describe("SchemaObject", async () => { + test("basic", async () => { + const m = new SchemaObject( + Type.Object({ a: Type.String({ default: "b" }) }), + { a: "test" }, + { + forceParse: true + } + ); + + expect(m.get()).toEqual({ a: "test" }); + expect(m.default()).toEqual({ a: "b" }); + + // direct modification is not allowed + expect(() => { + m.get().a = "test2"; + }).toThrow(); + }); + + test("patch", async () => { + const m = new SchemaObject( + Type.Object({ + s: Type.Object( + { + a: Type.String({ default: "b" }), + b: Type.Object( + { + c: Type.String({ default: "d" }), + e: Type.String({ default: "f" }) + }, + { default: {} } + ) + }, + { default: {}, additionalProperties: false } + ) + }) + ); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "d", e: "f" } } }); + + await m.patch("s.a", "c"); + + // non-existing path on no additional properties + expect(() => m.patch("s.s.s", "c")).toThrow(); + // wrong type + expect(() => m.patch("s.a", 1)).toThrow(); + + // should have only the valid change applied + expect(m.get().s.b.c).toBe("d"); + expect(m.get()).toEqual({ s: { a: "c", b: { c: "d", e: "f" } } }); + + await m.patch("s.b.c", "d2"); + expect(m.get()).toEqual({ s: { a: "c", b: { c: "d2", e: "f" } } }); + }); + + test("patch array", async () => { + const m = new SchemaObject( + Type.Object({ + methods: Type.Array(Type.String(), { default: ["GET", "PATCH"] }) + }) + ); + expect(m.get()).toEqual({ methods: ["GET", "PATCH"] }); + + // array values are fully overwritten, whether accessed by index ... + m.patch("methods[0]", "POST"); + expect(m.get()).toEqual({ methods: ["POST"] }); + + // or by path! + m.patch("methods", ["GET", "DELETE"]); + expect(m.get()).toEqual({ methods: ["GET", "DELETE"] }); + }); + + test("remove", async () => { + const m = new SchemaObject( + Type.Object({ + s: Type.Object( + { + a: Type.String({ default: "b" }), + b: Type.Object( + { + c: Type.String({ default: "d" }) + }, + { default: {} } + ) + }, + { default: {} } + ) + }) + ); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "d" } } }); + + // expect no change, because the default then applies + m.remove("s.a"); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "d" } } }); + + // adding another path, and then deleting it + m.patch("s.c", "d"); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "d" }, c: "d" } } as any); + + // now it should be removed without applying again + m.remove("s.c"); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "d" } } }); + }); + + test("set", async () => { + const m = new SchemaObject( + Type.Object({ + methods: Type.Array(Type.String(), { default: ["GET", "PATCH"] }) + }) + ); + expect(m.get()).toEqual({ methods: ["GET", "PATCH"] }); + + m.set({ methods: ["GET", "POST"] }); + expect(m.get()).toEqual({ methods: ["GET", "POST"] }); + + // wrong type + expect(() => m.set({ methods: [1] as any })).toThrow(); + }); + + test("listener", async () => { + let called = false; + let result: any; + const m = new SchemaObject( + Type.Object({ + methods: Type.Array(Type.String(), { default: ["GET", "PATCH"] }) + }), + undefined, + { + onUpdate: async (config) => { + await new Promise((r) => setTimeout(r, 10)); + called = true; + result = config; + } + } + ); + + await m.set({ methods: ["GET", "POST"] }); + expect(called).toBe(true); + expect(result).toEqual({ methods: ["GET", "POST"] }); + }); + + test("throwIfRestricted", async () => { + const m = new SchemaObject(Type.Object({}), undefined, { + restrictPaths: ["a.b"] + }); + + expect(() => m.throwIfRestricted("a.b")).toThrow(); + expect(m.throwIfRestricted("a.c")).toBeUndefined(); + expect(() => m.throwIfRestricted({ a: { b: "c" } })).toThrow(); + expect(m.throwIfRestricted({ a: { c: "d" } })).toBeUndefined(); + }); + + test("restriction bypass", async () => { + const m = new SchemaObject( + Type.Object({ + s: Type.Object( + { + a: Type.String({ default: "b" }), + b: Type.Object( + { + c: Type.String({ default: "d" }) + }, + { default: {} } + ) + }, + { default: {} } + ) + }), + undefined, + { + restrictPaths: ["s.b"] + } + ); + + expect(() => m.patch("s.b.c", "e")).toThrow(); + expect(m.bypass().patch("s.b.c", "e")).toBeDefined(); + expect(() => m.patch("s.b.c", "f")).toThrow(); + expect(m.get()).toEqual({ s: { a: "b", b: { c: "e" } } }); + }); + + const dataEntitiesSchema = Type.Object( + { + entities: Type.Object( + {}, + { + additionalProperties: Type.Object({ + fields: Type.Object( + {}, + { + additionalProperties: Type.Object({ + type: Type.String(), + config: Type.Optional( + Type.Object({}, { additionalProperties: Type.String() }) + ) + }) + } + ), + config: Type.Optional(Type.Object({}, { additionalProperties: Type.String() })) + }) + } + ) + }, + { + additionalProperties: false + } + ); + test("patch safe object, overwrite", async () => { + const data = { + entities: { + some: { + fields: { + a: { type: "string", config: { some: "thing" } } + } + } + } + }; + const m = new SchemaObject(dataEntitiesSchema, data, { + forceParse: true, + overwritePaths: [/^entities\..*\.fields\..*\.config/] + }); + + m.patch("entities.some.fields.a", { type: "string", config: { another: "one" } }); + + expect(m.get()).toEqual({ + entities: { + some: { + fields: { + a: { type: "string", config: { another: "one" } } + } + } + } + }); + }); + + test("patch safe object, overwrite 2", async () => { + const data = { + entities: { + users: { + fields: { + email: { type: "string" }, + password: { type: "string" } + } + } + } + }; + const m = new SchemaObject(dataEntitiesSchema, data, { + forceParse: true, + overwritePaths: [/^entities\..*\.fields\..*\.config\.html_config$/] + }); + + m.patch("entities.test", { + fields: { + content: { + type: "text" + } + } + }); + + expect(m.get()).toEqual({ + entities: { + users: { + fields: { + email: { type: "string" }, + password: { type: "string" } + } + }, + test: { + fields: { + content: { + type: "text" + } + } + } + } + }); + }); + + test("patch safe object, overwrite 3", async () => { + const data = { + entities: { + users: { + fields: { + email: { type: "string" }, + password: { type: "string" } + } + } + } + }; + const m = new SchemaObject(dataEntitiesSchema, data, { + forceParse: true, + overwritePaths: [/^entities\..*\.fields\..*\.config\.html_config$/] + }); + + expect(m.patch("desc", "entities.users.config.sort_dir")).rejects.toThrow(); + + m.patch("entities.test", { + fields: { + content: { + type: "text" + } + } + }); + + m.patch("entities.users.config", { + sort_dir: "desc" + }); + + expect(m.get()).toEqual({ + entities: { + users: { + fields: { + email: { type: "string" }, + password: { type: "string" } + }, + config: { + sort_dir: "desc" + } + }, + test: { + fields: { + content: { + type: "text" + } + } + } + } + }); + }); +}); diff --git a/app/__test__/core/object/object-query.spec.ts b/app/__test__/core/object/object-query.spec.ts new file mode 100644 index 0000000..66e7215 --- /dev/null +++ b/app/__test__/core/object/object-query.spec.ts @@ -0,0 +1,83 @@ +import { describe, expect, test } from "bun:test"; +import { type ObjectQuery, convert, validate } from "../../../src/core/object/query/object-query"; +import { deprecated__whereRepoSchema } from "../../../src/data"; + +describe("object-query", () => { + const q: ObjectQuery = { name: "Michael" }; + const q2: ObjectQuery = { name: { $isnull: 1 } }; + const q3: ObjectQuery = { name: "Michael", age: { $gt: 18 } }; + const bag = { q, q2, q3 }; + + test("translates into legacy", async () => { + for (const [key, value] of Object.entries(bag)) { + const obj = convert(value); + try { + const parsed = deprecated__whereRepoSchema.parse(obj); + expect(parsed).toBeDefined(); + } catch (e) { + console.log("errored", { obj, value }); + console.error(key, e); + } + } + }); + + test("validates", async () => { + const converted = convert({ + name: { $eq: "ch" } + }); + validate(converted, { name: "Michael" }); + }); + + test("single validation", () => { + const tests: [ObjectQuery, any, boolean][] = [ + [{ name: { $eq: 1 } }, { name: "Michael" }, false], + [{ name: "Michael", age: 40 }, { name: "Michael", age: 40 }, true], + [{ name: "Michael", age: 40 }, { name: "Michael", age: 41 }, false], + [{ name: { $eq: "Michael" } }, { name: "Michael" }, true], + [{ int: { $between: [1, 2] } }, { int: 1 }, true], + [{ int: { $between: [1, 2] } }, { int: 3 }, false], + [{ some: { $isnull: 1 } }, { some: null }, true], + [{ some: { $isnull: true } }, { some: null }, true], + [{ some: { $isnull: 0 } }, { some: null }, false], + [{ some: { $isnull: false } }, { some: null }, false], + [{ some: { $isnull: 1 } }, { some: 1 }, false], + [{ val: { $notnull: 1 } }, { val: 1 }, true], + [{ val: { $notnull: 1 } }, { val: null }, false], + [{ val: { $regex: ".*" } }, { val: "test" }, true], + [{ val: { $regex: /^t.*/ } }, { val: "test" }, true], + [{ val: { $regex: /^b.*/ } }, { val: "test" }, false] + ]; + + for (const [query, object, expected] of tests) { + const result = validate(query, object); + expect(result).toBe(expected); + } + }); + + test("multiple validations", () => { + const tests: [ObjectQuery, any, boolean][] = [ + // multiple constraints per property + [{ val: { $lt: 10, $gte: 3 } }, { val: 7 }, true], + [{ val: { $lt: 10, $gte: 3 } }, { val: 2 }, false], + [{ val: { $lt: 10, $gte: 3 } }, { val: 11 }, false], + + // multiple properties + [{ val1: { $eq: "foo" }, val2: { $eq: "bar" } }, { val1: "foo", val2: "bar" }, true], + [{ val1: { $eq: "foo" }, val2: { $eq: "bar" } }, { val1: "bar", val2: "foo" }, false], + + // or constructs + [ + { $or: { val1: { $eq: "foo" }, val2: { $eq: "bar" } } }, + { val1: "foo", val2: "bar" }, + true + ], + [{ val1: { $eq: 1 }, $or: { val1: { $eq: 2 } } }, { val1: 1 }, true], + [{ val1: { $eq: 1 }, $or: { val1: { $eq: 2 } } }, { val1: 3 }, false] + ]; + + for (const [query, object, expected] of tests) { + const result = validate(query, object); + expect(result).toBe(expected); + } + }); +}); diff --git a/app/__test__/core/utils.spec.ts b/app/__test__/core/utils.spec.ts new file mode 100644 index 0000000..c7ccd45 --- /dev/null +++ b/app/__test__/core/utils.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test"; +import { Perf } from "../../src/core/utils"; +import * as reqres from "../../src/core/utils/reqres"; +import * as strings from "../../src/core/utils/strings"; + +async function wait(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +describe("Core Utils", async () => { + describe("[core] strings", async () => { + test("objectToKeyValueArray", async () => { + const obj = { a: 1, b: 2, c: 3 }; + const result = strings.objectToKeyValueArray(obj); + expect(result).toEqual([ + { key: "a", value: 1 }, + { key: "b", value: 2 }, + { key: "c", value: 3 } + ]); + }); + + test("snakeToPascalWithSpaces", async () => { + const result = strings.snakeToPascalWithSpaces("snake_to_pascal"); + expect(result).toBe("Snake To Pascal"); + }); + + test("randomString", async () => { + const result = strings.randomString(10); + expect(result).toHaveLength(10); + }); + + test("pascalToKebab", async () => { + const result = strings.pascalToKebab("PascalCase"); + expect(result).toBe("pascal-case"); + }); + + test("replaceSimplePlaceholders", async () => { + const str = "Hello, {$name}!"; + const vars = { name: "John" }; + const result = strings.replaceSimplePlaceholders(str, vars); + expect(result).toBe("Hello, John!"); + }); + }); + + describe("reqres", async () => { + test("headersToObject", () => { + const headers = new Headers(); + headers.append("Content-Type", "application/json"); + headers.append("Authorization", "Bearer 123"); + const obj = reqres.headersToObject(headers); + expect(obj).toEqual({ + "content-type": "application/json", + authorization: "Bearer 123" + }); + }); + + test("replaceUrlParam", () => { + const url = "/api/:id/:name"; + const params = { id: "123", name: "test" }; + const result = reqres.replaceUrlParam(url, params); + expect(result).toBe("/api/123/test"); + }); + + test("encode", () => { + const obj = { id: "123", name: "test" }; + const result = reqres.encodeSearch(obj); + expect(result).toBe("id=123&name=test"); + + const obj2 = { id: "123", name: ["test1", "test2"] }; + const result2 = reqres.encodeSearch(obj2); + expect(result2).toBe("id=123&name=test1&name=test2"); + + const obj3 = { id: "123", name: { test: "test" } }; + const result3 = reqres.encodeSearch(obj3, { encode: true }); + expect(result3).toBe("id=123&name=%7B%22test%22%3A%22test%22%7D"); + }); + }); + + describe("perf", async () => { + test("marks", async () => { + const perf = Perf.start(); + await wait(20); + perf.mark("boot"); + + await wait(10); + perf.mark("another"); + perf.close(); + + const perf2 = Perf.start(); + await wait(40); + perf2.mark("booted"); + await wait(10); + perf2.mark("what"); + perf2.close(); + + expect(perf.result().total).toBeLessThan(perf2.result().total); + }); + + test("executes correctly", async () => { + // write a test for "execute" method + let count = 0; + await Perf.execute(async () => { + count += 1; + }, 2); + + expect(count).toBe(2); + }); + }); +}); diff --git a/app/__test__/data/DataController.spec.ts b/app/__test__/data/DataController.spec.ts new file mode 100644 index 0000000..42ded5f --- /dev/null +++ b/app/__test__/data/DataController.spec.ts @@ -0,0 +1,235 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; + +import { Guard } from "../../src/auth"; +import { parse } from "../../src/core/utils"; +import { + Entity, + type EntityData, + EntityManager, + ManyToOneRelation, + type MutatorResponse, + type RepositoryResponse, + TextField +} from "../../src/data"; +import { DataController } from "../../src/data/api/DataController"; +import { dataConfigSchema } from "../../src/data/data-schema"; +import { disableConsoleLog, enableConsoleLog, getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +beforeAll(() => disableConsoleLog(["log", "warn"])); +afterAll(async () => (await afterAllCleanup()) && enableConsoleLog()); + +const dataConfig = parse(dataConfigSchema, {}); +describe("[data] DataController", async () => { + test("repoResult", async () => { + const em = new EntityManager([], dummyConnection); + const ctx: any = { em, guard: new Guard() }; + const controller = new DataController(ctx, dataConfig); + + const res = controller.repoResult({ + entity: null as any, + data: [] as any, + sql: "", + parameters: [] as any, + result: [] as any, + meta: { + total: 0, + count: 0, + items: 0 + } + }); + + expect(res).toEqual({ + meta: { + total: 0, + count: 0, + items: 0 + }, + data: [] + }); + }); + + test("mutatorResult", async () => { + const em = new EntityManager([], dummyConnection); + const ctx: any = { em, guard: new Guard() }; + const controller = new DataController(ctx, dataConfig); + + const res = controller.mutatorResult({ + entity: null as any, + data: [] as any, + sql: "", + parameters: [] as any, + result: [] as any + }); + + expect(res).toEqual({ + data: [] + }); + }); + + describe("getController", async () => { + const users = new Entity("users", [ + new TextField("name", { required: true }), + new TextField("bio") + ]); + const posts = new Entity("posts", [new TextField("content")]); + const em = new EntityManager([users, posts], dummyConnection, [ + new ManyToOneRelation(posts, users) + ]); + + await em.schema().sync({ force: true }); + + const fixtures = { + users: [ + { name: "foo", bio: "bar" }, + { name: "bar", bio: null }, + { name: "baz", bio: "!!!" } + ], + posts: [ + { content: "post 1", users_id: 1 }, + { content: "post 2", users_id: 2 } + ] + }; + + const ctx: any = { em, guard: new Guard() }; + const controller = new DataController(ctx, dataConfig); + const app = controller.getController(); + + test("entityExists", async () => { + expect(controller.entityExists("users")).toBe(true); + expect(controller.entityExists("posts")).toBe(true); + expect(controller.entityExists("settings")).toBe(false); + }); + + // @todo: update + test("/ (get info)", async () => { + const res = await app.request("/"); + const data = (await res.json()) as any; + const entities = Object.keys(data.entities); + const relations = Object.values(data.relations).map((r: any) => r.type); + + expect(entities).toEqual(["users", "posts"]); + expect(relations).toEqual(["n:1"]); + }); + + test("/:entity (insert one)", async () => { + //console.log("app.routes", app.routes); + // create users + for await (const _user of fixtures.users) { + const res = await app.request("/users", { + method: "POST", + body: JSON.stringify(_user) + }); + //console.log("res", { _user }, res); + const result = (await res.json()) as MutatorResponse; + const { id, ...data } = result.data as any; + + expect(res.status).toBe(201); + expect(res.ok).toBe(true); + expect(data as any).toEqual(_user); + } + + // create posts + for await (const _post of fixtures.posts) { + const res = await app.request("/posts", { + method: "POST", + body: JSON.stringify(_post) + }); + const result = (await res.json()) as MutatorResponse; + const { id, ...data } = result.data as any; + + expect(res.status).toBe(201); + expect(res.ok).toBe(true); + expect(data as any).toEqual(_post); + } + }); + + test("/:entity (read many)", async () => { + const res = await app.request("/users"); + const data = (await res.json()) as RepositoryResponse; + + expect(data.meta.total).toBe(3); + expect(data.meta.count).toBe(3); + expect(data.meta.items).toBe(3); + expect(data.data.length).toBe(3); + expect(data.data[0].name).toBe("foo"); + }); + + test("/:entity/query (func query)", async () => { + const res = await app.request("/users/query", { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + where: { bio: { $isnull: 1 } } + }) + }); + const data = (await res.json()) as RepositoryResponse; + + expect(data.meta.total).toBe(3); + expect(data.meta.count).toBe(1); + expect(data.meta.items).toBe(1); + expect(data.data.length).toBe(1); + expect(data.data[0].name).toBe("bar"); + }); + + test("/:entity (read many, paginated)", async () => { + const res = await app.request("/users?limit=1&offset=2"); + const data = (await res.json()) as RepositoryResponse; + + expect(data.meta.total).toBe(3); + expect(data.meta.count).toBe(3); + expect(data.meta.items).toBe(1); + expect(data.data.length).toBe(1); + expect(data.data[0].name).toBe("baz"); + }); + + test("/:entity/:id (read one)", async () => { + const res = await app.request("/users/3"); + const data = (await res.json()) as RepositoryResponse; + console.log("data", data); + + expect(data.meta.total).toBe(3); + expect(data.meta.count).toBe(1); + expect(data.meta.items).toBe(1); + expect(data.data).toEqual({ id: 3, ...fixtures.users[2] }); + }); + + test("/:entity (update one)", async () => { + const res = await app.request("/users/3", { + method: "PATCH", + body: JSON.stringify({ name: "new name" }) + }); + const { data } = (await res.json()) as MutatorResponse; + + expect(res.ok).toBe(true); + expect(data as any).toEqual({ id: 3, ...fixtures.users[2], name: "new name" }); + }); + + test("/:entity/:id/:reference (read references)", async () => { + const res = await app.request("/users/1/posts"); + const data = (await res.json()) as RepositoryResponse; + console.log("data", data); + + expect(data.meta.total).toBe(2); + expect(data.meta.count).toBe(1); + expect(data.meta.items).toBe(1); + expect(data.data.length).toBe(1); + expect(data.data[0].content).toBe("post 1"); + }); + + test("/:entity/:id (delete one)", async () => { + const res = await app.request("/posts/2", { + method: "DELETE" + }); + const { data } = (await res.json()) as RepositoryResponse; + expect(data).toEqual({ id: 2, ...fixtures.posts[1] }); + + // verify + const res2 = await app.request("/posts"); + const data2 = (await res2.json()) as RepositoryResponse; + expect(data2.meta.total).toBe(1); + }); + }); +}); diff --git a/app/__test__/data/data-query-impl.spec.ts b/app/__test__/data/data-query-impl.spec.ts new file mode 100644 index 0000000..d5217df --- /dev/null +++ b/app/__test__/data/data-query-impl.spec.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; +import type { QueryObject } from "ufo"; +import { WhereBuilder, type WhereQuery } from "../../src/data/entities/query/WhereBuilder"; +import { getDummyConnection } from "./helper"; + +const t = "t"; +describe("data-query-impl", () => { + function qb() { + const c = getDummyConnection(); + const kysely = c.dummyConnection.kysely; + return kysely.selectFrom(t).selectAll(); + } + function compile(q: QueryObject) { + const { sql, parameters } = WhereBuilder.addClause(qb(), q).compile(); + return { sql, parameters }; + } + + test("single validation", () => { + const tests: [WhereQuery, string, any[]][] = [ + [{ name: "Michael", age: 40 }, '("name" = ? and "age" = ?)', ["Michael", 40]], + [{ name: { $eq: "Michael" } }, '"name" = ?', ["Michael"]], + [{ int: { $between: [1, 2] } }, '"int" between ? and ?', [1, 2]], + [{ val: { $isnull: 1 } }, '"val" is null', []], + [{ val: { $isnull: true } }, '"val" is null', []], + [{ val: { $isnull: 0 } }, '"val" is not null', []], + [{ val: { $isnull: false } }, '"val" is not null', []], + [{ val: { $like: "what" } }, '"val" like ?', ["what"]], + [{ val: { $like: "w*t" } }, '"val" like ?', ["w%t"]] + ]; + + for (const [query, expectedSql, expectedParams] of tests) { + const { sql, parameters } = compile(query); + expect(sql).toContain(`select * from "t" where ${expectedSql}`); + expect(parameters).toEqual(expectedParams); + } + }); + + test("multiple validations", () => { + const tests: [WhereQuery, string, any[]][] = [ + // multiple constraints per property + [{ val: { $lt: 10, $gte: 3 } }, '("val" < ? and "val" >= ?)', [10, 3]], + [{ val: { $lt: 10, $gte: 3 } }, '("val" < ? and "val" >= ?)', [10, 3]], + [{ val: { $lt: 10, $gte: 3 } }, '("val" < ? and "val" >= ?)', [10, 3]], + + // multiple properties + [ + { val1: { $eq: "foo" }, val2: { $eq: "bar" } }, + '("val1" = ? and "val2" = ?)', + ["foo", "bar"] + ], + [ + { val1: { $eq: "foo" }, val2: { $eq: "bar" } }, + '("val1" = ? and "val2" = ?)', + ["foo", "bar"] + ], + + // or constructs + [ + { $or: { val1: { $eq: "foo" }, val2: { $eq: "bar" } } }, + '("val1" = ? or "val2" = ?)', + ["foo", "bar"] + ], + [{ val1: { $eq: 1 }, $or: { val1: { $eq: 2 } } }, '("val1" = ? or "val1" = ?)', [1, 2]], + [{ val1: { $eq: 1 }, $or: { val1: { $eq: 2 } } }, '("val1" = ? or "val1" = ?)', [1, 2]] + ]; + + for (const [query, expectedSql, expectedParams] of tests) { + const { sql, parameters } = compile(query); + expect(sql).toContain(`select * from "t" where ${expectedSql}`); + expect(parameters).toEqual(expectedParams); + } + }); + + test("keys", () => { + const tests: [WhereQuery, string[]][] = [ + // multiple constraints per property + [{ val: { $lt: 10, $gte: 3 } }, ["val"]], + + // multiple properties + [{ val1: { $eq: "foo" }, val2: { $eq: "bar" } }, ["val1", "val2"]], + + // or constructs + [{ $or: { val1: { $eq: "foo" }, val2: { $eq: "bar" } } }, ["val1", "val2"]], + [{ val1: { $eq: 1 }, $or: { val1: { $eq: 2 } } }, ["val1"]] + ]; + + for (const [query, expectedKeys] of tests) { + const keys = WhereBuilder.getPropertyNames(query); + expect(keys).toEqual(expectedKeys); + } + }); +}); diff --git a/app/__test__/data/data.test.ts b/app/__test__/data/data.test.ts new file mode 100644 index 0000000..5d9c32a --- /dev/null +++ b/app/__test__/data/data.test.ts @@ -0,0 +1,113 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { + Entity, + EntityManager, + NumberField, + PrimaryField, + Repository, + TextField +} from "../../src/data"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("some tests", async () => { + //const connection = getLocalLibsqlConnection(); + const connection = dummyConnection; + + const users = new Entity("users", [ + new TextField("username", { required: true, default_value: "nobody" }), + new TextField("email", { max_length: 3 }) + ]); + + const posts = new Entity("posts", [ + new TextField("title"), + new TextField("content"), + new TextField("created_at"), + new NumberField("likes", { default_value: 0 }) + ]); + + const em = new EntityManager([users, posts], connection); + + await em.schema().sync({ force: true }); + + test("findId", async () => { + const query = await em.repository(users).findId(1); + /*const { result, total, count, time } = query; + console.log("query", query.result, { + result, + total, + count, + time, + });*/ + + expect(query.sql).toBe( + 'select "users"."id" as "id", "users"."username" as "username", "users"."email" as "email" from "users" where "id" = ? limit ?' + ); + expect(query.parameters).toEqual([1, 1]); + expect(query.result).toEqual([]); + }); + + test("findMany", async () => { + const query = await em.repository(users).findMany(); + + expect(query.sql).toBe( + 'select "users"."id" as "id", "users"."username" as "username", "users"."email" as "email" from "users" order by "users"."id" asc limit ? offset ?' + ); + expect(query.parameters).toEqual([10, 0]); + expect(query.result).toEqual([]); + }); + + test("findMany with number", async () => { + const query = await em.repository(posts).findMany(); + + expect(query.sql).toBe( + 'select "posts"."id" as "id", "posts"."title" as "title", "posts"."content" as "content", "posts"."created_at" as "created_at", "posts"."likes" as "likes" from "posts" order by "posts"."id" asc limit ? offset ?' + ); + expect(query.parameters).toEqual([10, 0]); + expect(query.result).toEqual([]); + }); + + test("try adding an existing field name", async () => { + expect(() => { + new Entity("users", [ + new TextField("username"), + new TextField("email"), + new TextField("email") // not throwing, it's just being ignored + ]); + }).toBeDefined(); + + expect(() => { + new Entity("users", [ + new TextField("username"), + new TextField("email"), + // field config differs, will throw + new TextField("email", { required: true }) + ]); + }).toThrow(); + + expect(() => { + new Entity("users", [ + new PrimaryField(), + new TextField("username"), + new TextField("email") + ]); + }).toBeDefined(); + }); + + test("try adding duplicate entities", async () => { + const entity = new Entity("users", [new TextField("username")]); + const entity2 = new Entity("users", [new TextField("userna1me")]); + + expect(() => { + // will not throw, just ignored + new EntityManager([entity, entity], connection); + }).toBeDefined(); + + expect(() => { + // the config differs, so it throws + new EntityManager([entity, entity2], connection); + }).toThrow(); + }); +}); diff --git a/app/__test__/data/helper.ts b/app/__test__/data/helper.ts new file mode 100644 index 0000000..dc4465b --- /dev/null +++ b/app/__test__/data/helper.ts @@ -0,0 +1,35 @@ +import { unlink } from "node:fs/promises"; +import type { SqliteDatabase } from "kysely"; +// @ts-ignore +import Database from "libsql"; +import { SqliteLocalConnection } from "../../src/data"; + +export function getDummyDatabase(memory: boolean = true): { + dummyDb: SqliteDatabase; + afterAllCleanup: () => Promise; +} { + const DB_NAME = memory ? ":memory:" : `${Math.random().toString(36).substring(7)}.db`; + const dummyDb = new Database(DB_NAME); + + return { + dummyDb, + afterAllCleanup: async () => { + if (!memory) await unlink(DB_NAME); + return true; + } + }; +} + +export function getDummyConnection(memory: boolean = true) { + const { dummyDb, afterAllCleanup } = getDummyDatabase(memory); + const dummyConnection = new SqliteLocalConnection(dummyDb); + + return { + dummyConnection, + afterAllCleanup + }; +} + +export function getLocalLibsqlConnection() { + return { url: "http://127.0.0.1:8080" }; +} diff --git a/app/__test__/data/mutation.relation.test.ts b/app/__test__/data/mutation.relation.test.ts new file mode 100644 index 0000000..32b8749 --- /dev/null +++ b/app/__test__/data/mutation.relation.test.ts @@ -0,0 +1,50 @@ +// eslint-disable-next-line import/no-unresolved +import { afterAll, describe, expect, test } from "bun:test"; +import { + Entity, + EntityManager, + ManyToOneRelation, + NumberField, + SchemaManager, + TextField +} from "../../src/data"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("Mutator relation", async () => { + const connection = dummyConnection; + //const connection = getLocalLibsqlConnection(); + //const connection = getCreds("DB_DATA"); + + const posts = new Entity("posts", [ + new TextField("title"), + new TextField("content", { default_value: "..." }), + new NumberField("count", { default_value: 0 }) + ]); + + const users = new Entity("users", [new TextField("username")]); + + const relations = [new ManyToOneRelation(posts, users)]; + + const em = new EntityManager([posts, users], connection, relations); + + const schema = new SchemaManager(em); + await schema.sync({ force: true }); + + test("add users", async () => { + const { data } = await em.mutator(users).insertOne({ username: "user1" }); + await em.mutator(users).insertOne({ username: "user2" }); + + // create some posts + await em.mutator(posts).insertOne({ title: "post1", content: "content1" }); + + // expect to throw + expect(em.mutator(posts).insertOne({ title: "post2", users_id: 10 })).rejects.toThrow(); + + expect( + em.mutator(posts).insertOne({ title: "post2", users_id: data.id }) + ).resolves.toBeDefined(); + }); +}); diff --git a/app/__test__/data/mutation.simple.test.ts b/app/__test__/data/mutation.simple.test.ts new file mode 100644 index 0000000..b3f0c77 --- /dev/null +++ b/app/__test__/data/mutation.simple.test.ts @@ -0,0 +1,145 @@ +// eslint-disable-next-line import/no-unresolved +import { afterAll, describe, expect, test } from "bun:test"; +import { Entity, EntityManager, Mutator, NumberField, TextField } from "../../src/data"; +import { TransformPersistFailedException } from "../../src/data/errors"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("Mutator simple", async () => { + const connection = dummyConnection; + //const connection = getLocalLibsqlConnection(); + //const connection = getCreds("DB_DATA"); + + const items = new Entity("items", [ + new TextField("label", { required: true, minLength: 1 }), + new NumberField("count", { default_value: 0 }) + ]); + const em = new EntityManager([items], connection); + + await em.connection.kysely.schema + .createTable("items") + .ifNotExists() + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull()) + .addColumn("label", "text") + .addColumn("count", "integer") + .execute(); + + test("insert single row", async () => { + const mutation = await em.mutator(items).insertOne({ + label: "test", + count: 1 + }); + + expect(mutation.sql).toBe( + 'insert into "items" ("count", "label") values (?, ?) returning "id", "label", "count"' + ); + expect(mutation.data).toEqual({ id: 1, label: "test", count: 1 }); + + const query = await em.repository(items).findMany({ + limit: 1, + sort: { + by: "id", + dir: "desc" + } + }); + + expect(query.result).toEqual([{ id: 1, label: "test", count: 1 }]); + }); + + test("update inserted row", async () => { + const query = await em.repository(items).findMany({ + limit: 1, + sort: { + by: "id", + dir: "desc" + } + }); + const id = query.data![0].id as number; + + const mutation = await em.mutator(items).updateOne(id, { + label: "new label", + count: 100 + }); + + expect(mutation.sql).toBe( + 'update "items" set "label" = ?, "count" = ? where "id" = ? returning "id", "label", "count"' + ); + expect(mutation.data).toEqual({ id, label: "new label", count: 100 }); + }); + + test("delete updated row", async () => { + const query = await em.repository(items).findMany({ + limit: 1, + sort: { + by: "id", + dir: "desc" + } + }); + + const id = query.data![0].id as number; + const mutation = await em.mutator(items).deleteOne(id); + + expect(mutation.sql).toBe( + 'delete from "items" where "id" = ? returning "id", "label", "count"' + ); + expect(mutation.data).toEqual({ id, label: "new label", count: 100 }); + + const query2 = await em.repository(items).findId(id); + expect(query2.result.length).toBe(0); + }); + + test("validation: insert incomplete row", async () => { + const incompleteCreate = async () => + await em.mutator(items).insertOne({ + //label: "test", + count: 1 + }); + + expect(incompleteCreate()).rejects.toThrow(); + }); + + test("validation: insert invalid row", async () => { + const invalidCreate1 = async () => + await em.mutator(items).insertOne({ + label: 111, // this should work + count: "1" // this should fail + }); + + expect(invalidCreate1()).rejects.toThrow(TransformPersistFailedException); + + const invalidCreate2 = async () => + await em.mutator(items).insertOne({ + label: "", // this should fail + count: 1 + }); + + expect(invalidCreate2()).rejects.toThrow(TransformPersistFailedException); + }); + + test("test default value", async () => { + const res = await em.mutator(items).insertOne({ label: "yo" }); + + expect(res.data.count).toBe(0); + }); + + test("deleteMany", async () => { + await em.mutator(items).insertOne({ label: "keep" }); + await em.mutator(items).insertOne({ label: "delete" }); + await em.mutator(items).insertOne({ label: "delete" }); + + const data = (await em.repository(items).findMany()).data; + //console.log(data); + + await em.mutator(items).deleteMany({ label: "delete" }); + + expect((await em.repository(items).findMany()).data.length).toBe(data.length - 2); + //console.log((await em.repository(items).findMany()).data); + + await em.mutator(items).deleteMany(); + expect((await em.repository(items).findMany()).data.length).toBe(0); + + //expect(res.data.count).toBe(0); + }); +}); diff --git a/app/__test__/data/polymorphic.test.ts b/app/__test__/data/polymorphic.test.ts new file mode 100644 index 0000000..44df2de --- /dev/null +++ b/app/__test__/data/polymorphic.test.ts @@ -0,0 +1,96 @@ +import { afterAll, expect as bunExpect, describe, test } from "bun:test"; +import { stripMark } from "../../src/core/utils"; +import { Entity, EntityManager, PolymorphicRelation, TextField } from "../../src/data"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +const expect = (value: any) => bunExpect(stripMark(value)); + +describe("Polymorphic", async () => { + test("Simple", async () => { + const categories = new Entity("categories", [new TextField("name")]); + const media = new Entity("media", [new TextField("path")]); + + const entities = [media, categories]; + const relation = new PolymorphicRelation(categories, media, { mappedBy: "image" }); + + const em = new EntityManager(entities, dummyConnection, [relation]); + + expect(em.relationsOf(categories.name).map((r) => r.toJSON())[0]).toEqual({ + type: "poly", + source: "categories", + target: "media", + config: { + mappedBy: "image" + } + }); + // media should not see categories + expect(em.relationsOf(media.name).map((r) => r.toJSON())).toEqual([]); + + // it's important that media cannot access categories + expect(em.relations.targetRelationsOf(categories).map((r) => r.source.entity.name)).toEqual( + [] + ); + expect(em.relations.targetRelationsOf(media).map((r) => r.source.entity.name)).toEqual([]); + + expect(em.relations.sourceRelationsOf(categories).map((r) => r.target.entity.name)).toEqual([ + "media" + ]); + expect(em.relations.sourceRelationsOf(categories).map((r) => r.target.reference)).toEqual([ + "image" + ]); + expect(em.relations.sourceRelationsOf(media).map((r) => r.target.entity.name)).toEqual([]); + + // expect that polymorphic fields are added to media + expect(media.getFields().map((f) => f.name)).toEqual([ + "id", + "path", + "reference", + "entity_id" + ]); + expect(media.getSelect()).toEqual(["id", "path"]); + }); + + test("Multiple to the same", async () => { + const categories = new Entity("categories", [new TextField("name")]); + const media = new Entity("media", [new TextField("path")]); + + const entities = [media, categories]; + const single = new PolymorphicRelation(categories, media, { + mappedBy: "single", + targetCardinality: 1 + }); + const multiple = new PolymorphicRelation(categories, media, { mappedBy: "multiple" }); + + const em = new EntityManager(entities, dummyConnection, [single, multiple]); + + // media should not see categories + expect(em.relationsOf(media.name).map((r) => r.toJSON())).toEqual([]); + + // it's important that media cannot access categories + expect(em.relations.targetRelationsOf(categories).map((r) => r.source.entity.name)).toEqual( + [] + ); + expect(em.relations.targetRelationsOf(media).map((r) => r.source.entity.name)).toEqual([]); + + expect(em.relations.sourceRelationsOf(categories).map((r) => r.target.entity.name)).toEqual([ + "media", + "media" + ]); + expect(em.relations.sourceRelationsOf(categories).map((r) => r.target.reference)).toEqual([ + "single", + "multiple" + ]); + expect(em.relations.sourceRelationsOf(media).map((r) => r.target.entity.name)).toEqual([]); + + // expect that polymorphic fields are added to media + expect(media.getFields().map((f) => f.name)).toEqual([ + "id", + "path", + "reference", + "entity_id" + ]); + }); +}); diff --git a/app/__test__/data/prototype.test.ts b/app/__test__/data/prototype.test.ts new file mode 100644 index 0000000..955546e --- /dev/null +++ b/app/__test__/data/prototype.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, test } from "bun:test"; +import { MediaField } from "../../src"; +import { + BooleanField, + DateField, + Entity, + EnumField, + JsonField, + ManyToManyRelation, + ManyToOneRelation, + NumberField, + OneToOneRelation, + PolymorphicRelation, + TextField +} from "../../src/data"; +import { + FieldPrototype, + type FieldSchema, + type InsertSchema, + type Schema, + boolean, + date, + datetime, + entity, + enumm, + json, + media, + medium, + number, + relation, + text +} from "../../src/data/prototype"; + +describe("prototype", () => { + test("...", () => { + const fieldPrototype = new FieldPrototype("text", {}, false); + //console.log("field", fieldPrototype, fieldPrototype.getField("name")); + /*const user = entity("users", { + name: text().required(), + bio: text(), + age: number(), + some: number().required(), + }); + + console.log("user", user);*/ + }); + + test("...2", async () => { + const user = entity("users", { + name: text().required(), + bio: text(), + age: number(), + some: number().required() + }); + + //console.log("user", user.toJSON()); + }); + + test("...3", async () => { + const user = entity("users", { + name: text({ default_value: "hello" }).required(), + bio: text(), + age: number(), + some: number().required() + }); + + const obj: InsertSchema = { name: "yo", some: 1 }; + + //console.log("user2", user.toJSON()); + }); + + test("Post example", async () => { + const posts1 = new Entity("posts", [ + new TextField("title", { required: true }), + new TextField("content"), + new DateField("created_at", { + type: "datetime" + }), + new MediaField("images", { entity: "posts" }), + new MediaField("cover", { entity: "posts", max_items: 1 }) + ]); + + const posts2 = entity("posts", { + title: text().required(), + content: text(), + created_at: datetime(), + images: media(), + cover: medium() + }); + + type Posts = Schema; + + expect(posts1.toJSON()).toEqual(posts2.toJSON()); + }); + + test("test example", async () => { + const test = new Entity("test", [ + new TextField("name"), + new BooleanField("checked", { default_value: false }), + new NumberField("count"), + new DateField("created_at"), + new DateField("updated_at", { type: "datetime" }), + new TextField("description"), + new EnumField("status", { + options: { + type: "objects", + values: [ + { value: "active", label: "Active" }, + { value: "inactive", label: "Not active" } + ] + } + }), + new JsonField("json") + ]); + + const test2 = entity("test", { + name: text(), + checked: boolean({ default_value: false }), + count: number(), + created_at: date(), + updated_at: datetime(), + description: text(), + status: enumm<"active" | "inactive">({ + enum: [ + { value: "active", label: "Active" }, + { value: "inactive", label: "Not active" } + ] + }), + json: json<{ some: number }>() + }); + + expect(test.toJSON()).toEqual(test2.toJSON()); + }); + + test("relations", async () => { + const posts = entity("posts", {}); + const users = entity("users", {}); + const comments = entity("comments", {}); + const categories = entity("categories", {}); + const settings = entity("settings", {}); + const _media = entity("media", {}); + + const relations = [ + new ManyToOneRelation(posts, users, { mappedBy: "author", required: true }), + new OneToOneRelation(users, settings), + new ManyToManyRelation(posts, categories), + new ManyToOneRelation(comments, users, { required: true }), + new ManyToOneRelation(comments, posts, { required: true }), + + // category has single image + new PolymorphicRelation(categories, _media, { + mappedBy: "image", + targetCardinality: 1 + }), + + // post has multiple images + new PolymorphicRelation(posts, _media, { mappedBy: "images" }), + new PolymorphicRelation(posts, _media, { mappedBy: "cover", targetCardinality: 1 }) + ]; + + const relations2 = [ + relation(posts).manyToOne(users, { mappedBy: "author", required: true }), + relation(users).oneToOne(settings), + relation(posts).manyToMany(categories), + + relation(comments).manyToOne(users, { required: true }), + relation(comments).manyToOne(posts, { required: true }), + + relation(categories).polyToOne(_media, { mappedBy: "image" }), + + relation(posts).polyToMany(_media, { mappedBy: "images" }), + relation(posts).polyToOne(_media, { mappedBy: "cover" }) + ]; + + expect(relations.map((r) => r.toJSON())).toEqual(relations2.map((r) => r.toJSON())); + }); + + test("many to many fields", async () => { + const posts = entity("posts", {}); + const categories = entity("categories", {}); + + const rel = new ManyToManyRelation( + posts, + categories, + { + connectionTableMappedName: "custom" + }, + [new TextField("description")] + ); + + const fields = { + description: text() + }; + let o: FieldSchema; + const rel2 = relation(posts).manyToMany( + categories, + { + connectionTableMappedName: "custom" + }, + fields + ); + + expect(rel.toJSON()).toEqual(rel2.toJSON()); + }); + + test("devexample", async () => { + const users = entity("users", { + username: text() + }); + + const comments = entity("comments", { + content: text() + }); + + const posts = entity("posts", { + title: text().required(), + content: text(), + created_at: datetime(), + images: media(), + cover: medium() + }); + + const categories = entity("categories", { + name: text(), + description: text(), + image: medium() + }); + + const settings = entity("settings", { + theme: text() + }); + + const test = entity("test", { + name: text(), + checked: boolean({ default_value: false }), + count: number(), + created_at: date(), + updated_at: datetime(), + description: text(), + status: enumm<"active" | "inactive">({ + enum: [ + { value: "active", label: "Active" }, + { value: "inactive", label: "Not active" } + ] + }), + json: json<{ some: number }>() + }); + + const _media = entity("media", {}); + + const relations = [ + relation(posts).manyToOne(users, { mappedBy: "author", required: true }), + relation(posts).manyToMany(categories), + relation(posts).polyToMany(_media, { mappedBy: "images" }), + relation(posts).polyToOne(_media, { mappedBy: "cover" }), + + relation(categories).polyToOne(_media, { mappedBy: "image" }), + + relation(users).oneToOne(settings), + + relation(comments).manyToOne(users, { required: true }), + relation(comments).manyToOne(posts, { required: true }) + ]; + + const obj: Schema = {} as any; + }); +}); diff --git a/app/__test__/data/relations.test.ts b/app/__test__/data/relations.test.ts new file mode 100644 index 0000000..ea56388 --- /dev/null +++ b/app/__test__/data/relations.test.ts @@ -0,0 +1,368 @@ +// eslint-disable-next-line import/no-unresolved +import { afterAll, describe, expect, test } from "bun:test"; +import { Entity, EntityManager, TextField } from "../../src/data"; +import { + ManyToManyRelation, + ManyToOneRelation, + OneToOneRelation, + PolymorphicRelation, + RelationField +} from "../../src/data/relations"; +import { getDummyConnection } from "./helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("Relations", async () => { + test("RelationField", async () => { + const em = new EntityManager([], dummyConnection); + const schema = em.connection.kysely.schema; + + //const r1 = new RelationField(new Entity("users")); + const r1 = new RelationField("users_id", { + reference: "users", + target: "users", + target_field: "id" + }); + + const sql1 = schema + .createTable("posts") + .addColumn(...r1.schema()!) + .compile().sql; + + expect(sql1).toBe( + 'create table "posts" ("users_id" integer references "users" ("id") on delete set null)' + ); + + //const r2 = new RelationField(new Entity("users"), "author"); + const r2 = new RelationField("author_id", { + reference: "author", + target: "users", + target_field: "id" + }); + + const sql2 = schema + .createTable("posts") + .addColumn(...r2.schema()!) + .compile().sql; + + expect(sql2).toBe( + 'create table "posts" ("author_id" integer references "users" ("id") on delete set null)' + ); + }); + + test("Required RelationField", async () => { + //const r1 = new RelationField(new Entity("users"), undefined, { required: true }); + const r1 = new RelationField("users_id", { + reference: "users", + target: "users", + target_field: "id", + required: true + }); + expect(r1.isRequired()).toBeTrue(); + }); + + test("ManyToOne", async () => { + const users = new Entity("users", [new TextField("username")]); + const posts = new Entity("posts", [ + new TextField("title", { + maxLength: 2 + }) + ]); + + const entities = [users, posts]; + + const relationName = "author"; + const relations = [new ManyToOneRelation(posts, users, { mappedBy: relationName })]; + const em = new EntityManager(entities, dummyConnection, relations); + + // verify naming + const rel = em.relations.all[0]; + expect(rel.source.entity.name).toBe(posts.name); + expect(rel.source.reference).toBe(posts.name); + expect(rel.target.entity.name).toBe(users.name); + expect(rel.target.reference).toBe(relationName); + + // verify field + expect(posts.field(relationName + "_id")).toBeInstanceOf(RelationField); + + // verify low level relation + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name)[0].source.entity).toBe(posts); + expect(posts.field("author_id")).toBeInstanceOf(RelationField); + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name)[0].source.entity).toBe(posts); + + // verify high level relation (from users) + const userPostsRel = em.relationOf(users.name, "posts"); + expect(userPostsRel).toBeInstanceOf(ManyToOneRelation); + expect(userPostsRel?.other(users).entity).toBe(posts); + + // verify high level relation (from posts) + const postAuthorRel = em.relationOf(posts.name, "author")! as ManyToOneRelation; + expect(postAuthorRel).toBeInstanceOf(ManyToOneRelation); + expect(postAuthorRel?.other(posts).entity).toBe(users); + + const kysely = em.connection.kysely; + const jsonFrom = (e) => e; + /** + * Relation Helper + */ + /** + * FROM POSTS + * ---------- + - lhs: posts.author_id + - rhs: users.id + - as: author + - select: users.* + - cardinality: 1 + */ + const selectPostsFromUsers = postAuthorRel.buildWith( + users, + kysely.selectFrom(users.name), + jsonFrom, + "posts" + ); + expect(selectPostsFromUsers.compile().sql).toBe( + 'select (select "posts"."id" as "id", "posts"."title" as "title", "posts"."author_id" as "author_id" from "posts" as "posts" where "posts"."author_id" = "users"."id" limit ?) as "posts" from "users"' + ); + expect(postAuthorRel!.getField()).toBeInstanceOf(RelationField); + const userObj = { id: 1, username: "test" }; + expect(postAuthorRel.hydrate(users, [userObj], em)).toEqual(userObj); + + /** + FROM USERS + ---------- + - lhs: posts.author_id + - rhs: users.id + - as: posts + - select: posts.* + - cardinality: + */ + const selectUsersFromPosts = postAuthorRel.buildWith( + posts, + kysely.selectFrom(posts.name), + jsonFrom, + "author" + ); + + expect(selectUsersFromPosts.compile().sql).toBe( + 'select (select "author"."id" as "id", "author"."username" as "username" from "users" as "author" where "author"."id" = "posts"."author_id" limit ?) as "author" from "posts"' + ); + expect(postAuthorRel.getField()).toBeInstanceOf(RelationField); + const postObj = { id: 1, title: "test" }; + expect(postAuthorRel.hydrate(posts, [postObj], em)).toEqual([postObj]); + + // mutation info + expect(postAuthorRel!.helper(users.name)!.getMutationInfo()).toEqual({ + reference: "posts", + local_field: undefined, + $set: false, + $create: false, + $attach: false, + $detach: false, + primary: undefined, + cardinality: undefined, + relation_type: "n:1" + }); + + expect(postAuthorRel!.helper(posts.name)!.getMutationInfo()).toEqual({ + reference: "author", + local_field: "author_id", + $set: true, + $create: false, + $attach: false, + $detach: false, + primary: "id", + cardinality: 1, + relation_type: "n:1" + }); + + /*console.log("ManyToOne (source=posts, target=users)"); + // prettier-ignore + console.log("users perspective",postAuthorRel!.helper(users.name)!.getMutationInfo()); + // prettier-ignore + console.log("posts perspective", postAuthorRel!.helper(posts.name)!.getMutationInfo()); + console.log("");*/ + }); + + test("OneToOne", async () => { + const users = new Entity("users", [new TextField("username")]); + const settings = new Entity("settings", [new TextField("theme")]); + + const entities = [users, settings]; + const relations = [new OneToOneRelation(users, settings)]; + + const em = new EntityManager(entities, dummyConnection, relations); + + // verify naming + const rel = em.relations.all[0]; + expect(rel.source.entity.name).toBe(users.name); + expect(rel.source.reference).toBe(users.name); + expect(rel.target.entity.name).toBe(settings.name); + expect(rel.target.reference).toBe(settings.name); + + // verify fields (only one added to users (source)) + expect(users.field("settings_id")).toBeInstanceOf(RelationField); + + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name).length).toBe(1); + expect(em.relationsOf(users.name)[0].source.entity).toBe(users); + expect(em.relationsOf(users.name)[0].target.entity).toBe(settings); + + // verify high level relation (from users) + const userSettingRel = em.relationOf(users.name, settings.name); + expect(userSettingRel).toBeInstanceOf(OneToOneRelation); + expect(userSettingRel?.other(users).entity.name).toBe(settings.name); + + // verify high level relation (from settings) + const settingUserRel = em.relationOf(settings.name, users.name); + expect(settingUserRel).toBeInstanceOf(OneToOneRelation); + expect(settingUserRel?.other(settings).entity.name).toBe(users.name); + + // mutation info + expect(userSettingRel!.helper(users.name)!.getMutationInfo()).toEqual({ + reference: "settings", + local_field: "settings_id", + $set: true, + $create: true, + $attach: false, + $detach: false, + primary: "id", + cardinality: 1, + relation_type: "1:1" + }); + expect(userSettingRel!.helper(settings.name)!.getMutationInfo()).toEqual({ + reference: "users", + local_field: undefined, + $set: false, + $create: false, + $attach: false, + $detach: false, + primary: undefined, + cardinality: 1, + relation_type: "1:1" + }); + + /*console.log(""); + console.log("OneToOne (source=users, target=settings)"); + // prettier-ignore + console.log("users perspective",userSettingRel!.helper(users.name)!.getMutationInfo()); + // prettier-ignore + console.log("settings perspective", userSettingRel!.helper(settings.name)!.getMutationInfo()); + console.log("");*/ + }); + + test("ManyToMany", async () => { + const posts = new Entity("posts", [new TextField("title")]); + const categories = new Entity("categories", [new TextField("label")]); + + const entities = [posts, categories]; + const relations = [new ManyToManyRelation(posts, categories)]; + + const em = new EntityManager(entities, dummyConnection, relations); + + //console.log((await em.schema().sync(true)).map((s) => s.sql).join(";\n")); + + // don't expect new fields bc of connection table + expect(posts.getFields().length).toBe(2); + expect(categories.getFields().length).toBe(2); + + // expect relations set + expect(em.relationsOf(posts.name).length).toBe(1); + expect(em.relationsOf(categories.name).length).toBe(1); + + // expect connection table with fields + expect(em.entity("posts_categories")).toBeInstanceOf(Entity); + expect(em.entity("posts_categories").getFields().length).toBe(3); + expect(em.entity("posts_categories").field("posts_id")).toBeInstanceOf(RelationField); + expect(em.entity("posts_categories").field("categories_id")).toBeInstanceOf(RelationField); + + // verify high level relation (from posts) + const postCategoriesRel = em.relationOf(posts.name, categories.name); + expect(postCategoriesRel).toBeInstanceOf(ManyToManyRelation); + expect(postCategoriesRel?.other(posts).entity.name).toBe(categories.name); + + //console.log("relation", postCategoriesRel); + + // verify high level relation (from posts) + const categoryPostsRel = em.relationOf(categories.name, posts.name); + expect(categoryPostsRel).toBeInstanceOf(ManyToManyRelation); + expect(categoryPostsRel?.other(categories.name).entity.name).toBe(posts.name); + + // now get connection table from relation (from posts) + if (postCategoriesRel instanceof ManyToManyRelation) { + expect(postCategoriesRel.connectionEntity.name).toBe("posts_categories"); + expect(em.entity(postCategoriesRel.connectionEntity.name).name).toBe("posts_categories"); + } else { + throw new Error("Expected ManyToManyRelation"); + } + + /** + * Relation Helper + */ + const kysely = em.connection.kysely; + const jsonFrom = (e) => e; + + /** + * FROM POSTS + * ---------- + - lhs: posts.author_id + - rhs: users.id + - as: author + - select: users.* + - cardinality: 1 + */ + const selectCategoriesFromPosts = postCategoriesRel.buildWith( + posts, + kysely.selectFrom(posts.name), + jsonFrom + ); + expect(selectCategoriesFromPosts.compile().sql).toBe( + 'select (select "categories"."id" as "id", "categories"."label" as "label" from "categories" inner join "posts_categories" on "categories"."id" = "posts_categories"."categories_id" where "posts"."id" = "posts_categories"."posts_id" limit ?) as "categories" from "posts"' + ); + + const selectPostsFromCategories = postCategoriesRel.buildWith( + categories, + kysely.selectFrom(categories.name), + jsonFrom + ); + expect(selectPostsFromCategories.compile().sql).toBe( + 'select (select "posts"."id" as "id", "posts"."title" as "title" from "posts" inner join "posts_categories" on "posts"."id" = "posts_categories"."posts_id" where "categories"."id" = "posts_categories"."categories_id" limit ?) as "posts" from "categories"' + ); + + // mutation info + expect(relations[0].helper(posts.name)!.getMutationInfo()).toEqual({ + reference: "categories", + local_field: undefined, + $set: false, + $create: false, + $attach: true, + $detach: true, + primary: "id", + cardinality: undefined, + relation_type: "m:n" + }); + expect(relations[0].helper(categories.name)!.getMutationInfo()).toEqual({ + reference: "posts", + local_field: undefined, + $set: false, + $create: false, + $attach: false, + $detach: false, + primary: undefined, + cardinality: undefined, + relation_type: "m:n" + }); + + /*console.log(""); + console.log("ManyToMany (source=posts, target=categories)"); + // prettier-ignore + console.log("posts perspective",relations[0].helper(posts.name)!.getMutationInfo()); + // prettier-ignore + console.log("categories perspective", relations[0]!.helper(categories.name)!.getMutationInfo()); + console.log("");*/ + }); +}); diff --git a/app/__test__/data/specs/Entity.spec.ts b/app/__test__/data/specs/Entity.spec.ts new file mode 100644 index 0000000..9b66b96 --- /dev/null +++ b/app/__test__/data/specs/Entity.spec.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { Entity, NumberField, TextField } from "../../../src/data"; + +describe("[data] Entity", async () => { + const entity = new Entity("test", [ + new TextField("name", { required: true }), + new TextField("description"), + new NumberField("age", { fillable: false, default_value: 18 }), + new TextField("hidden", { hidden: true, default_value: "secret" }) + ]); + + test("getSelect", async () => { + expect(entity.getSelect()).toEqual(["id", "name", "description", "age"]); + }); + + test("getFillableFields", async () => { + expect(entity.getFillableFields().map((f) => f.name)).toEqual([ + "name", + "description", + "hidden" + ]); + }); + + test("getRequiredFields", async () => { + expect(entity.getRequiredFields().map((f) => f.name)).toEqual(["name"]); + }); + + test("getDefaultObject", async () => { + expect(entity.getDefaultObject()).toEqual({ + age: 18, + hidden: "secret" + }); + }); + + test("getField", async () => { + expect(entity.getField("name")).toBeInstanceOf(TextField); + expect(entity.getField("age")).toBeInstanceOf(NumberField); + }); + + test("getPrimaryField", async () => { + expect(entity.getPrimaryField().name).toEqual("id"); + }); + + test("addField", async () => { + const field = new TextField("new_field"); + entity.addField(field); + expect(entity.getField("new_field")).toBe(field); + }); + + // @todo: move this to ClientApp + /*test("serialize and deserialize", async () => { + const json = entity.toJSON(); + //sconsole.log("json", json.fields); + const newEntity = Entity.deserialize(json); + //console.log("newEntity", newEntity.toJSON().fields); + expect(newEntity).toBeInstanceOf(Entity); + expect(json).toEqual(newEntity.toJSON()); + expect(json.fields).toEqual(newEntity.toJSON().fields); + });*/ +}); diff --git a/app/__test__/data/specs/EntityManager.spec.ts b/app/__test__/data/specs/EntityManager.spec.ts new file mode 100644 index 0000000..797a419 --- /dev/null +++ b/app/__test__/data/specs/EntityManager.spec.ts @@ -0,0 +1,106 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { + Entity, + EntityManager, + ManyToManyRelation, + ManyToOneRelation, + SchemaManager +} from "../../../src/data"; +import { UnableToConnectException } from "../../../src/data/errors"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("[data] EntityManager", async () => { + test("base empty throw", async () => { + // @ts-expect-error - testing invalid input, connection is required + expect(() => new EntityManager([], {})).toThrow(UnableToConnectException); + }); + + test("base w/o entities & relations", async () => { + const em = new EntityManager([], dummyConnection); + expect(em.entities).toEqual([]); + expect(em.relations.all).toEqual([]); + expect(await em.ping()).toBe(true); + expect(() => em.entity("...")).toThrow(); + expect(() => + em.addRelation(new ManyToOneRelation(new Entity("1"), new Entity("2"))) + ).toThrow(); + expect(em.schema()).toBeInstanceOf(SchemaManager); + + // the rest will all throw, since they depend on em.entity() + }); + + test("w/ 2 entities but no initial relations", async () => { + const users = new Entity("users"); + const posts = new Entity("posts"); + + const em = new EntityManager([users, posts], dummyConnection); + expect(em.entities).toEqual([users, posts]); + expect(em.relations.all).toEqual([]); + + expect(em.entity("users")).toBe(users); + expect(em.entity("posts")).toBe(posts); + + // expect adding relation to pass + em.addRelation(new ManyToOneRelation(posts, users)); + expect(em.relations.all.length).toBe(1); + expect(em.relations.all[0]).toBeInstanceOf(ManyToOneRelation); + expect(em.relationsOf("users")).toEqual([em.relations.all[0]]); + expect(em.relationsOf("posts")).toEqual([em.relations.all[0]]); + expect(em.hasRelations("users")).toBe(true); + expect(em.hasRelations("posts")).toBe(true); + expect(em.relatedEntitiesOf("users")).toEqual([posts]); + expect(em.relatedEntitiesOf("posts")).toEqual([users]); + expect(em.relationReferencesOf("users")).toEqual(["posts"]); + expect(em.relationReferencesOf("posts")).toEqual(["users"]); + }); + + test("test target relations", async () => { + const users = new Entity("users"); + const posts = new Entity("posts"); + const comments = new Entity("comments"); + const categories = new Entity("categories"); + + const em = new EntityManager([users, posts, comments, categories], dummyConnection); + em.addRelation(new ManyToOneRelation(posts, users)); + em.addRelation(new ManyToOneRelation(comments, users)); + em.addRelation(new ManyToOneRelation(comments, posts)); + em.addRelation(new ManyToManyRelation(posts, categories)); + + const userTargetRel = em.relations.targetRelationsOf(users); + const postTargetRel = em.relations.targetRelationsOf(posts); + const commentTargetRel = em.relations.targetRelationsOf(comments); + + expect(userTargetRel.map((r) => r.source.entity.name)).toEqual(["posts", "comments"]); + expect(postTargetRel.map((r) => r.source.entity.name)).toEqual(["comments"]); + expect(commentTargetRel.map((r) => r.source.entity.name)).toEqual([]); + }); + + test("test listable relations", async () => { + const users = new Entity("users"); + const posts = new Entity("posts"); + const comments = new Entity("comments"); + const categories = new Entity("categories"); + + const em = new EntityManager([users, posts, comments, categories], dummyConnection); + em.addRelation(new ManyToOneRelation(posts, users)); + em.addRelation(new ManyToOneRelation(comments, users)); + em.addRelation(new ManyToOneRelation(comments, posts)); + em.addRelation(new ManyToManyRelation(posts, categories)); + + const userTargetRel = em.relations.listableRelationsOf(users); + const postTargetRel = em.relations.listableRelationsOf(posts); + const commentTargetRel = em.relations.listableRelationsOf(comments); + const categoriesTargetRel = em.relations.listableRelationsOf(categories); + + expect(userTargetRel.map((r) => r.other(users).entity.name)).toEqual(["posts", "comments"]); + expect(postTargetRel.map((r) => r.other(posts).entity.name)).toEqual([ + "comments", + "categories" + ]); + expect(commentTargetRel.map((r) => r.other(comments).entity.name)).toEqual([]); + expect(categoriesTargetRel.map((r) => r.other(categories).entity.name)).toEqual(["posts"]); + }); +}); diff --git a/app/__test__/data/specs/JoinBuilder.spec.ts b/app/__test__/data/specs/JoinBuilder.spec.ts new file mode 100644 index 0000000..37afd08 --- /dev/null +++ b/app/__test__/data/specs/JoinBuilder.spec.ts @@ -0,0 +1,43 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { Entity, EntityManager, ManyToOneRelation, TextField } from "../../../src/data"; +import { JoinBuilder } from "../../../src/data/entities/query/JoinBuilder"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("[data] JoinBuilder", async () => { + test("missing relation", async () => { + const users = new Entity("users", [new TextField("username")]); + const em = new EntityManager([users], dummyConnection); + + expect(() => + JoinBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, ["posts"]) + ).toThrow('Relation "posts" not found'); + }); + + test("addClause: ManyToOne", async () => { + const users = new Entity("users", [new TextField("username")]); + const posts = new Entity("posts", [new TextField("content")]); + const relations = [new ManyToOneRelation(posts, users, { mappedBy: "author" })]; + const em = new EntityManager([users, posts], dummyConnection, relations); + + const qb = JoinBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, [ + "posts" + ]); + + const res = qb.compile(); + console.log("compiled", res.sql); + + /*expect(res.sql).toBe( + 'select from "users" inner join "posts" on "posts"."author_id" = "users"."id" group by "users"."id"', + );*/ + + const qb2 = JoinBuilder.addClause(em, em.connection.kysely.selectFrom("posts"), posts, [ + "author" + ]); + + const res2 = qb2.compile(); + console.log("compiled2", res2.sql); + }); +}); diff --git a/app/__test__/data/specs/Mutator.spec.ts b/app/__test__/data/specs/Mutator.spec.ts new file mode 100644 index 0000000..04bd8a3 --- /dev/null +++ b/app/__test__/data/specs/Mutator.spec.ts @@ -0,0 +1,302 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { + Entity, + EntityManager, + ManyToOneRelation, + MutatorEvents, + NumberField, + OneToOneRelation, + type RelationField, + RelationMutator, + TextField +} from "../../../src/data"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("[data] Mutator (base)", async () => { + const entity = new Entity("items", [ + new TextField("label", { required: true }), + new NumberField("count"), + new TextField("hidden", { hidden: true }), + new TextField("not_fillable", { fillable: false }) + ]); + const em = new EntityManager([entity], dummyConnection); + await em.schema().sync({ force: true }); + + const payload = { label: "item 1", count: 1 }; + + test("insertOne", async () => { + expect(em.mutator(entity).getValidatedData(payload, "create")).resolves.toEqual(payload); + const res = await em.mutator(entity).insertOne(payload); + + // checking params, because we can't know the id + // if it wouldn't be successful, it would throw an error + expect(res.parameters).toEqual(Object.values(payload)); + + // but expect additional fields to be present + expect((res.data as any).not_fillable).toBeDefined(); + }); + + test("updateOne", async () => { + const { data } = await em.mutator(entity).insertOne(payload); + const updated = await em.mutator(entity).updateOne(data.id, { + count: 2 + }); + + expect(updated.parameters).toEqual([2, data.id]); + expect(updated.data.count).toBe(2); + }); + + test("deleteOne", async () => { + const { data } = await em.mutator(entity).insertOne(payload); + const deleted = await em.mutator(entity).deleteOne(data.id); + + expect(deleted.parameters).toEqual([data.id]); + }); +}); + +describe("[data] Mutator (ManyToOne)", async () => { + const posts = new Entity("posts", [new TextField("title")]); + const users = new Entity("users", [new TextField("username")]); + const relations = [new ManyToOneRelation(posts, users)]; + const em = new EntityManager([posts, users], dummyConnection, relations); + await em.schema().sync({ force: true }); + + test("RelationMutator", async () => { + // create entries + const userData = await em.mutator(users).insertOne({ username: "user1" }); + const postData = await em.mutator(posts).insertOne({ title: "post1" }); + + const postRelMutator = new RelationMutator(posts, em); + const postRelField = posts.getField("users_id")! as RelationField; + expect(postRelMutator.getRelationalKeys()).toEqual(["users", "users_id"]); + + // persisting relational field should just return key value to be added + expect( + postRelMutator.persistRelationField(postRelField, "users_id", userData.data.id) + ).resolves.toEqual(["users_id", userData.data.id]); + + // persisting invalid value should throw + expect(postRelMutator.persistRelationField(postRelField, "users_id", 0)).rejects.toThrow(); + + // persisting reference should ... + expect( + postRelMutator.persistReference(relations[0], "users", { + $set: { id: userData.data.id } + }) + ).resolves.toEqual(["users_id", userData.data.id]); + // @todo: add what methods are allowed to relation, like $create should not be allowed for post<>users + + process.exit(0); + + const userRelMutator = new RelationMutator(users, em); + expect(userRelMutator.getRelationalKeys()).toEqual(["posts"]); + }); + + test("insertOne: missing ref", async () => { + expect( + em.mutator(posts).insertOne({ + title: "post1", + users_id: 1 // user does not exist yet + }) + ).rejects.toThrow(); + }); + + test("insertOne: missing required relation", async () => { + const items = new Entity("items", [new TextField("label")]); + const cats = new Entity("cats"); + const relations = [new ManyToOneRelation(items, cats, { required: true })]; + const em = new EntityManager([items, cats], dummyConnection, relations); + + expect(em.mutator(items).insertOne({ label: "test" })).rejects.toThrow( + 'Field "cats_id" is required' + ); + }); + + test("insertOne: using field name", async () => { + const { data } = await em.mutator(users).insertOne({ username: "user1" }); + const res = await em.mutator(posts).insertOne({ + title: "post1", + users_id: data.id + }); + expect(res.data.users_id).toBe(data.id); + + // setting "null" should be allowed + const res2 = await em.mutator(posts).insertOne({ + title: "post1", + users_id: null + }); + expect(res2.data.users_id).toBe(null); + }); + + test("insertOne: using reference", async () => { + const { data } = await em.mutator(users).insertOne({ username: "user1" }); + const res = await em.mutator(posts).insertOne({ + title: "post1", + users: { $set: { id: data.id } } + }); + expect(res.data.users_id).toBe(data.id); + + // setting "null" should be allowed + const res2 = await em.mutator(posts).insertOne({ + title: "post1", + users: { $set: { id: null } } + }); + expect(res2.data.users_id).toBe(null); + }); + + test("insertOne: performing unsupported operations", async () => { + expect( + em.mutator(posts).insertOne({ + title: "test", + users: { $create: { username: "test" } } + }) + ).rejects.toThrow(); + }); + + test("updateOne", async () => { + const res1 = await em.mutator(users).insertOne({ username: "user1" }); + const res1_1 = await em.mutator(users).insertOne({ username: "user1" }); + const res2 = await em.mutator(posts).insertOne({ title: "post1" }); + + const up1 = await em.mutator(posts).updateOne(res2.data.id, { + users: { $set: { id: res1.data.id } } + }); + expect(up1.data.users_id).toBe(res1.data.id); + + const up2 = await em.mutator(posts).updateOne(res2.data.id, { + users: { $set: { id: res1_1.data.id } } + }); + expect(up2.data.users_id).toBe(res1_1.data.id); + + const up3_1 = await em.mutator(posts).updateOne(res2.data.id, { + users_id: res1.data.id + }); + expect(up3_1.data.users_id).toBe(res1.data.id); + + const up3_2 = await em.mutator(posts).updateOne(res2.data.id, { + users_id: res1_1.data.id + }); + expect(up3_2.data.users_id).toBe(res1_1.data.id); + + const up4 = await em.mutator(posts).updateOne(res2.data.id, { + users_id: null + }); + expect(up4.data.users_id).toBe(null); + }); +}); + +describe("[data] Mutator (OneToOne)", async () => { + const users = new Entity("users", [new TextField("username")]); + const settings = new Entity("settings", [new TextField("theme")]); + const relations = [new OneToOneRelation(users, settings)]; + const em = new EntityManager([users, settings], dummyConnection, relations); + await em.schema().sync({ force: true }); + + test("insertOne: missing ref", async () => { + expect( + em.mutator(users).insertOne({ + username: "test", + settings_id: 1 // todo: throws because it doesn't exist, but it shouldn't be allowed + }) + ).rejects.toThrow(); + }); + + test("insertOne: using reference", async () => { + // $set is not allowed in OneToOne + const { data } = await em.mutator(settings).insertOne({ theme: "dark" }); + expect( + em.mutator(users).insertOne({ + username: "test", + settings: { $set: { id: data.id } } + }) + ).rejects.toThrow(); + }); + + test("insertOne: using $create", async () => { + const res = await em.mutator(users).insertOne({ + username: "test", + settings: { $create: { theme: "dark" } } + }); + expect(res.data.settings_id).toBeDefined(); + }); +}); +/* +describe("[data] Mutator (ManyToMany)", async () => { + const posts = new Entity("posts", [new TextField("title")]); + const tags = new Entity("tags", [new TextField("name")]); + const relations = [new ManyToOneRelation(posts, tags)]; + const em = new EntityManager([posts, tags], dummyConnection, relations); + await em.schema().sync({ force: true }); + + test("insertOne: missing ref", async () => { + expect( + em.mutator(posts).insertOne({ + title: "post1", + tags_id: 1, // tag does not exist yet + }), + ).rejects.toThrow(); + }); + + test("insertOne: using reference", async () => { + const { data } = await em.mutator(tags).insertOne({ name: "tag1" }); + const res = await em.mutator(posts).insertOne({ + title: "post1", + tags: { $attach: { id: data.id } }, + }); + expect(res.data.tags).toContain(data.id); + }); + + test("insertOne: using $create", async () => { + const res = await em.mutator(posts).insertOne({ + title: "post1", + tags: { $create: { name: "tag1" } }, + }); + expect(res.data.tags).toBeDefined(); + }); + + test("insertOne: using $detach", async () => { + const { data: tagData } = await em.mutator(tags).insertOne({ name: "tag1" }); + const { data: postData } = await em.mutator(posts).insertOne({ title: "post1" }); + + const res = await em.mutator(posts).insertOne({ + title: "post1", + tags: { $attach: { id: tagData.id } }, + }); + expect(res.data.tags).toContain(tagData.id); + + const res2 = await em.mutator(posts).updateOne(postData.id, { + tags: { $detach: { id: tagData.id } }, + }); + expect(res2.data.tags).not.toContain(tagData.id); + }); +});*/ + +describe("[data] Mutator (Events)", async () => { + const entity = new Entity("test", [new TextField("label")]); + const em = new EntityManager([entity], dummyConnection); + await em.schema().sync({ force: true }); + const events = new Map(); + + const mutator = em.mutator(entity); + mutator.emgr.onAny((event) => { + // @ts-ignore + events.set(event.constructor.slug, event); + }); + + test("events were fired", async () => { + const { data } = await mutator.insertOne({ label: "test" }); + expect(events.has(MutatorEvents.MutatorInsertBefore.slug)).toBeTrue(); + expect(events.has(MutatorEvents.MutatorInsertAfter.slug)).toBeTrue(); + + await mutator.updateOne(data.id, { label: "test2" }); + expect(events.has(MutatorEvents.MutatorUpdateBefore.slug)).toBeTrue(); + expect(events.has(MutatorEvents.MutatorUpdateAfter.slug)).toBeTrue(); + + await mutator.deleteOne(data.id); + expect(events.has(MutatorEvents.MutatorDeleteBefore.slug)).toBeTrue(); + expect(events.has(MutatorEvents.MutatorDeleteAfter.slug)).toBeTrue(); + }); +}); diff --git a/app/__test__/data/specs/Repository.spec.ts b/app/__test__/data/specs/Repository.spec.ts new file mode 100644 index 0000000..0ce8da1 --- /dev/null +++ b/app/__test__/data/specs/Repository.spec.ts @@ -0,0 +1,222 @@ +import { afterAll, describe, expect, test } from "bun:test"; +// @ts-ignore +import { Perf } from "@bknd/core/utils"; +import type { Kysely, Transaction } from "kysely"; +import { + Entity, + EntityManager, + LibsqlConnection, + ManyToOneRelation, + RepositoryEvents, + TextField +} from "../../../src/data"; +import { getDummyConnection } from "../helper"; + +type E = Kysely | Transaction; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +async function sleep(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +describe("[Repository]", async () => { + test("bulk", async () => { + //const connection = dummyConnection; + //const connection = getLocalLibsqlConnection(); + const credentials = null as any; // @todo: determine what to do here + const connection = new LibsqlConnection(credentials); + + const em = new EntityManager([], connection); + /*const emLibsql = new EntityManager([], { + url: connection.url.replace("https", "libsql"), + authToken: connection.authToken, + });*/ + const table = "posts"; + + const client = connection.getClient(); + if (!client) { + console.log("Cannot perform test without libsql connection"); + return; + } + + const conn = em.connection.kysely; + const selectQ = (e: E) => e.selectFrom(table).selectAll().limit(2); + const countQ = (e: E) => e.selectFrom(table).select(e.fn.count("*").as("count")); + + async function executeTransaction(em: EntityManager) { + return await em.connection.kysely.transaction().execute(async (e) => { + const res = await selectQ(e).execute(); + const count = await countQ(e).execute(); + + return [res, count]; + }); + } + + async function executeBatch(em: EntityManager) { + const queries = [selectQ(conn), countQ(conn)]; + return await em.connection.batchQuery(queries); + } + + async function executeSingleKysely(em: EntityManager) { + const res = await selectQ(conn).execute(); + const count = await countQ(conn).execute(); + return [res, count]; + } + + async function executeSingleClient(em: EntityManager) { + const q1 = selectQ(conn).compile(); + const res = await client.execute({ + sql: q1.sql, + args: q1.parameters as any + }); + + const q2 = countQ(conn).compile(); + const count = await client.execute({ + sql: q2.sql, + args: q2.parameters as any + }); + return [res, count]; + } + + const transaction = await executeTransaction(em); + const batch = await executeBatch(em); + + expect(batch).toEqual(transaction as any); + + const testperf = false; + if (testperf) { + const times = 5; + + const exec = async ( + name: string, + fn: (em: EntityManager) => Promise, + em: EntityManager + ) => { + const res = await Perf.execute(() => fn(em), times); + await sleep(1000); + const info = { + name, + total: res.total.toFixed(2), + avg: (res.total / times).toFixed(2), + first: res.marks[0].time.toFixed(2), + last: res.marks[res.marks.length - 1].time.toFixed(2) + }; + console.log(info.name, info, res.marks); + return info; + }; + + const data: any[] = []; + data.push(await exec("transaction.http", executeTransaction, em)); + data.push(await exec("bulk.http", executeBatch, em)); + data.push(await exec("singleKy.http", executeSingleKysely, em)); + data.push(await exec("singleCl.http", executeSingleClient, em)); + + /*data.push(await exec("transaction.libsql", executeTransaction, emLibsql)); + data.push(await exec("bulk.libsql", executeBatch, emLibsql)); + data.push(await exec("singleKy.libsql", executeSingleKysely, emLibsql)); + data.push(await exec("singleCl.libsql", executeSingleClient, emLibsql));*/ + + console.table(data); + /** + * ┌───┬────────────────────┬────────┬────────┬────────┬────────┐ + * │ │ name │ total │ avg │ first │ last │ + * ├───┼────────────────────┼────────┼────────┼────────┼────────┤ + * │ 0 │ transaction.http │ 681.29 │ 136.26 │ 136.46 │ 396.09 │ + * │ 1 │ bulk.http │ 164.82 │ 32.96 │ 32.95 │ 99.91 │ + * │ 2 │ singleKy.http │ 330.01 │ 66.00 │ 65.86 │ 195.41 │ + * │ 3 │ singleCl.http │ 326.17 │ 65.23 │ 61.32 │ 198.08 │ + * │ 4 │ transaction.libsql │ 856.79 │ 171.36 │ 132.31 │ 595.24 │ + * │ 5 │ bulk.libsql │ 180.63 │ 36.13 │ 35.39 │ 107.71 │ + * │ 6 │ singleKy.libsql │ 347.11 │ 69.42 │ 65.00 │ 207.14 │ + * │ 7 │ singleCl.libsql │ 328.60 │ 65.72 │ 62.19 │ 195.04 │ + * └───┴────────────────────┴────────┴────────┴────────┴────────┘ + */ + } + }); + + test("count & exists", async () => { + const items = new Entity("items", [new TextField("label")]); + const em = new EntityManager([items], dummyConnection); + + await em.connection.kysely.schema + .createTable("items") + .ifNotExists() + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull()) + .addColumn("label", "text") + .execute(); + + // fill + await em.connection.kysely + .insertInto("items") + .values([{ label: "a" }, { label: "b" }, { label: "c" }]) + .execute(); + + // count all + const res = await em.repository(items).count(); + expect(res.sql).toBe('select count(*) as "count" from "items"'); + expect(res.count).toBe(3); + + // count filtered + const res2 = await em.repository(items).count({ label: { $in: ["a", "b"] } }); + + expect(res2.sql).toBe('select count(*) as "count" from "items" where "label" in (?, ?)'); + expect(res2.parameters).toEqual(["a", "b"]); + expect(res2.count).toBe(2); + + // check exists + const res3 = await em.repository(items).exists({ label: "a" }); + expect(res3.exists).toBe(true); + + const res4 = await em.repository(items).exists({ label: "d" }); + expect(res4.exists).toBe(false); + + // for now, allow empty filter + const res5 = await em.repository(items).exists({}); + expect(res5.exists).toBe(true); + }); +}); + +describe("[data] Repository (Events)", async () => { + const items = new Entity("items", [new TextField("label")]); + const categories = new Entity("categories", [new TextField("label")]); + const em = new EntityManager([items, categories], dummyConnection, [ + new ManyToOneRelation(categories, items) + ]); + await em.schema().sync({ force: true }); + const events = new Map(); + + em.repository(items).emgr.onAny((event) => { + // @ts-ignore + events.set(event.constructor.slug, event); + }); + em.repository(categories).emgr.onAny((event) => { + // @ts-ignore + events.set(event.constructor.slug, event); + }); + + test("events were fired", async () => { + await em.repository(items).findId(1); + expect(events.has(RepositoryEvents.RepositoryFindOneBefore.slug)).toBeTrue(); + expect(events.has(RepositoryEvents.RepositoryFindOneAfter.slug)).toBeTrue(); + events.clear(); + + await em.repository(items).findOne({ id: 1 }); + expect(events.has(RepositoryEvents.RepositoryFindOneBefore.slug)).toBeTrue(); + expect(events.has(RepositoryEvents.RepositoryFindOneAfter.slug)).toBeTrue(); + events.clear(); + + await em.repository(items).findMany({ where: { id: 1 } }); + expect(events.has(RepositoryEvents.RepositoryFindManyBefore.slug)).toBeTrue(); + expect(events.has(RepositoryEvents.RepositoryFindManyAfter.slug)).toBeTrue(); + events.clear(); + + await em.repository(items).findManyByReference(1, "categories"); + expect(events.has(RepositoryEvents.RepositoryFindManyBefore.slug)).toBeTrue(); + expect(events.has(RepositoryEvents.RepositoryFindManyAfter.slug)).toBeTrue(); + events.clear(); + }); +}); diff --git a/app/__test__/data/specs/SchemaManager.spec.ts b/app/__test__/data/specs/SchemaManager.spec.ts new file mode 100644 index 0000000..4cfb7d0 --- /dev/null +++ b/app/__test__/data/specs/SchemaManager.spec.ts @@ -0,0 +1,269 @@ +// eslint-disable-next-line import/no-unresolved +import { afterAll, describe, expect, test } from "bun:test"; +import { randomString } from "../../../src/core/utils"; +import { Entity, EntityIndex, EntityManager, SchemaManager, TextField } from "../../../src/data"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("SchemaManager tests", async () => { + test("introspect entity", async () => { + const email = new TextField("email"); + const entity = new Entity("test", [new TextField("username"), email, new TextField("bio")]); + const index = new EntityIndex(entity, [email]); + const em = new EntityManager([entity], dummyConnection, [], [index]); + const schema = new SchemaManager(em); + + const introspection = schema.getIntrospectionFromEntity(em.entities[0]); + expect(introspection).toEqual({ + name: "test", + isView: false, + columns: [ + { + name: "id", + dataType: "TEXT", + isNullable: true, + isAutoIncrementing: true, + hasDefaultValue: false, + comment: undefined + }, + { + name: "username", + dataType: "TEXT", + isNullable: true, + isAutoIncrementing: false, + hasDefaultValue: false, + comment: undefined + }, + { + name: "email", + dataType: "TEXT", + isNullable: true, + isAutoIncrementing: false, + hasDefaultValue: false, + comment: undefined + }, + { + name: "bio", + dataType: "TEXT", + isNullable: true, + isAutoIncrementing: false, + hasDefaultValue: false, + comment: undefined + } + ], + indices: [ + { + name: "idx_test_email", + table: "test", + isUnique: false, + columns: [ + { + name: "email", + order: 0 + } + ] + } + ] + }); + }); + + test("add column", async () => { + const table = "add_column"; + const index = "idx_add_column"; + const em = new EntityManager( + [ + new Entity(table, [ + new TextField("username"), + new TextField("email"), + new TextField("bio") + ]) + ], + dummyConnection + ); + const kysely = em.connection.kysely; + + await kysely.schema + .createTable(table) + .ifNotExists() + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull()) + .addColumn("username", "text") + .addColumn("email", "text") + .execute(); + await kysely.schema.createIndex(index).on(table).columns(["username"]).execute(); + + const schema = new SchemaManager(em); + const diff = await schema.getDiff(); + + expect(diff).toEqual([ + { + name: table, + isNew: false, + columns: { add: ["bio"], drop: [], change: [] }, + indices: { add: [], drop: [index] } + } + ]); + + // now sync + await schema.sync({ force: true, drop: true }); + const diffAfter = await schema.getDiff(); + + console.log("diffAfter", diffAfter); + expect(diffAfter.length).toBe(0); + + await kysely.schema.dropTable(table).execute(); + }); + + test("drop column", async () => { + const table = "drop_column"; + const em = new EntityManager( + [new Entity(table, [new TextField("username")])], + dummyConnection + ); + const kysely = em.connection.kysely; + + await kysely.schema + .createTable(table) + .ifNotExists() + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull()) + .addColumn("username", "text") + .addColumn("email", "text") + .execute(); + + const schema = new SchemaManager(em); + const diff = await schema.getDiff(); + + expect(diff).toEqual([ + { + name: table, + isNew: false, + columns: { + add: [], + drop: ["email"], + change: [] + }, + indices: { add: [], drop: [] } + } + ]); + + // now sync + await schema.sync({ force: true, drop: true }); + const diffAfter = await schema.getDiff(); + + //console.log("diffAfter", diffAfter); + expect(diffAfter.length).toBe(0); + + await kysely.schema.dropTable(table).execute(); + }); + + test("create table and add column", async () => { + const usersTable = "create_users"; + const postsTable = "create_posts"; + const em = new EntityManager( + [ + new Entity(usersTable, [ + new TextField("username"), + new TextField("email"), + new TextField("bio") + ]), + new Entity(postsTable, [ + new TextField("title"), + new TextField("content"), + new TextField("created_at") + ]) + ], + dummyConnection + ); + const kysely = em.connection.kysely; + + await kysely.schema + .createTable(usersTable) + .addColumn("id", "integer", (col) => col.primaryKey().autoIncrement().notNull()) + .addColumn("username", "text") + .addColumn("email", "text") + .execute(); + + const schema = new SchemaManager(em); + const diff = await schema.getDiff(); + + expect(diff).toEqual([ + { + name: usersTable, + isNew: false, + columns: { add: ["bio"], drop: [], change: [] }, + indices: { add: [], drop: [] } + }, + { + name: postsTable, + isNew: true, + columns: { + add: ["id", "title", "content", "created_at"], + drop: [], + change: [] + }, + indices: { add: [], drop: [] } + } + ]); + + // now sync + await schema.sync({ force: true }); + const diffAfter = await schema.getDiff(); + + //console.log("diffAfter", diffAfter); + expect(diffAfter.length).toBe(0); + + await kysely.schema.dropTable(usersTable).execute(); + await kysely.schema.dropTable(postsTable).execute(); + }); + + test("adds index on create", async () => { + const entity = new Entity(randomString(16), [new TextField("email")]); + const index = new EntityIndex(entity, [entity.getField("email")!]); + const em = new EntityManager([entity], dummyConnection, [], [index]); + + const diff = await em.schema().getDiff(); + expect(diff).toEqual([ + { + name: entity.name, + isNew: true, + columns: { add: ["id", "email"], drop: [], change: [] }, + indices: { add: [index.name!], drop: [] } + } + ]); + + // sync and then check again + await em.schema().sync({ force: true }); + + const diffAfter = await em.schema().getDiff(); + expect(diffAfter.length).toBe(0); + }); + + test("adds index after", async () => { + const { dummyConnection } = getDummyConnection(); + + const entity = new Entity(randomString(16), [new TextField("email", { required: true })]); + const em = new EntityManager([entity], dummyConnection); + await em.schema().sync({ force: true }); + + // now add index + const index = new EntityIndex(entity, [entity.getField("email")!], true); + em.addIndex(index); + + const diff = await em.schema().getDiff(); + expect(diff).toEqual([ + { + name: entity.name, + isNew: false, + columns: { add: [], drop: [], change: [] }, + indices: { add: [index.name!], drop: [] } + } + ]); + + // sync and then check again + await em.schema().sync({ force: true }); + + const diffAfter = await em.schema().getDiff(); + expect(diffAfter.length).toBe(0); + }); +}); diff --git a/app/__test__/data/specs/WithBuilder.spec.ts b/app/__test__/data/specs/WithBuilder.spec.ts new file mode 100644 index 0000000..9141b62 --- /dev/null +++ b/app/__test__/data/specs/WithBuilder.spec.ts @@ -0,0 +1,195 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { + Entity, + EntityManager, + ManyToManyRelation, + ManyToOneRelation, + PolymorphicRelation, + TextField, + WithBuilder +} from "../../../src/data"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("[data] WithBuilder", async () => { + test("missing relation", async () => { + const users = new Entity("users", [new TextField("username")]); + const em = new EntityManager([users], dummyConnection); + + expect(() => + WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, ["posts"]) + ).toThrow('Relation "posts" not found'); + }); + + test("addClause: ManyToOne", async () => { + const users = new Entity("users", [new TextField("username")]); + const posts = new Entity("posts", [new TextField("content")]); + const relations = [new ManyToOneRelation(posts, users, { mappedBy: "author" })]; + const em = new EntityManager([users, posts], dummyConnection, relations); + + const qb = WithBuilder.addClause(em, em.connection.kysely.selectFrom("users"), users, [ + "posts" + ]); + + const res = qb.compile(); + + expect(res.sql).toBe( + 'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'content\', "agg"."content", \'author_id\', "agg"."author_id")), \'[]\') from (select "posts"."id" as "id", "posts"."content" as "content", "posts"."author_id" as "author_id" from "posts" where "users"."id" = "posts"."author_id" limit ?) as agg) as "posts" from "users"' + ); + expect(res.parameters).toEqual([5]); + + const qb2 = WithBuilder.addClause( + em, + em.connection.kysely.selectFrom("posts"), + posts, // @todo: try with "users", it gives output! + ["author"] + ); + + const res2 = qb2.compile(); + + expect(res2.sql).toBe( + 'select (select json_object(\'id\', "obj"."id", \'username\', "obj"."username") from (select "users"."id" as "id", "users"."username" as "username" from "users" where "posts"."author_id" = "users"."id" limit ?) as obj) as "author" from "posts"' + ); + expect(res2.parameters).toEqual([1]); + }); + + test("test with empty join", async () => { + const qb = { qb: 1 } as any; + + expect(WithBuilder.addClause(null as any, qb, null as any, [])).toBe(qb); + }); + + test("test manytomany", async () => { + const posts = new Entity("posts", [new TextField("title")]); + const categories = new Entity("categories", [new TextField("label")]); + + const entities = [posts, categories]; + const relations = [new ManyToManyRelation(posts, categories)]; + + const em = new EntityManager(entities, dummyConnection, relations); + await em.schema().sync({ force: true }); + + await em.mutator(posts).insertOne({ title: "fashion post" }); + await em.mutator(posts).insertOne({ title: "beauty post" }); + + await em.mutator(categories).insertOne({ label: "fashion" }); + await em.mutator(categories).insertOne({ label: "beauty" }); + await em.mutator(categories).insertOne({ label: "tech" }); + + await em.connection.kysely + .insertInto("posts_categories") + .values([ + { posts_id: 1, categories_id: 1 }, + { posts_id: 2, categories_id: 2 }, + { posts_id: 1, categories_id: 2 } + ]) + .execute(); + + //console.log((await em.repository().findMany("posts_categories")).result); + + const res = await em.repository(posts).findMany({ with: ["categories"] }); + + expect(res.data).toEqual([ + { + id: 1, + title: "fashion post", + categories: [ + { id: 1, label: "fashion" }, + { id: 2, label: "beauty" } + ] + }, + { + id: 2, + title: "beauty post", + categories: [{ id: 2, label: "beauty" }] + } + ]); + + const res2 = await em.repository(categories).findMany({ with: ["posts"] }); + + //console.log(res2.sql, res2.data); + + expect(res2.data).toEqual([ + { + id: 1, + label: "fashion", + posts: [{ id: 1, title: "fashion post" }] + }, + { + id: 2, + label: "beauty", + posts: [ + { id: 2, title: "beauty post" }, + { id: 1, title: "fashion post" } + ] + }, + { + id: 3, + label: "tech", + posts: [] + } + ]); + }); + + test("polymorphic", async () => { + const categories = new Entity("categories", [new TextField("name")]); + const media = new Entity("media", [new TextField("path")]); + + const entities = [media, categories]; + const single = new PolymorphicRelation(categories, media, { + mappedBy: "single", + targetCardinality: 1 + }); + const multiple = new PolymorphicRelation(categories, media, { mappedBy: "multiple" }); + + const em = new EntityManager(entities, dummyConnection, [single, multiple]); + + const qb = WithBuilder.addClause( + em, + em.connection.kysely.selectFrom("categories"), + categories, + ["single"] + ); + const res = qb.compile(); + expect(res.sql).toBe( + 'select (select json_object(\'id\', "obj"."id", \'path\', "obj"."path") from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" limit ?) as obj) as "single" from "categories"' + ); + expect(res.parameters).toEqual(["categories.single", 1]); + + const qb2 = WithBuilder.addClause( + em, + em.connection.kysely.selectFrom("categories"), + categories, + ["multiple"] + ); + const res2 = qb2.compile(); + expect(res2.sql).toBe( + 'select (select coalesce(json_group_array(json_object(\'id\', "agg"."id", \'path\', "agg"."path")), \'[]\') from (select "media"."id" as "id", "media"."path" as "path" from "media" where "media"."reference" = ? and "categories"."id" = "media"."entity_id" limit ?) as agg) as "multiple" from "categories"' + ); + expect(res2.parameters).toEqual(["categories.multiple", 5]); + }); + + /*test("test manytoone", async () => { + const posts = new Entity("posts", [new TextField("title")]); + const users = new Entity("users", [new TextField("username")]); + const relations = [ + new ManyToOneRelation(posts, users, { mappedBy: "author" }), + ]; + const em = new EntityManager([users, posts], dummyConnection, relations); + console.log((await em.schema().sync(true)).map((s) => s.sql).join("\n")); + await em.schema().sync(); + + await em.mutator().insertOne("users", { username: "user1" }); + await em.mutator().insertOne("users", { username: "user2" }); + + await em.mutator().insertOne("posts", { title: "post1", author_id: 1 }); + await em.mutator().insertOne("posts", { title: "post2", author_id: 2 }); + + console.log((await em.repository().findMany("posts")).result); + + const res = await em.repository().findMany("posts", { join: ["author"] }); + console.log(res.sql, res.parameters, res.result); + });*/ +}); diff --git a/app/__test__/data/specs/connection/Connection.spec.ts b/app/__test__/data/specs/connection/Connection.spec.ts new file mode 100644 index 0000000..cc73f7b --- /dev/null +++ b/app/__test__/data/specs/connection/Connection.spec.ts @@ -0,0 +1,92 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { EntityManager } from "../../../../src/data"; +import { getDummyConnection } from "../../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); +afterAll(afterAllCleanup); + +describe("Connection", async () => { + test("it introspects indices correctly", async () => { + const em = new EntityManager([], dummyConnection); + const kysely = em.connection.kysely; + + await kysely.schema.createTable("items").ifNotExists().addColumn("name", "text").execute(); + await kysely.schema.createIndex("idx_items_name").on("items").columns(["name"]).execute(); + + const indices = await em.connection.getIntrospector().getIndices("items"); + expect(indices).toEqual([ + { + name: "idx_items_name", + table: "items", + isUnique: false, + columns: [ + { + name: "name", + order: 0 + } + ] + } + ]); + }); + + test("it introspects indices on multiple columns correctly", async () => { + const em = new EntityManager([], dummyConnection); + const kysely = em.connection.kysely; + + await kysely.schema + .createTable("items_multiple") + .ifNotExists() + .addColumn("name", "text") + .addColumn("desc", "text") + .execute(); + await kysely.schema + .createIndex("idx_items_multiple") + .on("items_multiple") + .columns(["name", "desc"]) + .execute(); + + const indices = await em.connection.getIntrospector().getIndices("items_multiple"); + expect(indices).toEqual([ + { + name: "idx_items_multiple", + table: "items_multiple", + isUnique: false, + columns: [ + { + name: "name", + order: 0 + }, + { + name: "desc", + order: 1 + } + ] + } + ]); + }); + + test("it introspects unique indices correctly", async () => { + const em = new EntityManager([], dummyConnection); + const kysely = em.connection.kysely; + const tbl_name = "items_unique"; + const idx_name = "idx_items_unique"; + + await kysely.schema.createTable(tbl_name).ifNotExists().addColumn("name", "text").execute(); + await kysely.schema.createIndex(idx_name).on(tbl_name).columns(["name"]).unique().execute(); + + const indices = await em.connection.getIntrospector().getIndices(tbl_name); + expect(indices).toEqual([ + { + name: idx_name, + table: tbl_name, + isUnique: true, + columns: [ + { + name: "name", + order: 0 + } + ] + } + ]); + }); +}); diff --git a/app/__test__/data/specs/fields/BooleanField.spec.ts b/app/__test__/data/specs/fields/BooleanField.spec.ts new file mode 100644 index 0000000..7ed5036 --- /dev/null +++ b/app/__test__/data/specs/fields/BooleanField.spec.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from "bun:test"; +import { BooleanField } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +describe("[data] BooleanField", async () => { + runBaseFieldTests(BooleanField, { defaultValue: true, schemaType: "boolean" }); + + test("transformRetrieve", async () => { + const field = new BooleanField("test"); + expect(field.transformRetrieve(1)).toBe(true); + expect(field.transformRetrieve(0)).toBe(false); + expect(field.transformRetrieve("1")).toBe(true); + expect(field.transformRetrieve("0")).toBe(false); + expect(field.transformRetrieve(true)).toBe(true); + expect(field.transformRetrieve(false)).toBe(false); + expect(field.transformRetrieve(null)).toBe(null); + expect(field.transformRetrieve(undefined)).toBe(null); + }); + + test("transformPersist (specific)", async () => { + const field = new BooleanField("test"); + expect(transformPersist(field, 1)).resolves.toBe(true); + expect(transformPersist(field, 0)).resolves.toBe(false); + expect(transformPersist(field, "1")).rejects.toThrow(); + expect(transformPersist(field, "0")).rejects.toThrow(); + expect(transformPersist(field, true)).resolves.toBe(true); + expect(transformPersist(field, false)).resolves.toBe(false); + }); +}); diff --git a/app/__test__/data/specs/fields/DateField.spec.ts b/app/__test__/data/specs/fields/DateField.spec.ts new file mode 100644 index 0000000..3e29bf0 --- /dev/null +++ b/app/__test__/data/specs/fields/DateField.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from "bun:test"; +import { DateField } from "../../../../src/data"; +import { runBaseFieldTests } from "./inc"; + +describe("[data] DateField", async () => { + runBaseFieldTests(DateField, { defaultValue: new Date(), schemaType: "date" }); + + // @todo: add datefield tests + test("week", async () => { + const field = new DateField("test", { type: "week" }); + console.log(field.getValue("2021-W01", "submit")); + }); +}); diff --git a/app/__test__/data/specs/fields/EnumField.spec.ts b/app/__test__/data/specs/fields/EnumField.spec.ts new file mode 100644 index 0000000..d60f2e7 --- /dev/null +++ b/app/__test__/data/specs/fields/EnumField.spec.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "bun:test"; +import { EnumField } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +function options(strings: string[]) { + return { type: "strings", values: strings }; +} + +describe("[data] EnumField", async () => { + runBaseFieldTests( + EnumField, + { defaultValue: "a", schemaType: "text" }, + { options: options(["a", "b", "c"]) } + ); + + test("yields if no options", async () => { + expect(() => new EnumField("test", { options: options([]) })).toThrow(); + }); + + test("yields if default value is not a valid option", async () => { + expect( + () => new EnumField("test", { options: options(["a", "b"]), default_value: "c" }) + ).toThrow(); + }); + + test("transformPersist (config)", async () => { + const field = new EnumField("test", { options: options(["a", "b", "c"]) }); + + expect(transformPersist(field, null)).resolves.toBeUndefined(); + expect(transformPersist(field, "a")).resolves.toBe("a"); + expect(transformPersist(field, "d")).rejects.toThrow(); + }); + + test("transformRetrieve", async () => { + const field = new EnumField("test", { + options: options(["a", "b", "c"]), + default_value: "a", + required: true + }); + + expect(field.transformRetrieve(null)).toBe("a"); + expect(field.transformRetrieve("d")).toBe("a"); + }); +}); diff --git a/app/__test__/data/specs/fields/Field.spec.ts b/app/__test__/data/specs/fields/Field.spec.ts new file mode 100644 index 0000000..6fd8e04 --- /dev/null +++ b/app/__test__/data/specs/fields/Field.spec.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from "bun:test"; +import { Default, parse, stripMark } from "../../../../src/core/utils"; +import { Field, type SchemaResponse, TextField, baseFieldConfigSchema } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +describe("[data] Field", async () => { + class FieldSpec extends Field { + schema(): SchemaResponse { + return this.useSchemaHelper("text"); + } + getSchema() { + return baseFieldConfigSchema; + } + } + + runBaseFieldTests(FieldSpec, { defaultValue: "test", schemaType: "text" }); + + test.only("default config", async () => { + const field = new FieldSpec("test"); + const config = Default(baseFieldConfigSchema, {}); + expect(stripMark(new FieldSpec("test").config)).toEqual(config); + console.log("config", new TextField("test", { required: true }).toJSON()); + }); + + test("transformPersist (specific)", async () => { + const required = new FieldSpec("test", { required: true }); + const requiredDefault = new FieldSpec("test", { + required: true, + default_value: "test" + }); + + expect(required.transformPersist(null, undefined as any, undefined as any)).rejects.toThrow(); + expect( + required.transformPersist(undefined, undefined as any, undefined as any) + ).rejects.toThrow(); + + // works because it has a default value + expect( + requiredDefault.transformPersist(null, undefined as any, undefined as any) + ).resolves.toBeDefined(); + expect( + requiredDefault.transformPersist(undefined, undefined as any, undefined as any) + ).resolves.toBeDefined(); + }); +}); diff --git a/app/__test__/data/specs/fields/FieldIndex.spec.ts b/app/__test__/data/specs/fields/FieldIndex.spec.ts new file mode 100644 index 0000000..3ba9606 --- /dev/null +++ b/app/__test__/data/specs/fields/FieldIndex.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; +import { Type } from "../../../../src/core/utils"; +import { + Entity, + EntityIndex, + type EntityManager, + Field, + type SchemaResponse +} from "../../../../src/data"; + +class TestField extends Field { + protected getSchema(): any { + return Type.Any(); + } + + schema(em: EntityManager): SchemaResponse { + return undefined as any; + } +} + +describe("FieldIndex", async () => { + const entity = new Entity("test", []); + test("it constructs", async () => { + const field = new TestField("name"); + const index = new EntityIndex(entity, [field]); + + expect(index.fields).toEqual([field]); + expect(index.name).toEqual("idx_test_name"); + expect(index.unique).toEqual(false); + }); + + test("it fails on non-unique", async () => { + const field = new TestField("name", { required: false }); + + expect(() => new EntityIndex(entity, [field], true)).toThrowError(); + expect(() => new EntityIndex(entity, [field])).toBeDefined(); + }); +}); diff --git a/app/__test__/data/specs/fields/JsonField.spec.ts b/app/__test__/data/specs/fields/JsonField.spec.ts new file mode 100644 index 0000000..f13968a --- /dev/null +++ b/app/__test__/data/specs/fields/JsonField.spec.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { JsonField } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +describe("[data] JsonField", async () => { + const field = new JsonField("test"); + runBaseFieldTests(JsonField, { + defaultValue: { a: 1 }, + sampleValues: ["string", { test: 1 }, 1], + schemaType: "text" + }); + + test("transformPersist (no config)", async () => { + expect(transformPersist(field, Function)).rejects.toThrow(); + expect(transformPersist(field, undefined)).resolves.toBeUndefined(); + }); + + test("isSerializable", async () => { + expect(field.isSerializable(1)).toBe(true); + expect(field.isSerializable("test")).toBe(true); + expect(field.isSerializable({ test: 1 })).toBe(true); + expect(field.isSerializable({ test: [1, 2] })).toBe(true); + expect(field.isSerializable(Function)).toBe(false); + expect(field.isSerializable(undefined)).toBe(false); + }); + + test("isSerialized", async () => { + expect(field.isSerialized(1)).toBe(false); + expect(field.isSerialized({ test: 1 })).toBe(false); + expect(field.isSerialized('{"test":1}')).toBe(true); + expect(field.isSerialized("1")).toBe(true); + }); + + test("getValue", async () => { + expect(field.getValue({ test: 1 }, "form")).toBe('{"test":1}'); + expect(field.getValue("string", "form")).toBe('"string"'); + expect(field.getValue(1, "form")).toBe("1"); + + expect(field.getValue('{"test":1}', "submit")).toEqual({ test: 1 }); + expect(field.getValue('"string"', "submit")).toBe("string"); + expect(field.getValue("1", "submit")).toBe(1); + + expect(field.getValue({ test: 1 }, "table")).toBe('{"test":1}'); + expect(field.getValue("string", "table")).toBe('"string"'); + expect(field.getValue(1, "form")).toBe("1"); + }); +}); diff --git a/app/__test__/data/specs/fields/JsonSchemaField.spec.ts b/app/__test__/data/specs/fields/JsonSchemaField.spec.ts new file mode 100644 index 0000000..f9f2f54 --- /dev/null +++ b/app/__test__/data/specs/fields/JsonSchemaField.spec.ts @@ -0,0 +1,9 @@ +import { describe, expect, test } from "bun:test"; +import { JsonSchemaField } from "../../../../src/data"; +import { runBaseFieldTests } from "./inc"; + +describe("[data] JsonSchemaField", async () => { + runBaseFieldTests(JsonSchemaField, { defaultValue: {}, schemaType: "text" }); + + // @todo: add JsonSchemaField tests +}); diff --git a/app/__test__/data/specs/fields/NumberField.spec.ts b/app/__test__/data/specs/fields/NumberField.spec.ts new file mode 100644 index 0000000..6708449 --- /dev/null +++ b/app/__test__/data/specs/fields/NumberField.spec.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "bun:test"; +import { NumberField } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +describe("[data] NumberField", async () => { + test("transformPersist (config)", async () => { + const field = new NumberField("test", { minimum: 3, maximum: 5 }); + + expect(transformPersist(field, 2)).rejects.toThrow(); + expect(transformPersist(field, 6)).rejects.toThrow(); + expect(transformPersist(field, 4)).resolves.toBe(4); + + const field2 = new NumberField("test"); + expect(transformPersist(field2, 0)).resolves.toBe(0); + expect(transformPersist(field2, 10000)).resolves.toBe(10000); + }); + + runBaseFieldTests(NumberField, { defaultValue: 12, schemaType: "integer" }); +}); diff --git a/app/__test__/data/specs/fields/PrimaryField.spec.ts b/app/__test__/data/specs/fields/PrimaryField.spec.ts new file mode 100644 index 0000000..5d6dd54 --- /dev/null +++ b/app/__test__/data/specs/fields/PrimaryField.spec.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from "bun:test"; +import { PrimaryField } from "../../../../src/data"; + +describe("[data] PrimaryField", async () => { + const field = new PrimaryField("primary"); + + test("name", async () => { + expect(field.name).toBe("primary"); + }); + + test("schema", () => { + expect(field.name).toBe("primary"); + expect(field.schema()).toEqual(["primary", "integer", expect.any(Function)]); + }); + + test("hasDefault", async () => { + expect(field.hasDefault()).toBe(false); + expect(field.getDefault()).toBe(undefined); + }); + + test("isFillable", async () => { + expect(field.isFillable()).toBe(false); + }); + + test("isHidden", async () => { + expect(field.isHidden()).toBe(false); + }); + + test("isRequired", async () => { + expect(field.isRequired()).toBe(false); + }); + + test("transformPersist/Retrieve", async () => { + expect(field.transformPersist(1)).rejects.toThrow(); + expect(field.transformRetrieve(1)).toBe(1); + }); +}); diff --git a/app/__test__/data/specs/fields/TextField.spec.ts b/app/__test__/data/specs/fields/TextField.spec.ts new file mode 100644 index 0000000..fe83767 --- /dev/null +++ b/app/__test__/data/specs/fields/TextField.spec.ts @@ -0,0 +1,15 @@ +import { describe, expect, test } from "bun:test"; +import { TextField } from "../../../../src/data"; +import { runBaseFieldTests, transformPersist } from "./inc"; + +describe("[data] TextField", async () => { + test("transformPersist (config)", async () => { + const field = new TextField("test", { minLength: 3, maxLength: 5 }); + + expect(transformPersist(field, "a")).rejects.toThrow(); + expect(transformPersist(field, "abcdefghijklmn")).rejects.toThrow(); + expect(transformPersist(field, "abc")).resolves.toBe("abc"); + }); + + runBaseFieldTests(TextField, { defaultValue: "abc", schemaType: "text" }); +}); diff --git a/app/__test__/data/specs/fields/inc.ts b/app/__test__/data/specs/fields/inc.ts new file mode 100644 index 0000000..98b5e5f --- /dev/null +++ b/app/__test__/data/specs/fields/inc.ts @@ -0,0 +1,162 @@ +import { expect, test } from "bun:test"; +import type { ColumnDataType } from "kysely"; +import { omit } from "lodash-es"; +import type { BaseFieldConfig, Field, TActionContext } from "../../../../src/data"; + +type ConstructableField = new (name: string, config?: Partial) => Field; + +type FieldTestConfig = { + defaultValue: any; + sampleValues?: any[]; + schemaType: ColumnDataType; +}; + +export function transformPersist(field: Field, value: any, context?: TActionContext) { + return field.transformPersist(value, undefined as any, context as any); +} + +export function runBaseFieldTests( + fieldClass: ConstructableField, + config: FieldTestConfig, + _requiredConfig: any = {} +) { + const noConfigField = new fieldClass("no_config", _requiredConfig); + const fillable = new fieldClass("fillable", { ..._requiredConfig, fillable: true }); + const required = new fieldClass("required", { ..._requiredConfig, required: true }); + const hidden = new fieldClass("hidden", { ..._requiredConfig, hidden: true }); + const dflt = new fieldClass("dflt", { ..._requiredConfig, default_value: config.defaultValue }); + const requiredAndDefault = new fieldClass("full", { + ..._requiredConfig, + fillable: true, + required: true, + default_value: config.defaultValue + }); + + test("schema", () => { + expect(noConfigField.name).toBe("no_config"); + expect(noConfigField.schema(null as any)).toEqual([ + "no_config", + config.schemaType, + expect.any(Function) + ]); + }); + + test("hasDefault", async () => { + expect(noConfigField.hasDefault()).toBe(false); + expect(noConfigField.getDefault()).toBeUndefined(); + expect(dflt.hasDefault()).toBe(true); + expect(dflt.getDefault()).toBe(config.defaultValue); + }); + + test("isFillable", async () => { + expect(noConfigField.isFillable()).toBe(true); + expect(fillable.isFillable()).toBe(true); + expect(hidden.isFillable()).toBe(true); + expect(required.isFillable()).toBe(true); + }); + + test("isHidden", async () => { + expect(noConfigField.isHidden()).toBe(false); + expect(hidden.isHidden()).toBe(true); + expect(fillable.isHidden()).toBe(false); + expect(required.isHidden()).toBe(false); + }); + + test("isRequired", async () => { + expect(noConfigField.isRequired()).toBe(false); + expect(required.isRequired()).toBe(true); + expect(hidden.isRequired()).toBe(false); + expect(fillable.isRequired()).toBe(false); + }); + + test.if(Array.isArray(config.sampleValues))("getValue (RenderContext)", async () => { + const isPrimitive = (v) => ["string", "number"].includes(typeof v); + for (const value of config.sampleValues!) { + // "form" + expect(isPrimitive(noConfigField.getValue(value, "form"))).toBeTrue(); + // "table" + expect(isPrimitive(noConfigField.getValue(value, "table"))).toBeTrue(); + // "read" + // "submit" + } + }); + + test("transformPersist", async () => { + const persist = await transformPersist(noConfigField, config.defaultValue); + expect(config.defaultValue).toEqual(noConfigField.transformRetrieve(config.defaultValue)); + expect(transformPersist(noConfigField, null)).resolves.toBeUndefined(); + expect(transformPersist(noConfigField, undefined)).resolves.toBeUndefined(); + expect(transformPersist(requiredAndDefault, null)).resolves.toBe(persist); + expect(transformPersist(dflt, null)).resolves.toBe(persist); + }); + + test("toJSON", async () => { + const _config = { + ..._requiredConfig, + //order: 1, + fillable: true, + required: false, + hidden: false + //virtual: false, + //default_value: undefined + }; + + function fieldJson(field: Field) { + const json = field.toJSON(); + return { + ...json, + config: omit(json.config, ["html"]) + }; + } + + expect(fieldJson(noConfigField)).toEqual({ + //name: "no_config", + type: noConfigField.type, + config: _config + }); + + expect(fieldJson(fillable)).toEqual({ + //name: "fillable", + type: noConfigField.type, + config: _config + }); + + expect(fieldJson(required)).toEqual({ + //name: "required", + type: required.type, + config: { + ..._config, + required: true + } + }); + + expect(fieldJson(hidden)).toEqual({ + //name: "hidden", + type: required.type, + config: { + ..._config, + hidden: true + } + }); + + expect(fieldJson(dflt)).toEqual({ + //name: "dflt", + type: dflt.type, + config: { + ..._config, + default_value: config.defaultValue + } + }); + + expect(fieldJson(requiredAndDefault)).toEqual({ + //name: "full", + type: requiredAndDefault.type, + config: { + ..._config, + fillable: true, + required: true, + default_value: config.defaultValue + } + }); + }); +} diff --git a/app/__test__/data/specs/relations/EntityRelation.spec.ts b/app/__test__/data/specs/relations/EntityRelation.spec.ts new file mode 100644 index 0000000..989b4f9 --- /dev/null +++ b/app/__test__/data/specs/relations/EntityRelation.spec.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, test } from "bun:test"; +import { Entity, type EntityManager } from "../../../../src/data"; +import { + type BaseRelationConfig, + EntityRelation, + EntityRelationAnchor, + RelationTypes +} from "../../../../src/data/relations"; + +class TestEntityRelation extends EntityRelation { + constructor(config?: BaseRelationConfig) { + super( + new EntityRelationAnchor(new Entity("source"), "source"), + new EntityRelationAnchor(new Entity("target"), "target"), + config + ); + } + initialize(em: EntityManager) {} + type() { + return RelationTypes.ManyToOne; /* doesn't matter */ + } + setDirections(directions: ("source" | "target")[]) { + this.directions = directions; + return this; + } + + buildWith(a: any, b: any, c: any): any { + return; + } + + buildJoin(a: any, b: any): any { + return; + } +} + +describe("[data] EntityRelation", async () => { + test("other", async () => { + const relation = new TestEntityRelation(); + expect(relation.other("source").entity.name).toBe("target"); + expect(relation.other("target").entity.name).toBe("source"); + }); + + it("visibleFrom", async () => { + const relation = new TestEntityRelation(); + // by default, both sides are visible + expect(relation.visibleFrom("source")).toBe(true); + expect(relation.visibleFrom("target")).toBe(true); + + // make source invisible + relation.setDirections(["target"]); + expect(relation.visibleFrom("source")).toBe(false); + expect(relation.visibleFrom("target")).toBe(true); + + // make target invisible + relation.setDirections(["source"]); + expect(relation.visibleFrom("source")).toBe(true); + expect(relation.visibleFrom("target")).toBe(false); + }); + + it("hydrate", async () => { + // @todo: implement + }); + + it("isListableFor", async () => { + // by default, the relation is listable from target side + const relation = new TestEntityRelation(); + expect(relation.isListableFor(relation.target.entity)).toBe(true); + expect(relation.isListableFor(relation.source.entity)).toBe(false); + }); + + it("required", async () => { + const relation1 = new TestEntityRelation(); + expect(relation1.config.required).toBe(false); + + const relation2 = new TestEntityRelation({ required: true }); + expect(relation2.config.required).toBe(true); + }); +}); diff --git a/app/__test__/flows/FetchTask.spec.ts b/app/__test__/flows/FetchTask.spec.ts new file mode 100644 index 0000000..fe8e731 --- /dev/null +++ b/app/__test__/flows/FetchTask.spec.ts @@ -0,0 +1,114 @@ +import { afterAll, beforeAll, describe, expect, jest, test } from "bun:test"; +import { FetchTask, Flow } from "../../src/flows"; + +let _oldFetch: typeof fetch; +function mockFetch(responseMethods: Partial) { + _oldFetch = global.fetch; + // @ts-ignore + global.fetch = jest.fn(() => Promise.resolve(responseMethods)); +} + +function mockFetch2(newFetch: (input: RequestInfo, init: RequestInit) => Promise) { + _oldFetch = global.fetch; + // @ts-ignore + global.fetch = jest.fn(newFetch); +} + +function unmockFetch() { + global.fetch = _oldFetch; +} + +beforeAll(() => + /*mockFetch({ + ok: true, + status: 200, + json: () => Promise.resolve({ todos: [1, 2] }) + })*/ + mockFetch2(async (input, init) => { + const request = { + url: String(input), + method: init?.method ?? "GET", + // @ts-ignore + headers: Object.fromEntries(init?.headers?.entries() ?? []), + body: init?.body + }; + + return new Response(JSON.stringify({ todos: [1, 2], request }), { + status: 200, + headers: { "Content-Type": "application/json" } + }); + }) +); +afterAll(unmockFetch); + +describe("FetchTask", async () => { + test("Simple fetch", async () => { + const task = new FetchTask("Fetch Something", { + url: "https://jsonplaceholder.typicode.com/todos/1", + method: "GET", + headers: [{ key: "Content-Type", value: "application/json" }] + }); + + const result = await task.run(); + //console.log("result", result); + expect(result.output!.todos).toEqual([1, 2]); + expect(result.error).toBeUndefined(); + expect(result.success).toBe(true); + }); + + test("verify config", async () => { + // // @ts-expect-error + expect(() => new FetchTask("", {})).toThrow(); + + expect( + // // @ts-expect-error + () => new FetchTask("", { url: "https://jsonplaceholder.typicode.com", method: 1 }) + ).toThrow(); + + expect( + new FetchTask("", { + url: "https://jsonplaceholder.typicode.com", + method: "invalid" + }).execute() + ).rejects.toThrow(/^Invalid method/); + + expect( + () => new FetchTask("", { url: "https://jsonplaceholder.typicode.com", method: "GET" }) + ).toBeDefined(); + + expect(() => new FetchTask("", { url: "", method: "Invalid" })).toThrow(); + }); + + test("template", async () => { + const task = new FetchTask("fetch", { + url: "https://example.com/?email={{ flow.output.email }}", + method: "{{ flow.output.method }}", + headers: [ + { key: "Content-{{ flow.output.headerKey }}", value: "application/json" }, + { key: "Authorization", value: "Bearer {{ flow.output.apiKey }}" } + ], + body: JSON.stringify({ + email: "{{ flow.output.email }}" + }) + }); + const inputs = { + headerKey: "Type", + apiKey: 123, + email: "what@else.com", + method: "PATCH" + }; + + const flow = new Flow("", [task]); + const exec = await flow.start(inputs); + console.log("errors", exec.getErrors()); + expect(exec.hasErrors()).toBe(false); + + const { request } = exec.getResponse(); + + expect(request.url).toBe(`https://example.com/?email=${inputs.email}`); + expect(request.method).toBe(inputs.method); + expect(request.headers["content-type"]).toBe("application/json"); + expect(request.headers.authorization).toBe(`Bearer ${inputs.apiKey}`); + expect(request.body).toBe(JSON.stringify({ email: inputs.email })); + }); +}); diff --git a/app/__test__/flows/SubWorkflowTask.spec.ts b/app/__test__/flows/SubWorkflowTask.spec.ts new file mode 100644 index 0000000..c05faf7 --- /dev/null +++ b/app/__test__/flows/SubWorkflowTask.spec.ts @@ -0,0 +1,91 @@ +import { describe, expect, test } from "bun:test"; +import { Flow, LogTask, RenderTask, SubFlowTask } from "../../src/flows"; + +describe("SubFlowTask", async () => { + test("Simple Subflow", async () => { + const subTask = new RenderTask("render", { + render: "subflow" + }); + const subflow = new Flow("subflow", [subTask]); + + const task = new LogTask("log"); + const task2 = new SubFlowTask("sub", { + flow: subflow + }); + const task3 = new RenderTask("render2", { + render: "Subflow output: {{ sub.output }}" + }); + + const flow = new Flow("test", [task, task2, task3], []); + flow.task(task).asInputFor(task2); + flow.task(task2).asInputFor(task3); + + const execution = flow.createExecution(); + await execution.start(); + /*console.log(execution.logs); + console.log(execution.getResponse());*/ + + expect(execution.getResponse()).toEqual("Subflow output: subflow"); + }); + + test("Simple loop", async () => { + const subTask = new RenderTask("render", { + render: "run {{ flow.output }}" + }); + const subflow = new Flow("subflow", [subTask]); + + const task = new LogTask("log"); + const task2 = new SubFlowTask("sub", { + flow: subflow, + loop: true, + input: [1, 2, 3] + }); + const task3 = new RenderTask("render2", { + render: `Subflow output: {{ sub.output | join: ", " }}` + }); + + const flow = new Flow("test", [task, task2, task3], []); + flow.task(task).asInputFor(task2); + flow.task(task2).asInputFor(task3); + + const execution = flow.createExecution(); + await execution.start(); + + console.log("errors", execution.getErrors()); + + /*console.log(execution.logs); + console.log(execution.getResponse());*/ + + expect(execution.getResponse()).toEqual("Subflow output: run 1, run 2, run 3"); + }); + + test("Simple loop from flow input", async () => { + const subTask = new RenderTask("render", { + render: "run {{ flow.output }}" + }); + + const subflow = new Flow("subflow", [subTask]); + + const task = new LogTask("log"); + const task2 = new SubFlowTask("sub", { + flow: subflow, + loop: true, + input: "{{ flow.output | json }}" + }); + const task3 = new RenderTask("render2", { + render: `Subflow output: {{ sub.output | join: ", " }}` + }); + + const flow = new Flow("test", [task, task2, task3], []); + flow.task(task).asInputFor(task2); + flow.task(task2).asInputFor(task3); + + const execution = flow.createExecution(); + await execution.start([4, 5, 6]); + + /*console.log(execution.logs); + console.log(execution.getResponse());*/ + + expect(execution.getResponse()).toEqual("Subflow output: run 4, run 5, run 6"); + }); +}); diff --git a/app/__test__/flows/Task.spec.ts b/app/__test__/flows/Task.spec.ts new file mode 100644 index 0000000..8519288 --- /dev/null +++ b/app/__test__/flows/Task.spec.ts @@ -0,0 +1,112 @@ +import { describe, expect, test } from "bun:test"; +import { Type } from "../../src/core/utils"; +import { Task } from "../../src/flows"; +import { dynamic } from "../../src/flows/tasks/Task"; + +describe("Task", async () => { + test("resolveParams: template with parse", async () => { + const result = await Task.resolveParams( + Type.Object({ test: dynamic(Type.Number()) }), + { + test: "{{ some.path }}" + }, + { + some: { + path: 1 + } + } + ); + + expect(result.test).toBe(1); + }); + + test("resolveParams: with string", async () => { + const result = await Task.resolveParams( + Type.Object({ test: Type.String() }), + { + test: "{{ some.path }}" + }, + { + some: { + path: "1/1" + } + } + ); + + expect(result.test).toBe("1/1"); + }); + + test("resolveParams: with object", async () => { + const result = await Task.resolveParams( + Type.Object({ test: dynamic(Type.Object({ key: Type.String(), value: Type.String() })) }), + { + test: { key: "path", value: "{{ some.path }}" } + }, + { + some: { + path: "1/1" + } + } + ); + + expect(result.test).toEqual({ key: "path", value: "1/1" }); + }); + + test("resolveParams: with json", async () => { + const result = await Task.resolveParams( + Type.Object({ + test: dynamic(Type.Object({ key: Type.String(), value: Type.String() })) + }), + { + test: "{{ some | json }}" + }, + { + some: { + key: "path", + value: "1/1" + } + } + ); + + expect(result.test).toEqual({ key: "path", value: "1/1" }); + }); + + test("resolveParams: with array", async () => { + const result = await Task.resolveParams( + Type.Object({ + test: dynamic(Type.Array(Type.String())) + }), + { + test: '{{ "1,2,3" | split: "," | json }}' + } + ); + + expect(result.test).toEqual(["1", "2", "3"]); + }); + + test("resolveParams: boolean", async () => { + const result = await Task.resolveParams( + Type.Object({ + test: dynamic(Type.Boolean()) + }), + { + test: "{{ true }}" + } + ); + + expect(result.test).toEqual(true); + }); + + test("resolveParams: float", async () => { + const result = await Task.resolveParams( + Type.Object({ + test: dynamic(Type.Number(), Number.parseFloat) + }), + { + test: "{{ 3.14 }}" + } + ); + + expect(result.test).toEqual(3.14); + }); +}); diff --git a/app/__test__/flows/inc/back.ts b/app/__test__/flows/inc/back.ts new file mode 100644 index 0000000..36c9d50 --- /dev/null +++ b/app/__test__/flows/inc/back.ts @@ -0,0 +1,24 @@ +import { Condition, Flow } from "../../../src/flows"; +import { getNamedTask } from "./helper"; + +const first = getNamedTask("first"); +const second = getNamedTask("second"); +const fourth = getNamedTask("fourth"); + +let thirdRuns = 0; +const third = getNamedTask("third", async () => { + thirdRuns++; + if (thirdRuns === 3) { + return true; + } + + throw new Error("Third failed"); +}); + +const back = new Flow("back", [first, second, third, fourth]); +back.task(first).asInputFor(second); +back.task(second).asInputFor(third); +back.task(third).asInputFor(second, Condition.error(), 2); +back.task(third).asInputFor(fourth, Condition.success()); + +export { back }; diff --git a/app/__test__/flows/inc/fanout-condition.ts b/app/__test__/flows/inc/fanout-condition.ts new file mode 100644 index 0000000..c2ec3f5 --- /dev/null +++ b/app/__test__/flows/inc/fanout-condition.ts @@ -0,0 +1,23 @@ +import { Condition, Flow } from "../../../src/flows"; +import { getNamedTask } from "./helper"; + +const first = getNamedTask( + "first", + async () => { + //throw new Error("Error"); + return { + inner: { + result: 2 + } + }; + }, + 1000 +); +const second = getNamedTask("second (if match)"); +const third = getNamedTask("third (if error)"); + +const fanout = new Flow("fanout", [first, second, third]); +fanout.task(first).asInputFor(third, Condition.error(), 2); +fanout.task(first).asInputFor(second, Condition.matches("inner.result", 2)); + +export { fanout }; diff --git a/app/__test__/flows/inc/helper.tsx b/app/__test__/flows/inc/helper.tsx new file mode 100644 index 0000000..2893e3a --- /dev/null +++ b/app/__test__/flows/inc/helper.tsx @@ -0,0 +1,61 @@ +import { Task } from "../../../src/flows"; + +// @todo: polyfill +const Handle = (props: any) => null; +type NodeProps = any; +const Position = { Top: "top", Bottom: "bottom" }; + +class ExecTask extends Task { + type = "exec"; + + constructor( + name: string, + params: any, + private fn: () => any + ) { + super(name, params); + } + + override clone(name: string, params: any) { + return new ExecTask(name, params, this.fn); + } + + async execute() { + //console.log("executing", this.name); + return await this.fn(); + } +} + +/*const ExecNode = ({ + data, + isConnectable, + targetPosition = Position.Top, + sourcePosition = Position.Bottom, + selected, +}: NodeProps) => { + //console.log("data", data, data.hasDelay()); + return ( + <> + + {data?.name} ({selected ? "selected" : "exec"}) + + + ); +};*/ + +export function getNamedTask(name: string, _func?: () => Promise, delay?: number) { + const func = + _func ?? + (async () => { + //console.log(`[DONE] Task: ${name}`); + return true; + }); + + return new ExecTask( + name, + { + delay + }, + func + ); +} diff --git a/app/__test__/flows/inc/parallel.ts b/app/__test__/flows/inc/parallel.ts new file mode 100644 index 0000000..b053507 --- /dev/null +++ b/app/__test__/flows/inc/parallel.ts @@ -0,0 +1,15 @@ +import { Flow } from "../../../src/flows"; +import { getNamedTask } from "./helper"; + +const first = getNamedTask("first"); +const second = getNamedTask("second", undefined, 1000); +const third = getNamedTask("third"); +const fourth = getNamedTask("fourth"); +const fifth = getNamedTask("fifth"); // without connection + +const parallel = new Flow("Parallel", [first, second, third, fourth, fifth]); +parallel.task(first).asInputFor(second); +parallel.task(first).asInputFor(third); +parallel.task(third).asInputFor(fourth); + +export { parallel }; diff --git a/app/__test__/flows/inc/simple-fetch.ts b/app/__test__/flows/inc/simple-fetch.ts new file mode 100644 index 0000000..b5d21c3 --- /dev/null +++ b/app/__test__/flows/inc/simple-fetch.ts @@ -0,0 +1,18 @@ +import { FetchTask, Flow, LogTask } from "../../../src/flows"; + +const first = new LogTask("First", { delay: 1000 }); +const second = new LogTask("Second", { delay: 1000 }); +const third = new LogTask("Long Third", { delay: 2500 }); +const fourth = new FetchTask("Fetch Something", { + url: "https://jsonplaceholder.typicode.com/todos/1" +}); +const fifth = new LogTask("Task 4", { delay: 500 }); // without connection + +const simpleFetch = new Flow("simpleFetch", [first, second, third, fourth, fifth]); +simpleFetch.task(first).asInputFor(second); +simpleFetch.task(first).asInputFor(third); +simpleFetch.task(fourth).asOutputFor(third); + +simpleFetch.setRespondingTask(fourth); + +export { simpleFetch }; diff --git a/app/__test__/flows/inputs.test.ts b/app/__test__/flows/inputs.test.ts new file mode 100644 index 0000000..22d69f8 --- /dev/null +++ b/app/__test__/flows/inputs.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, test } from "bun:test"; +import { Hono } from "hono"; +import { Event, EventManager } from "../../src/core/events"; +import { type Static, type StaticDecode, Type, parse } from "../../src/core/utils"; +import { EventTrigger, Flow, HttpTrigger, type InputsMap, Task } from "../../src/flows"; +import { dynamic } from "../../src/flows/tasks/Task"; + +class Passthrough extends Task { + type = "passthrough"; + + async execute(inputs: Map) { + //console.log("executing passthrough", this.name, inputs); + return Array.from(inputs.values()).pop().output + "/" + this.name; + } +} + +type OutputIn = Static; +type OutputOut = StaticDecode; + +class OutputParamTask extends Task { + type = "output-param"; + + static override schema = Type.Object({ + number: dynamic( + Type.Number({ + title: "Output number" + }), + Number.parseInt + ) + }); + + async execute(inputs: InputsMap) { + //console.log("--***--- executing output", this.params); + return this.params.number; + } +} + +class PassthroughFlowInput extends Task { + type = "passthrough-flow-input"; + + async execute(inputs: InputsMap) { + return inputs.get("flow")?.output; + } +} + +describe("Flow task inputs", async () => { + test("types", async () => { + const schema = OutputParamTask.schema; + + expect(parse(schema, { number: 123 })).toBeDefined(); + expect(parse(schema, { number: "{{ some.path }}" })).toBeDefined(); + + const task = new OutputParamTask("", { number: 123 }); + expect(task.params.number).toBe(123); + }); + + test("passthrough", async () => { + const task = new Passthrough("log"); + const task2 = new Passthrough("log_2"); + + const flow = new Flow("test", [task, task2]); + flow.task(task).asInputFor(task2); + flow.setRespondingTask(task2); + + const exec = await flow.start("pass-through"); + + /*console.log( + "---- log", + exec.logs.map(({ task, ...l }) => ({ ...l, ...task.toJSON() })), + ); + console.log("---- result", exec.getResponse());*/ + expect(exec.getResponse()).toBe("pass-through/log/log_2"); + }); + + test("output/input", async () => { + const task = new OutputParamTask("task1", { number: 111 }); + const task2 = new OutputParamTask("task2", { + number: "{{ task1.output }}" + }); + + const flow = new Flow("test", [task, task2]); + flow.task(task).asInputFor(task2); + flow.setRespondingTask(task2); + + const exec = await flow.start(); + + /*console.log( + "---- log", + exec.logs.map(({ task, ...l }) => ({ ...l, ...task.toJSON() })), + ); + console.log("---- result", exec.getResponse());*/ + expect(exec.getResponse()).toBe(111); + }); + + test("input from flow", async () => { + const task = new OutputParamTask("task1", { + number: "{{flow.output.someFancyParam}}" + }); + const task2 = new OutputParamTask("task2", { + number: "{{task1.output}}" + }); + + const flow = new Flow("test", [task, task2]); + flow.task(task).asInputFor(task2); + flow.setRespondingTask(task2); + + // expect to throw because of missing input + //expect(flow.start()).rejects.toThrow(); + + const exec = await flow.start({ someFancyParam: 123 }); + + /*console.log( + "---- log", + exec.logs.map(({ task, ...l }) => ({ ...l, ...task.toJSON() })), + ); + console.log("---- result", exec.getResponse());*/ + + expect(exec.getResponse()).toBe(123); + }); + + test("manual event trigger with inputs", async () => { + class EventTriggerClass extends Event<{ number: number }> { + static override slug = "test-event"; + } + + const emgr = new EventManager({ EventTriggerClass }); + + const task = new OutputParamTask("event", { + number: "{{flow.output.number}}" + }); + const flow = new Flow( + "test", + [task], + [], + new EventTrigger({ + event: "test-event", + mode: "sync" + }) + ); + flow.setRespondingTask(task); + flow.trigger.register(flow, emgr); + + await emgr.emit(new EventTriggerClass({ number: 120 })); + const execs = flow.trigger.executions; + expect(execs.length).toBe(1); + expect(execs[0]!.getResponse()).toBe(120); + }); + + test("http trigger with response", async () => { + const task = new PassthroughFlowInput(""); + const flow = new Flow( + "test", + [task], + [], + new HttpTrigger({ + path: "/test", + method: "GET", + mode: "sync" + }) + ); + flow.setRespondingTask(task); + + const hono = new Hono(); + + flow.trigger.register(flow, hono); + + const res = await hono.request("/test?input=123"); + const data = await res.json(); + //console.log("response", data); + const execs = flow.trigger.executions; + expect(execs.length).toBe(1); + expect(execs[0]!.getResponse()).toBeInstanceOf(Request); + expect(execs[0]!.getResponse()?.url).toBe("http://localhost/test?input=123"); + }); +}); diff --git a/app/__test__/flows/render.tsx b/app/__test__/flows/render.tsx new file mode 100644 index 0000000..8080b5c --- /dev/null +++ b/app/__test__/flows/render.tsx @@ -0,0 +1,186 @@ +import { Box, Text, render, useApp, useInput } from "ink"; +import React, { useEffect } from "react"; +import { ExecutionEvent, type Flow, type Task } from "../../src/flows"; +import { back } from "./inc/back"; +import { fanout } from "./inc/fanout-condition"; +import { parallel } from "./inc/parallel"; +import { simpleFetch } from "./inc/simple-fetch"; + +const flows = { + back, + fanout, + parallel, + simpleFetch +}; + +const arg = process.argv[2]; +if (!arg) { + console.log("Please provide a flow name:", Object.keys(flows).join(", ")); + process.exit(1); +} +if (!flows[arg]) { + console.log("Flow not found:", arg, Object.keys(flows).join(", ")); + process.exit(1); +} + +console.log(JSON.stringify(flows[arg].toJSON(), null, 2)); +process.exit(); + +const colors = [ + "#B5E61D", // Lime Green + "#4A90E2", // Bright Blue + "#F78F1E", // Saffron + "#BD10E0", // Vivid Purple + "#50E3C2", // Turquoise + "#9013FE" // Grape +]; + +const colorsCache: Record = {}; + +type Sequence = { source: string; target: string }[]; +type Layout = Task[][]; +type State = { layout: Layout; connections: Sequence }; +type TaskWithStatus = { task: Task; status: string }; + +function TerminalFlow({ flow }: { flow: Flow }) { + const [tasks, setTasks] = React.useState([]); + const sequence = flow.getSequence(); + const connections = flow.connections; + + const { exit } = useApp(); + useInput((input, key) => { + if (input === "q") { + exit(); + return; + } + + if (key.return) { + // Left arrow key pressed + console.log("Enter pressed"); + } else { + console.log(input); + } + }); + + useEffect(() => { + setTasks(flow.tasks.map((t) => ({ task: t, status: "pending" }))); + + const execution = flow.createExecution(); + execution.subscribe((event) => { + if (event instanceof ExecutionEvent) { + setTasks((prev) => + prev.map((t) => { + if (t.task.name === event.task().name) { + let newStatus = "pending"; + if (event.isStart()) { + newStatus = "running"; + } else { + newStatus = event.succeeded() ? "success" : "failure"; + } + + return { task: t.task, status: newStatus }; + } + + return t; + }) + ); + } + }); + + execution.start().then(() => { + const response = execution.getResponse(); + console.log("done", response ? response : "(no response)"); + console.log( + "Executed tasks:", + execution.logs.map((l) => l.task.name) + ); + console.log("Executed count:", execution.logs.length); + }); + }, []); + + function getColor(key: string) { + if (!colorsCache[key]) { + colorsCache[key] = colors[Object.keys(colorsCache).length]; + } + return colorsCache[key]; + } + + if (tasks.length === 0) { + return Loading...; + } + + return ( + + {sequence.map((step, stepIndex) => { + return ( + + {step.map((_task, index) => { + const find = tasks.find((t) => t.task.name === _task.name)!; + + if (!find) { + //console.log("couldnt find", _task.name); + return null; + } + const { task, status } = find; + + const inTasks = flow.task(task).getInTasks(); + + return ( + + {inTasks.length > 0 && ( + + In: + + {inTasks.map((inTask, i) => ( + + {i > 0 ? ", " : ""} + {inTask.name} + + ))} + + + )} + + + {task.name} + + + + + ); + })} + + ); + })} + + ); +} + +const Status = ({ status }: { status: string }) => { + let color: string | undefined; + switch (status) { + case "running": + color = "orange"; + break; + case "success": + color = "green"; + break; + case "failure": + color = "red"; + break; + } + + return ( + + {status} + + ); +}; + +render(); diff --git a/app/__test__/flows/trigger.test.ts b/app/__test__/flows/trigger.test.ts new file mode 100644 index 0000000..1fec803 --- /dev/null +++ b/app/__test__/flows/trigger.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, test } from "bun:test"; +import { Hono } from "hono"; +import { Event, EventManager } from "../../src/core/events"; +import { EventTrigger, Flow, HttpTrigger, Task } from "../../src/flows"; + +const ALL_TESTS = !!process.env.ALL_TESTS; + +class ExecTask extends Task { + type = "exec"; + + constructor( + name: string, + params: any, + private fn: () => any + ) { + super(name, params); + } + + static create(name: string, fn: () => any) { + return new ExecTask(name, undefined, fn); + } + + override clone(name: string, params: any) { + return new ExecTask(name, params, this.fn); + } + + async execute() { + //console.log("executing", this.name); + return await this.fn(); + } +} + +describe("Flow trigger", async () => { + test("manual trigger", async () => { + let called = false; + + const task = ExecTask.create("manual", () => { + called = true; + }); + const flow = new Flow("", [task]); + + expect(flow.trigger.type).toBe("manual"); + + await flow.trigger.register(flow); + expect(called).toBe(true); + }); + + test("event trigger", async () => { + class EventTriggerClass extends Event { + static override slug = "test-event"; + } + + const emgr = new EventManager({ EventTriggerClass }); + let called = false; + + const task = ExecTask.create("event", () => { + called = true; + }); + const flow = new Flow( + "test", + [task], + [], + new EventTrigger({ event: "test-event", mode: "sync" }) + ); + + flow.trigger.register(flow, emgr); + + await emgr.emit(new EventTriggerClass({ test: 1 })); + expect(called).toBe(true); + }); + + /*test("event trigger with match", async () => { + class EventTriggerClass extends Event<{ number: number }> { + static slug = "test-event"; + } + + const emgr = new EventManager({ EventTriggerClass }); + let called: number = 0; + + const task = ExecTask.create("event", () => { + called++; + }); + const flow = new Flow( + "test", + [task], + [], + new EventTrigger(EventTriggerClass, "sync", (e) => e.params.number === 2) + ); + + flow.trigger.register(flow, emgr); + + await emgr.emit(new EventTriggerClass({ number: 1 })); + await emgr.emit(new EventTriggerClass({ number: 2 })); + expect(called).toBe(1); + });*/ + + test("http trigger", async () => { + let called = false; + + const task = ExecTask.create("http", () => { + called = true; + }); + const flow = new Flow( + "test", + [task], + [], + new HttpTrigger({ + path: "/test", + method: "GET", + mode: "sync" + }) + ); + + const hono = new Hono(); + + flow.trigger.register(flow, hono); + + const res = await hono.request("/test"); + //const data = await res.json(); + //console.log("response", data); + expect(called).toBe(true); + }); + + test("http trigger with response", async () => { + const task = ExecTask.create("http", () => ({ + called: true + })); + const flow = new Flow( + "test", + [task], + [], + new HttpTrigger({ + path: "/test", + method: "GET", + mode: "sync" + }) + ); + flow.setRespondingTask(task); + + const hono = new Hono(); + + flow.trigger.register(flow, hono); + + const res = await hono.request("/test"); + const data = await res.json(); + //console.log("response", data); + expect(data).toEqual({ called: true }); + }); + + /*test.skipIf(ALL_TESTS)("template with email", async () => { + console.log("apikey", process.env.RESEND_API_KEY); + const task = new FetchTask("fetch", { + url: "https://api.resend.com/emails", + method: "POST", + headers: [ + { key: "Content-Type", value: "application/json" }, + { key: "Authorization", value: "Bearer {{ flow.output.apiKey }}" } + ], + body: JSON.stringify({ + from: "onboarding@resend.dev", + to: "dennis.senn@gmail.com", + subject: + "test from {% if flow.output.someFancyParam > 100 %}flow{% else %}task{% endif %}!", + html: "Hello" + }) + }); + + const flow = new Flow("test", [task]); + + const exec = await flow.start({ someFancyParam: 80, apiKey: process.env.RESEND_API_KEY }); + //console.log("exec", exec.logs, exec.finished()); + expect(exec.finished()).toBe(true); + expect(exec.hasErrors()).toBe(false); + });*/ +}); diff --git a/app/__test__/flows/workflow-basic.test.ts b/app/__test__/flows/workflow-basic.test.ts new file mode 100644 index 0000000..f199762 --- /dev/null +++ b/app/__test__/flows/workflow-basic.test.ts @@ -0,0 +1,449 @@ +// eslint-disable-next-line import/no-unresolved +import { describe, expect, test } from "bun:test"; +import { isEqual } from "lodash-es"; +import { type Static, Type, _jsonp } from "../../src/core/utils"; +import { Condition, ExecutionEvent, FetchTask, Flow, LogTask, Task } from "../../src/flows"; + +/*beforeAll(disableConsoleLog); +afterAll(enableConsoleLog);*/ + +class ExecTask extends Task { + type = "exec"; + + static override schema = Type.Object({ + delay: Type.Number({ default: 10 }) + }); + + constructor( + name: string, + params: Static, + private func: () => Promise + ) { + super(name, params); + } + + override clone(name: string, params: Static) { + return new ExecTask(name, params, this.func); + } + + async execute() { + await new Promise((resolve) => setTimeout(resolve, this.params.delay ?? 0)); + return await this.func(); + } +} + +function getTask(num: number = 0, delay: number = 5) { + return new ExecTask( + `Task ${num}`, + { + delay + }, + async () => { + //console.log(`[DONE] Task: ${num}`); + return true; + } + ); + //return new LogTask(`Log ${num}`, { delay }); +} + +function getNamedTask(name: string, _func?: () => Promise, delay?: number) { + const func = + _func ?? + (async () => { + //console.log(`[DONE] Task: ${name}`); + return true; + }); + return new ExecTask( + name, + { + delay: delay ?? 0 + }, + func + ); +} + +function getObjectDiff(obj1, obj2) { + const diff = Object.keys(obj1).reduce((result, key) => { + // biome-ignore lint/suspicious/noPrototypeBuiltins: + if (!obj2.hasOwnProperty(key)) { + result.push(key); + } else if (isEqual(obj1[key], obj2[key])) { + const resultKeyIndex = result.indexOf(key); + result.splice(resultKeyIndex, 1); + } + return result; + }, Object.keys(obj2)); + + return diff; +} + +describe("Flow tests", async () => { + test("Simple single task", async () => { + const simple = getTask(0); + + const result = await simple.run(); + expect(result.success).toBe(true); + // @todo: add more + }); + + function getNamedQueue(flow: Flow) { + const namedSequence = flow.getSequence().map((step) => step.map((t) => t.name)); + //console.log(namedSequence); + return namedSequence; + } + + test("Simple flow", async () => { + const first = getTask(0); + const second = getTask(1); + + // simple + const simple = new Flow("simple", [first, second]); + simple.task(first).asInputFor(second); + expect(getNamedQueue(simple)).toEqual([["Task 0"], ["Task 1"]]); + expect(simple.task(first).getDepth()).toBe(0); + expect(simple.task(second).getDepth()).toBe(1); + + const execution = simple.createExecution(); + await execution.start(); + + //console.log("execution", execution.logs); + //process.exit(0); + expect(execution.logs.length).toBe(2); + expect(execution.logs.every((log) => log.success)).toBe(true); + }); + + test("Test connection uniqueness", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2, 5); + const fourth = getTask(3); + + // should be fine + expect(() => { + const condition = new Flow("", [first, second, third]); + condition.task(first).asInputFor(second); + condition.task(first).asInputFor(third); + }).toBeDefined(); + + // should throw + expect(() => { + const condition = new Flow("", [first, second, third]); + condition.task(first).asInputFor(second); + condition.task(first).asInputFor(second); + }).toThrow(); + + expect(() => { + const condition = new Flow("", [first, second, third]); + condition.task(first).asInputFor(second); + condition.task(second).asInputFor(third); + condition.task(third).asInputFor(second); + condition.task(third).asInputFor(fourth); // this should fail + }).toThrow(); + + expect(() => { + const condition = new Flow("", [first, second, third]); + condition.task(first).asInputFor(second); + condition.task(second).asInputFor(third); + condition.task(third).asInputFor(second); + condition.task(third).asInputFor(fourth, Condition.error()); + }).toBeDefined(); + }); + + test("Flow with 3 steps", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2); + + const three = new Flow("", [first, second, third]); + three.task(first).asInputFor(second); + three.task(second).asInputFor(third); + expect(getNamedQueue(three)).toEqual([["Task 0"], ["Task 1"], ["Task 2"]]); + expect(three.task(first).getDepth()).toBe(0); + expect(three.task(second).getDepth()).toBe(1); + expect(three.task(third).getDepth()).toBe(2); + + const execution = three.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(3); + expect(execution.logs.every((log) => log.success)).toBe(true); + }); + + test("Flow with parallel tasks", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2); + const fourth = getTask(3); + const fifth = getTask(4); // without connection + + const parallel = new Flow("", [first, second, third, fourth, fifth]); + parallel.task(first).asInputFor(second); + parallel.task(first).asInputFor(third); + parallel.task(third).asInputFor(fourth); + expect(getNamedQueue(parallel)).toEqual([["Task 0"], ["Task 1", "Task 2"], ["Task 3"]]); + expect(parallel.task(first).getDepth()).toBe(0); + expect(parallel.task(second).getDepth()).toBe(1); + expect(parallel.task(third).getDepth()).toBe(1); + expect(parallel.task(fourth).getDepth()).toBe(2); + + const execution = parallel.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(4); + expect(execution.logs.every((log) => log.success)).toBe(true); + }); + + test("Flow with condition", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2); + + const condition = new Flow("", [first, second, third]); + condition.task(first).asInputFor(second); + condition.task(first).asInputFor(third); + }); + + test("Flow with back step", async () => { + const first = getNamedTask("first"); + const second = getNamedTask("second"); + const fourth = getNamedTask("fourth"); + + let thirdRuns: number = 0; + const third = getNamedTask("third", async () => { + thirdRuns++; + if (thirdRuns === 4) { + return true; + } + + throw new Error("Third failed"); + }); + + const back = new Flow("", [first, second, third, fourth]); + back.task(first).asInputFor(second); + back.task(second).asInputFor(third); + back.task(third).asInputFor(second, Condition.error(), 2); + back.task(third).asInputFor(fourth, Condition.success()); + expect(getNamedQueue(back)).toEqual([["first"], ["second"], ["third"], ["fourth"]]); + expect( + back + .task(third) + .getOutTasks() + .map((t) => t.name) + ).toEqual(["second", "fourth"]); + + const execution = back.createExecution(); + expect(execution.start()).rejects.toThrow(); + }); + + test("Flow with back step: enough retries", async () => { + const first = getNamedTask("first"); + const second = getNamedTask("second"); + const fourth = getNamedTask("fourth"); + + let thirdRuns: number = 0; + const third = getNamedTask("third", async () => { + thirdRuns++; + //console.log("--- third runs", thirdRuns); + if (thirdRuns === 2) { + return true; + } + + throw new Error("Third failed"); + }); + + const back = new Flow("", [first, second, third, fourth]); + back.task(first).asInputFor(second); + back.task(second).asInputFor(third); + back.task(third).asInputFor(second, Condition.error(), 1); + back.task(third).asInputFor(fourth, Condition.success()); + expect(getNamedQueue(back)).toEqual([["first"], ["second"], ["third"], ["fourth"]]); + expect( + back + .task(third) + .getOutTasks() + .map((t) => t.name) + ).toEqual(["second", "fourth"]); + + const execution = back.createExecution(); + await execution.start(); + }); + + test("flow fanout", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2, 20); + + const fanout = new Flow("", [first, second, third]); + fanout.task(first).asInputFor(second); + fanout.task(first).asInputFor(third); + + const execution = fanout.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(3); + expect(execution.logs.every((log) => log.success)).toBe(true); + }); + + test("flow fanout with condition", async () => { + const first = getTask(0); + const second = getTask(1); + const third = getTask(2); + + const fanout = new Flow("", [first, second, third]); + fanout.task(first).asInputFor(second, Condition.success()); + fanout.task(first).asInputFor(third, Condition.error()); + + const execution = fanout.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(2); + expect(execution.logs.every((log) => log.success)).toBe(true); + }); + + test("flow fanout with condition error", async () => { + const first = getNamedTask("first", async () => { + throw new Error("Error"); + }); + const second = getNamedTask("second"); + const third = getNamedTask("third"); + + const fanout = new Flow("", [first, second, third]); + fanout.task(first).asInputFor(third, Condition.error()); + fanout.task(first).asInputFor(second, Condition.success()); + + const execution = fanout.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(2); + expect(execution.logs.map((l) => l.task.name)).toEqual(["first", "third"]); + }); + + test("flow fanout with condition matches", async () => { + const first = getNamedTask("first", async () => { + return { + inner: { + result: 2 + } + }; + }); + const second = getNamedTask("second"); + const third = getNamedTask("third"); + + const fanout = new Flow("", [first, second, third]); + fanout.task(first).asInputFor(third, Condition.error()); + fanout.task(first).asInputFor(second, Condition.matches("inner.result", 2)); + + const execution = fanout.createExecution(); + await execution.start(); + + expect(execution.logs.length).toBe(2); + expect(execution.logs.map((l) => l.task.name)).toEqual(["first", "second"]); + }); + + test("flow: responding task", async () => { + const first = getNamedTask("first"); + const second = getNamedTask("second", async () => ({ result: 2 })); + const third = getNamedTask("third"); + + const flow = new Flow("", [first, second, third]); + flow.task(first).asInputFor(second); + flow.task(second).asInputFor(third); + + flow.setRespondingTask(second); + + const execution = flow.createExecution(); + + execution.subscribe(async (event) => { + if (event instanceof ExecutionEvent) { + console.log( + "[event]", + event.isStart() ? "start" : "end", + event.task().name, + event.isStart() ? undefined : event.succeeded() + ); + } + }); + + await execution.start(); + + const response = execution.getResponse(); + + expect(response).toEqual({ result: 2 }); + expect(execution.logs.length).toBe(2); + expect(execution.logs.map((l) => l.task.name)).toEqual(["first", "second"]); + + /*console.log("response", response); + console.log("execution.logs.length", execution.logs.length); + console.log( + "executed", + execution.logs.map((l) => l.task.name), + );*/ + /*expect(execution.logs.length).toBe(3); + expect(execution.logs.every((log) => log.success)).toBe(true);*/ + }); + + test("serialize/deserialize", async () => { + const first = new LogTask("Task 0"); + const second = new LogTask("Task 1"); + const third = new LogTask("Task 2", { delay: 50 }); + const fourth = new FetchTask("Fetch Something", { + url: "https://jsonplaceholder.typicode.com/todos/1" + }); + const fifth = new LogTask("Task 4"); // without connection + + const flow = new Flow("", [first, second, third, fourth, fifth]); + flow.task(first).asInputFor(second); + flow.task(first).asInputFor(third); + flow.task(fourth).asOutputFor(third, Condition.matches("some", 1)); + + flow.setRespondingTask(fourth); + + const original = flow.toJSON(); + //console.log("flow", original); + // @todo: fix + const deserialized = Flow.fromObject("", original, { + fetch: { cls: FetchTask }, + log: { cls: LogTask } + } as any); + + const diffdeep = getObjectDiff(original, deserialized.toJSON()); + expect(diffdeep).toEqual([]); + + expect(flow.startTask.name).toEqual(deserialized.startTask.name); + expect(flow.respondingTask?.name).toEqual( + // @ts-ignore + deserialized.respondingTask?.name + ); + + //console.log("--- creating original sequence"); + const originalSequence = flow.getSequence(); + //console.log("--- creating deserialized sequence"); + const deserializedSequence = deserialized.getSequence(); + //console.log("--- "); + + expect(originalSequence).toEqual(deserializedSequence); + }); + + test("error end", async () => { + const first = getNamedTask("first", async () => "first"); + const second = getNamedTask("error", async () => { + throw new Error("error"); + }); + const third = getNamedTask("third", async () => "third"); + const errorhandlertask = getNamedTask("errorhandler", async () => "errorhandler"); + + const flow = new Flow("", [first, second, third, errorhandlertask]); + flow.task(first).asInputFor(second); + flow.task(second).asInputFor(third); + flow.task(second).asInputFor(errorhandlertask, Condition.error()); + + const exec = await flow.start(); + + //console.log("logs", JSON.stringify(exec.logs, null, 2)); + //console.log("errors", exec.hasErrors(), exec.errorCount()); + + expect(exec.hasErrors()).toBe(true); + expect(exec.errorCount()).toBe(1); + expect(exec.getResponse()).toBe("errorhandler"); + }); +}); diff --git a/app/__test__/helper.ts b/app/__test__/helper.ts new file mode 100644 index 0000000..f93f4e6 --- /dev/null +++ b/app/__test__/helper.ts @@ -0,0 +1,53 @@ +import { unlink } from "node:fs/promises"; +import type { SqliteDatabase } from "kysely"; +import Database from "libsql"; +import { SqliteLocalConnection } from "../src/data"; + +export function getDummyDatabase(memory: boolean = true): { + dummyDb: SqliteDatabase; + afterAllCleanup: () => Promise; +} { + const DB_NAME = memory ? ":memory:" : `${Math.random().toString(36).substring(7)}.db`; + const dummyDb = new Database(DB_NAME); + + return { + dummyDb, + afterAllCleanup: async () => { + if (!memory) await unlink(DB_NAME); + return true; + } + }; +} + +export function getDummyConnection(memory: boolean = true) { + const { dummyDb, afterAllCleanup } = getDummyDatabase(memory); + const dummyConnection = new SqliteLocalConnection(dummyDb); + + return { + dummyConnection, + afterAllCleanup + }; +} + +export function getLocalLibsqlConnection() { + return { url: "http://127.0.0.1:8080" }; +} + +type ConsoleSeverity = "log" | "warn" | "error"; +const _oldConsoles = { + log: console.log, + warn: console.warn, + error: console.error +}; + +export function disableConsoleLog(severities: ConsoleSeverity[] = ["log"]) { + severities.forEach((severity) => { + console[severity] = () => null; + }); +} + +export function enableConsoleLog() { + Object.entries(_oldConsoles).forEach(([severity, fn]) => { + console[severity as ConsoleSeverity] = fn; + }); +} diff --git a/app/__test__/media/MediaController.spec.ts b/app/__test__/media/MediaController.spec.ts new file mode 100644 index 0000000..8c8b9a9 --- /dev/null +++ b/app/__test__/media/MediaController.spec.ts @@ -0,0 +1,56 @@ +import { describe, test } from "bun:test"; +import { Hono } from "hono"; +import { Guard } from "../../src/auth"; +import { EventManager } from "../../src/core/events"; +import { EntityManager } from "../../src/data"; +import { AppMedia } from "../../src/media/AppMedia"; +import { MediaController } from "../../src/media/api/MediaController"; +import { getDummyConnection } from "../helper"; + +const { dummyConnection, afterAllCleanup } = getDummyConnection(); + +/** + * R2 + * value: ReadableStream | ArrayBuffer | ArrayBufferView | string | Blob | null, + * Node writefile + * data: string | NodeJS.ArrayBufferView | Iterable | AsyncIterable | Stream, + */ + +describe("MediaController", () => { + test("..", async () => { + const ctx: any = { + em: new EntityManager([], dummyConnection, []), + guard: new Guard(), + emgr: new EventManager(), + server: new Hono() + }; + + const media = new AppMedia( + // @ts-ignore + { + enabled: true, + adapter: { + type: "s3", + config: { + access_key: process.env.R2_ACCESS_KEY as string, + secret_access_key: process.env.R2_SECRET_ACCESS_KEY as string, + url: process.env.R2_URL as string + } + } + }, + ctx + ); + await media.build(); + const app = new MediaController(media).getController(); + + const file = Bun.file(`${import.meta.dir}/adapters/icon.png`); + console.log("file", file); + const form = new FormData(); + form.append("file", file); + + await app.request("/upload/test.png", { + method: "POST", + body: file + }); + }); +}); diff --git a/app/__test__/media/Storage.spec.ts b/app/__test__/media/Storage.spec.ts new file mode 100644 index 0000000..69d1f0e --- /dev/null +++ b/app/__test__/media/Storage.spec.ts @@ -0,0 +1,81 @@ +import { describe, expect, test } from "bun:test"; +import { type FileBody, Storage, type StorageAdapter } from "../../src/media/storage/Storage"; +import * as StorageEvents from "../../src/media/storage/events"; + +class TestAdapter implements StorageAdapter { + files: Record = {}; + + getName() { + return "test"; + } + + getSchema() { + return undefined; + } + + async listObjects(prefix?: string) { + return []; + } + + async putObject(key: string, body: FileBody) { + this.files[key] = body; + return "etag-string"; + } + + async deleteObject(key: string) { + delete this.files[key]; + } + + async objectExists(key: string) { + return key in this.files; + } + + async getObject(key: string) { + return new Response(this.files[key]); + } + + getObjectUrl(key: string) { + return key; + } + + async getObjectMeta(key: string) { + return { type: "text/plain", size: 0 }; + } + + toJSON(secrets?: boolean): any { + return { name: this.getName() }; + } +} + +describe("Storage", async () => { + const adapter = new TestAdapter(); + const storage = new Storage(adapter); + const events = new Map(); + + storage.emgr.onAny((event) => { + // @ts-ignore + events.set(event.constructor.slug, event); + //console.log("event", event.constructor.slug, event); + }); + + test("uploads a file", async () => { + const { + meta: { type, size } + } = await storage.uploadFile("hello", "world.txt"); + expect({ type, size }).toEqual({ type: "text/plain", size: 0 }); + }); + + test("deletes the file", async () => { + expect(await storage.deleteFile("hello")).toBeUndefined(); + expect(await storage.fileExists("hello")).toBeFalse(); + }); + + test("events were fired", async () => { + expect(events.has(StorageEvents.FileUploadedEvent.slug)).toBeTrue(); + expect(events.has(StorageEvents.FileDeletedEvent.slug)).toBeTrue(); + // @todo: file access must be tested in controllers + //expect(events.has(StorageEvents.FileAccessEvent.slug)).toBeTrue(); + }); + + // @todo: test controllers +}); diff --git a/app/__test__/media/StorageR2Adapter.native-spec.ts b/app/__test__/media/StorageR2Adapter.native-spec.ts new file mode 100644 index 0000000..8f33ca6 --- /dev/null +++ b/app/__test__/media/StorageR2Adapter.native-spec.ts @@ -0,0 +1,34 @@ +import * as assert from "node:assert/strict"; +import { createWriteStream } from "node:fs"; +import { test } from "node:test"; +import { Miniflare } from "miniflare"; + +// https://github.com/nodejs/node/issues/44372#issuecomment-1736530480 +console.log = async (message: any) => { + const tty = createWriteStream("/dev/tty"); + const msg = typeof message === "string" ? message : JSON.stringify(message, null, 2); + return tty.write(`${msg}\n`); +}; + +test("what", async () => { + const mf = new Miniflare({ + modules: true, + script: "export default { async fetch() { return new Response(null); } }", + r2Buckets: ["BUCKET"] + }); + + const bucket = await mf.getR2Bucket("BUCKET"); + console.log(await bucket.put("count", "1")); + + const object = await bucket.get("count"); + if (object) { + /*const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set("etag", object.httpEtag);*/ + console.log("yo -->", await object.text()); + + assert.strictEqual(await object.text(), "1"); + } + + await mf.dispose(); +}); diff --git a/app/__test__/media/adapters/StorageCloudinaryAdapter.spec.ts b/app/__test__/media/adapters/StorageCloudinaryAdapter.spec.ts new file mode 100644 index 0000000..1294275 --- /dev/null +++ b/app/__test__/media/adapters/StorageCloudinaryAdapter.spec.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { randomString } from "../../../src/core/utils"; +import { StorageCloudinaryAdapter } from "../../../src/media"; + +import { config } from "dotenv"; +const dotenvOutput = config({ path: `${import.meta.dir}/../../.env` }); +const { + CLOUDINARY_CLOUD_NAME, + CLOUDINARY_API_KEY, + CLOUDINARY_API_SECRET, + CLOUDINARY_UPLOAD_PRESET +} = dotenvOutput.parsed!; + +const ALL_TESTS = !!process.env.ALL_TESTS; + +describe.skipIf(ALL_TESTS)("StorageCloudinaryAdapter", () => { + const adapter = new StorageCloudinaryAdapter({ + cloud_name: CLOUDINARY_CLOUD_NAME as string, + api_key: CLOUDINARY_API_KEY as string, + api_secret: CLOUDINARY_API_SECRET as string, + upload_preset: CLOUDINARY_UPLOAD_PRESET as string + }); + + const file = Bun.file(`${import.meta.dir}/icon.png`); + const _filename = randomString(10); + const filename = `${_filename}.png`; + + test("object exists", async () => { + expect(await adapter.objectExists("7fCTBi6L8c.png")).toBeTrue(); + process.exit(); + }); + + test("puts object", async () => { + expect(await adapter.objectExists(filename)).toBeFalse(); + + const result = await adapter.putObject(filename, file); + console.log("result", result); + expect(result).toBeDefined(); + expect(result?.name).toBe(filename); + }); + + test("object exists", async () => { + await Bun.sleep(10000); + const one = await adapter.objectExists(_filename); + const two = await adapter.objectExists(filename); + expect(await adapter.objectExists(filename)).toBeTrue(); + }); + + test("object meta", async () => { + const result = await adapter.getObjectMeta(filename); + console.log("objectMeta:result", result); + expect(result).toBeDefined(); + expect(result.type).toBe("image/png"); + expect(result.size).toBeGreaterThan(0); + }); + + test("list objects", async () => { + const result = await adapter.listObjects(); + console.log("listObjects:result", result); + }); +}); diff --git a/app/__test__/media/adapters/StorageLocalAdapter.spec.ts b/app/__test__/media/adapters/StorageLocalAdapter.spec.ts new file mode 100644 index 0000000..29746d1 --- /dev/null +++ b/app/__test__/media/adapters/StorageLocalAdapter.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { randomString } from "../../../src/core/utils"; +import { StorageLocalAdapter } from "../../../src/media/storage/adapters/StorageLocalAdapter"; + +describe("StorageLocalAdapter", () => { + const adapter = new StorageLocalAdapter({ + path: `${import.meta.dir}/local` + }); + + const file = Bun.file(`${import.meta.dir}/icon.png`); + const _filename = randomString(10); + const filename = `${_filename}.png`; + + let objects = 0; + + test("puts an object", async () => { + objects = (await adapter.listObjects()).length; + expect(await adapter.putObject(filename, await file.arrayBuffer())).toBeString(); + }); + + test("lists objects", async () => { + expect((await adapter.listObjects()).length).toBe(objects + 1); + }); + + test("file exists", async () => { + expect(await adapter.objectExists(filename)).toBeTrue(); + }); + + test("gets an object", async () => { + const res = await adapter.getObject(filename, new Headers()); + expect(res.ok).toBeTrue(); + // @todo: check the content + }); + + test("gets object meta", async () => { + expect(await adapter.getObjectMeta(filename)).toEqual({ + type: file.type, // image/png + size: file.size + }); + }); + + test("deletes an object", async () => { + expect(await adapter.deleteObject(filename)).toBeUndefined(); + expect(await adapter.objectExists(filename)).toBeFalse(); + }); +}); diff --git a/app/__test__/media/adapters/StorageS3Adapter.spec.ts b/app/__test__/media/adapters/StorageS3Adapter.spec.ts new file mode 100644 index 0000000..d6274dc --- /dev/null +++ b/app/__test__/media/adapters/StorageS3Adapter.spec.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; +import { randomString } from "../../../src/core/utils"; +import { StorageS3Adapter } from "../../../src/media"; + +import { config } from "dotenv"; +const dotenvOutput = config({ path: `${import.meta.dir}/../../.env` }); +const { R2_ACCESS_KEY, R2_SECRET_ACCESS_KEY, R2_URL, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_S3_URL } = + dotenvOutput.parsed!; + +// @todo: mock r2/s3 responses for faster tests +const ALL_TESTS = process.env.ALL_TESTS; + +describe("Storage", async () => { + console.log("ALL_TESTS", process.env.ALL_TESTS); + const versions = [ + [ + "r2", + new StorageS3Adapter({ + access_key: R2_ACCESS_KEY as string, + secret_access_key: R2_SECRET_ACCESS_KEY as string, + url: R2_URL as string + }) + ], + [ + "s3", + new StorageS3Adapter({ + access_key: AWS_ACCESS_KEY as string, + secret_access_key: AWS_SECRET_KEY as string, + url: AWS_S3_URL as string + }) + ] + ] as const; + + const _conf = { + adapters: ["r2", "s3"], + tests: [ + "listObjects", + "putObject", + "objectExists", + "getObject", + "deleteObject", + "getObjectMeta" + ] + }; + + const file = Bun.file(`${import.meta.dir}/icon.png`); + const filename = `${randomString(10)}.png`; + + // single (dev) + //_conf = { adapters: [/*"r2",*/ "s3"], tests: [/*"putObject",*/ "listObjects"] }; + + function disabled(test: (typeof _conf.tests)[number]) { + return !_conf.tests.includes(test); + } + + // @todo: add mocked fetch for faster tests + describe.each(versions)("StorageS3Adapter for %s", async (name, adapter) => { + if (!_conf.adapters.includes(name)) { + console.log("Skipping", name); + return; + } + + let objects = 0; + + test.skipIf(disabled("putObject"))("puts an object", async () => { + objects = (await adapter.listObjects()).length; + expect(await adapter.putObject(filename, file)).toBeString(); + }); + + test.skipIf(disabled("listObjects"))("lists objects", async () => { + expect((await adapter.listObjects()).length).toBe(objects + 1); + }); + + test.skipIf(disabled("objectExists"))("file exists", async () => { + expect(await adapter.objectExists(filename)).toBeTrue(); + }); + + test.skipIf(disabled("getObject"))("gets an object", async () => { + const res = await adapter.getObject(filename, new Headers()); + expect(res.ok).toBeTrue(); + // @todo: check the content + }); + + test.skipIf(disabled("getObjectMeta"))("gets object meta", async () => { + expect(await adapter.getObjectMeta(filename)).toEqual({ + type: file.type, // image/png + size: file.size + }); + }); + + test.skipIf(disabled("deleteObject"))("deletes an object", async () => { + expect(await adapter.deleteObject(filename)).toBeUndefined(); + expect(await adapter.objectExists(filename)).toBeFalse(); + }); + }); +}); diff --git a/app/__test__/media/adapters/icon.png b/app/__test__/media/adapters/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..fb107d8ddd59ab4558b7c9273f028e79f72f3581 GIT binary patch literal 2789 zcmVUJ_!rkg}VYEOJ>AjSC_Fd9A1K)KU)+&41AK{)Jx@ zsQ9KNxse!u5MrFDHqb1DLgP0(#zy}rsBo4S-$P)?S6s$nF9$xrPOhb65a5hXs&xSO7VP1(0)C z06C{bjxsy;0o}Ry9^F%4AAE6xYSS}R=&Vrj(e1RR=LxDD=%dok$BBuD*zx|TB!|Fa z->jYp7JfB4_U)k!M~?)eV4~ra3%GrLgueXMkx+{d?p*w@di*QF^O$Hj;{vdVH>=0V zl-l&I`RA?O-$#rYPFMh#{i(q>C=OGDL&57XW;h{$$-&_u$csbW5}Y8$jMN2i|D&sP zSNU=qAa)2FKQMJt7r^a**$v?Cs;wgvp40_k`-+`q3t&X*0(kJn1leTH^6U9d>H)ritdDYW zsuyK1|KK_&fTG}3K?p?f$lJpzFx(UB@#O}8f9uRY_#h|(&Xt5|a7u`SqIyl>L|_%* z=V9g%G-CIFuiqT2&Yvb0ySCA`w}we@GAK083Lq>H#>BHP%)5_0aDJ-Y%kyyy$S{=YG9xH)i~Bp(We!kfmBP09-BTmp6k-<>>7%<6a}0Cok@@TxV8 zU4c2?^#-ylFvB|muq!ab%kE%y1>y0wJ)B)ZIJ^}AyMl0JJ{7zgVb2QSBJN&#k7lo2 z4bC3!fAn#XUT$nzE2-zB6&;6AIEN_jdMxv;>D~)qrLzJo`po%}pqnVFb$}4o9_Y(# zXR=iRpetwQ{HXeRndTmz&*N$-?|Pg%_U^G>{}&d_5%^YF0?phL0 zU|lNvp9u<6vV;`@gq`Yv4{7V4PNZTs8(l#w!Oyt<AfzVhqe9t(XXAvoKj zL{Lt+coQPMaUi*?6Ru{xvIef#CdAqW;6pW8d;njJ=Oj7I0sq!FN2`=1*mX?&`iS#G zD`E-YE5pUtBst6h3xM~D3ZtY!{^6^H#coPu0U)r7Q;3vSE`rDcz@9lS zx7CiJRbY#k_Tp>s9^ispOh-o+!2K)nP3k?hU2GLd{kl{t4j1HNIy$-lEcnz`Ef7Y| zAqaB~7t_({&j?bBW!pb~ck(afDz2SOM(zwkUVP5N6|9Fm>RSJH^)*g|=d@aUm%D`U z-fo>a>rBQlvH;ff>{bG}K(^4kf+?zJ(;x_>kJt44)F_!UlYz0Za0TlQ9dsf|R8(op zDn&qaR$|ZS#UTnZgUy4Yq{=NI#<(+bk*5FrH+8n~VQ`aMObcJBix}_6_Sm^rAZKT( zxMef7{p81#HZd$fy{|uNP*dvKPMi8)3w{p)XbXVFp1Zyx%TF1AX!*^ z4e}@zWd+hE)>!}+4%59&yLoXHqX~4mc5W=TKB8S|w*iX)SA^IBvw^Vh&k|klo=x2i zSU3#2`tUp;oCr)0gS2(P%_DH!?9&)H48}s#@I)YjX4BmaOgDb$ID;G@2Kf4= zTt(J!p^??ffCwz;ws%ekehn#NWC^wD#&)qynd!CrpSCg!sBxx{whp?DJZn5~V5+G% zC1DfK{>DmPB1sU?Zhz;mlqTL(1!ryXfwWYP76-mj#usBj5!_&ph8&# zL15Z=BY@J*A5qpQInY()j{q__CA93oTw`wpfbOHLfuruU@kRjcd(&ws^l*#)BBNkX6-0fuPq)@AeZe(5nD6vK_ zKx3COTmcG3lNbnL0xEz|O+9nw?1GcPy6bv=L9akkLpL%j01T#>8XR8q0o4aHyhu|0 z_jO8b;Y)$Q?~!0YRVAndLibX06|aiR zwi>**Ex}c~F!+NwF}tYw{o!ECbnZgE(t60)5@bpM@Q2?H9AD7-g&!RC^__=)8C+e> zq%DHw7QYb`xlW%v9s4~({P?#_$bVhp&KQ`$|x&`LAYXUrd@Hfha?x@6*~6EcL5!Sl~Gm%V0$pb zJxm12A&rf%Q9z*FZfL=*2*7q{?re_l?198ql%f`W9$@@DvViS2=Q#)3rqcqk2o};r z1bXqK6q;BOfbAwnpQeCB0lOZI(1A%`4Y&+EHtJ5v$}Rv3w7F>9=a<~j4QmrZYziQY zT|TV>+tOBUO|7^9+x7Tu8LIEkI#vW=yB@#dNX>j6ux0^lSpk^4k^@bXdBIv)c+~xC zBl=X3Ne*iQKp+@*@z$&7yVNw12dq{E>2{~7!6Bnv0j_*3ekG%K-^?=BJA)tsBqqY< zq;(>L7e5_ZS`>=Jg)X*0x)@dgS3AtSE~Lm(OSg}BfAeT&Wq3|t&S3%M92P*%VFBbE z7C_En0puJOK+Y+U5aYz;mCFSoW$SL?%{Yyewa*Gt(j{VYi71G3g>tEMN)SyFlP`oE zzt+=tsxbcK&nE?mmx#$51NbG_1$^H7)Hx-B*NMpmLe>Uw^#{Li{EHv_Z?q}}^%LVU z^7U)IeXr{8MDt(o7#lgPzVwq6-NbkU;=CHwUb^1<%(do%Tmfq< { + moduleTestSuite(AppAuth); + + let ctx: ModuleBuildContext; + + beforeEach(() => { + ctx = makeCtx(); + }); + + test("secrets", async () => { + // auth must be enabled, otherwise default config is returned + const auth = new AppAuth({ enabled: true }, ctx); + await auth.build(); + + const config = auth.toJSON(); + expect(config.jwt).toBeUndefined(); + expect(config.strategies.password.config).toBeUndefined(); + }); + + test("creates user on register", async () => { + const auth = new AppAuth( + { + enabled: true + }, + ctx + ); + + await auth.build(); + await ctx.em.schema().sync({ force: true }); + + // expect no users, but the query to pass + const res = await ctx.em.repository("users").findMany(); + expect(res.data.length).toBe(0); + + const app = new AuthController(auth).getController(); + + { + disableConsoleLog(); + const res = await app.request("/password/register", { + method: "POST", + body: JSON.stringify({ + email: "some@body.com", + password: "123456" + }) + }); + enableConsoleLog(); + expect(res.status).toBe(200); + + const { data: users } = await ctx.em.repository("users").findMany(); + expect(users.length).toBe(1); + expect(users[0].email).toBe("some@body.com"); + } + }); +}); diff --git a/app/__test__/modules/AppData.spec.ts b/app/__test__/modules/AppData.spec.ts new file mode 100644 index 0000000..19461b3 --- /dev/null +++ b/app/__test__/modules/AppData.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from "bun:test"; +import { parse } from "../../src/core/utils"; +import { fieldsSchema } from "../../src/data/data-schema"; +import { AppData } from "../../src/modules"; +import { moduleTestSuite } from "./module-test-suite"; + +describe("AppData", () => { + moduleTestSuite(AppData); + + test("field config construction", () => { + expect(parse(fieldsSchema, { type: "text" })).toBeDefined(); + }); +}); diff --git a/app/__test__/modules/AppMedia.spec.ts b/app/__test__/modules/AppMedia.spec.ts new file mode 100644 index 0000000..6f1b0f5 --- /dev/null +++ b/app/__test__/modules/AppMedia.spec.ts @@ -0,0 +1,7 @@ +import { describe } from "bun:test"; +import { AppMedia } from "../../src/modules"; +import { moduleTestSuite } from "./module-test-suite"; + +describe("AppMedia", () => { + moduleTestSuite(AppMedia); +}); diff --git a/app/__test__/modules/module-test-suite.ts b/app/__test__/modules/module-test-suite.ts new file mode 100644 index 0000000..5c1cce7 --- /dev/null +++ b/app/__test__/modules/module-test-suite.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it } from "bun:test"; + +import { Hono } from "hono"; +import { Guard } from "../../src/auth"; +import { EventManager } from "../../src/core/events"; +import { Default, stripMark } from "../../src/core/utils"; +import { EntityManager } from "../../src/data"; +import type { Module, ModuleBuildContext } from "../../src/modules/Module"; +import { getDummyConnection } from "../helper"; + +export function makeCtx(overrides?: Partial): ModuleBuildContext { + const { dummyConnection } = getDummyConnection(); + return { + connection: dummyConnection, + server: new Hono(), + em: new EntityManager([], dummyConnection), + emgr: new EventManager(), + guard: new Guard(), + ...overrides + }; +} + +export function moduleTestSuite(module: { new (): Module }) { + let ctx: ModuleBuildContext; + + beforeEach(() => { + ctx = makeCtx(); + }); + + describe("Module Tests", () => { + it("should build without exceptions", async () => { + const m = new module(); + await m.setContext(ctx).build(); + expect(m.toJSON()).toBeDefined(); + }); + + it("uses the default config", async () => { + const m = new module(); + await m.setContext(ctx).build(); + expect(stripMark(m.toJSON())).toEqual(Default(m.getSchema(), {})); + }); + }); +} diff --git a/app/bknd.config.js b/app/bknd.config.js new file mode 100644 index 0000000..6b4b0e1 --- /dev/null +++ b/app/bknd.config.js @@ -0,0 +1,12 @@ +//import type { BkndConfig } from "./src"; + +export default { + app: { + connection: { + type: "libsql", + config: { + url: "http://localhost:8080" + } + } + } +}; diff --git a/app/build-cf.ts b/app/build-cf.ts new file mode 100644 index 0000000..3015fbe --- /dev/null +++ b/app/build-cf.ts @@ -0,0 +1,42 @@ +import process from "node:process"; +import { $ } from "bun"; +import * as esbuild from "esbuild"; +import type { BuildOptions } from "esbuild"; + +const isDev = process.env.NODE_ENV !== "production"; + +const metafile = true; +const sourcemap = false; + +const config: BuildOptions = { + entryPoints: ["worker.ts"], + bundle: true, + format: "esm", + external: ["__STATIC_CONTENT_MANIFEST", "@xyflow/react"], + platform: "browser", + conditions: ["worker", "browser"], + target: "es2022", + sourcemap, + metafile, + minify: !isDev, + loader: { + ".html": "copy" + }, + outfile: "dist/worker.js" +}; + +const dist = config.outfile!.split("/")[0]; +if (!isDev) { + await $`rm -rf ${dist}`; +} + +const result = await esbuild.build(config); + +if (result.metafile) { + console.log("writing metafile to", `${dist}/meta.json`); + await Bun.write(`${dist}/meta.json`, JSON.stringify(result.metafile!)); +} + +if (!isDev) { + await $`gzip ${dist}/worker.js -c > ${dist}/worker.js.gz`; +} diff --git a/app/bunfig.toml b/app/bunfig.toml new file mode 100644 index 0000000..82e1cd0 --- /dev/null +++ b/app/bunfig.toml @@ -0,0 +1,2 @@ +[install] +registry = "http://localhost:4873" \ No newline at end of file diff --git a/app/env.d.ts b/app/env.d.ts new file mode 100644 index 0000000..99dbca3 --- /dev/null +++ b/app/env.d.ts @@ -0,0 +1,56 @@ +/// +/// +/// + +import {} from "hono"; +import type { App } from "./src/App"; +import type { Env as AppEnv } from "./src/core/env"; + +declare module "__STATIC_CONTENT_MANIFEST" { + const manifest: string; + export default manifest; +} + +type TURSO_DB = { + url: string; + authToken: string; +}; +/* +// automatically add bindings everywhere (also when coming from controllers) +declare module "hono" { + interface Env { + // c.var types + Variables: { + app: App; + }; + // c.env types + Bindings: AppEnv; + } +}*/ + +declare const __isDev: boolean; +declare global { + /*interface Request { + cf: IncomingRequestCfProperties; + }*/ + + type AppContext = { + app: App; + }; + + type HonoEnv = { + Variables: { + app: App; + }; + Bindings: AppEnv; + }; + + type Prettify = { + [K in keyof T]: T[K]; + } & NonNullable; + + // prettify recursively + type PrettifyRec = { + [K in keyof T]: T[K] extends object ? Prettify : T[K]; + } & NonNullable; +} diff --git a/app/index.html b/app/index.html new file mode 100644 index 0000000..4807db2 --- /dev/null +++ b/app/index.html @@ -0,0 +1,13 @@ + + + + + + BKND + + +
+
+ + + diff --git a/app/package.json b/app/package.json new file mode 100644 index 0000000..3709b51 --- /dev/null +++ b/app/package.json @@ -0,0 +1,175 @@ +{ + "name": "bknd", + "type": "module", + "sideEffects": false, + "bin": "./dist/cli/index.js", + "version": "0.0.75", + "scripts": { + "build:all": "rm -rf dist && bun build:css && bun run build && bun build:vite && bun build:adapters && bun build:cli", + "dev": "vite", + "test": "ALL_TESTS=1 bun test --bail", + "build": "bun tsup && bun build:types", + "watch": "bun tsup --watch --onSuccess 'bun run build:types'", + "types": "bun tsc --noEmit", + "build:types": "tsc --emitDeclarationOnly", + "build:css": "bun tailwindcss -i ./src/ui/styles.css -o ./dist/styles.css", + "watch:css": "bun tailwindcss --watch -i ./src/ui/styles.css -o ./dist/styles.css", + "build:vite": "vite build", + "build:adapters": "bun tsup.adapters.ts --minify", + "watch:adapters": "bun tsup.adapters.ts --watch", + "updater": "bun x npm-check-updates -ui", + "build:cli": "bun build src/cli/index.ts --target node --outdir dist/cli --minify", + "cli": "LOCAL=1 bun src/cli/index.ts" + }, + "dependencies": { + "@cfworker/json-schema": "^2.0.1", + "@codemirror/lang-html": "^6.4.9", + "@codemirror/lang-json": "^6.0.1", + "@codemirror/lang-liquid": "^6.2.1", + "@dagrejs/dagre": "^1.1.4", + "@hello-pangea/dnd": "^17.0.0", + "@hono/typebox-validator": "^0.2.4", + "@hono/zod-validator": "^0.2.2", + "@hookform/resolvers": "^3.9.1", + "@libsql/client": "^0.14.0", + "@libsql/kysely-libsql": "^0.4.1", + "@mantine/core": "^7.13.4", + "@mantine/hooks": "^7.13.4", + "@mantine/modals": "^7.13.4", + "@mantine/notifications": "^7.13.5", + "@radix-ui/react-scroll-area": "^1.2.0", + "@rjsf/core": "^5.22.2", + "@sinclair/typebox": "^0.32.34", + "@tabler/icons-react": "3.18.0", + "@tanstack/react-form": "0.19.2", + "@tanstack/react-query": "^5.59.16", + "@uiw/react-codemirror": "^4.23.6", + "@xyflow/react": "^12.3.2", + "aws4fetch": "^1.0.18", + "codemirror-lang-liquid": "^1.0.0", + "dayjs": "^1.11.13", + "fast-xml-parser": "^4.4.0", + "hono": "^4.4.12", + "jose": "^5.6.3", + "jotai": "^2.10.1", + "kysely": "^0.27.4", + "liquidjs": "^10.15.0", + "lodash-es": "^4.17.21", + "oauth4webapi": "^2.11.1", + "react-hook-form": "^7.53.1", + "react-icons": "5.2.1", + "react-json-view-lite": "^2.0.1", + "reactflow": "^11.11.4", + "tailwind-merge": "^2.5.4", + "tailwindcss-animate": "^1.0.7", + "wouter": "^3.3.5", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.2" + }, + "devDependencies": { + "@aws-sdk/client-s3": "^3.613.0", + "@hono/node-server": "^1.13.3", + "@hono/vite-dev-server": "^0.16.0", + "@tanstack/react-query-devtools": "^5.59.16", + "@types/diff": "^5.2.3", + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.3", + "autoprefixer": "^10.4.20", + "node-fetch": "^3.3.2", + "openapi-types": "^12.1.3", + "postcss": "^8.4.47", + "postcss-preset-mantine": "^1.17.0", + "postcss-simple-vars": "^7.0.1", + "tailwindcss": "^3.4.14", + "tsup": "^8.3.5", + "vite": "^5.4.10", + "vite-plugin-static-copy": "^2.0.0", + "vite-tsconfig-paths": "^5.0.1" + }, + "tsup": { + "entry": ["src/index.ts", "src/ui/index.ts", "src/data/index.ts", "src/core/index.ts", "src/core/utils/index.ts"], + "minify": true, + "outDir": "dist", + "external": ["bun:test"], + "sourcemap": true, + "metafile": true, + "platform": "browser", + "format": ["esm", "cjs"], + "splitting": false, + "loader": { + ".svg": "dataurl" + } + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "./ui": { + "types": "./dist/ui/index.d.ts", + "import": "./dist/ui/index.js", + "require": "./dist/ui/index.cjs" + }, + "./data": { + "types": "./dist/data/index.d.ts", + "import": "./dist/data/index.js", + "require": "./dist/data/index.cjs" + }, + "./core": { + "types": "./dist/core/index.d.ts", + "import": "./dist/core/index.js", + "require": "./dist/core/index.cjs" + }, + "./utils": { + "types": "./dist/core/utils/index.d.ts", + "import": "./dist/core/utils/index.js", + "require": "./dist/core/utils/index.cjs" + }, + "./cli": { + "types": "./dist/cli/index.d.ts", + "import": "./dist/cli/index.js", + "require": "./dist/cli/index.cjs" + }, + "./adapter/cloudflare": { + "types": "./dist/adapter/cloudflare/index.d.ts", + "import": "./dist/adapter/cloudflare/index.js", + "require": "./dist/adapter/cloudflare/index.cjs" + }, + "./adapter/vite": { + "types": "./dist/adapter/vite/index.d.ts", + "import": "./dist/adapter/vite/index.js", + "require": "./dist/adapter/vite/index.cjs" + }, + "./adapter/nextjs": { + "types": "./dist/adapter/nextjs/index.d.ts", + "import": "./dist/adapter/nextjs/index.js", + "require": "./dist/adapter/nextjs/index.cjs" + }, + "./adapter/remix": { + "types": "./dist/adapter/remix/index.d.ts", + "import": "./dist/adapter/remix/index.js", + "require": "./dist/adapter/remix/index.cjs" + }, + "./adapter/bun": { + "types": "./dist/adapter/bun/index.d.ts", + "import": "./dist/adapter/bun/index.js", + "require": "./dist/adapter/bun/index.cjs" + }, + "./dist/static/manifest.json": "./dist/static/.vite/manifest.json", + "./dist/styles.css": "./dist/styles.css", + "./dist/index.html": "./dist/static/index.html" + }, + "files": [ + "dist", + "!dist/*.tsbuildinfo", + "!dist/*.map", + "!dist/**/*.map", + "!dist/metafile*" + ] +} diff --git a/app/postcss.config.js b/app/postcss.config.js new file mode 100644 index 0000000..0c65377 --- /dev/null +++ b/app/postcss.config.js @@ -0,0 +1,18 @@ +export default { + plugins: { + "postcss-import": {}, + "tailwindcss/nesting": {}, + tailwindcss: {}, + autoprefixer: {}, + "postcss-preset-mantine": {}, + "postcss-simple-vars": { + variables: { + "mantine-breakpoint-xs": "36em", + "mantine-breakpoint-sm": "48em", + "mantine-breakpoint-md": "62em", + "mantine-breakpoint-lg": "75em", + "mantine-breakpoint-xl": "88em" + } + } + } +}; diff --git a/app/src/Api.ts b/app/src/Api.ts new file mode 100644 index 0000000..b413819 --- /dev/null +++ b/app/src/Api.ts @@ -0,0 +1,95 @@ +import { AuthApi } from "auth/api/AuthApi"; +import { DataApi } from "data/api/DataApi"; +import { decodeJwt } from "jose"; +import { MediaApi } from "media/api/MediaApi"; +import { SystemApi } from "modules/SystemApi"; + +export type ApiOptions = { + host: string; + token?: string; + tokenStorage?: "localStorage"; + localStorage?: { + key?: string; + }; +}; + +export class Api { + private token?: string; + private user?: object; + private verified = false; + + public system!: SystemApi; + public data!: DataApi; + public auth!: AuthApi; + public media!: MediaApi; + + constructor(private readonly options: ApiOptions) { + if (options.token) { + this.updateToken(options.token); + } else { + this.extractToken(); + } + + this.buildApis(); + } + + private extractToken() { + if (this.options.tokenStorage === "localStorage") { + const key = this.options.localStorage?.key ?? "auth"; + const raw = localStorage.getItem(key); + + if (raw) { + const { token } = JSON.parse(raw); + this.token = token; + this.user = decodeJwt(token) as any; + } + } + } + + updateToken(token?: string, rebuild?: boolean) { + this.token = token; + this.user = token ? (decodeJwt(token) as any) : undefined; + + if (this.options.tokenStorage === "localStorage") { + const key = this.options.localStorage?.key ?? "auth"; + + if (token) { + localStorage.setItem(key, JSON.stringify({ token })); + } else { + localStorage.removeItem(key); + } + } + + if (rebuild) this.buildApis(); + } + + markAuthVerified(verfied: boolean) { + this.verified = verfied; + return this; + } + + getAuthState() { + if (!this.token) return; + + return { + token: this.token, + user: this.user, + verified: this.verified + }; + } + + private buildApis() { + const baseParams = { + host: this.options.host, + token: this.token + }; + + this.system = new SystemApi(baseParams); + this.data = new DataApi(baseParams); + this.auth = new AuthApi({ + ...baseParams, + onTokenUpdate: (token) => this.updateToken(token, true) + }); + this.media = new MediaApi(baseParams); + } +} diff --git a/app/src/App.ts b/app/src/App.ts new file mode 100644 index 0000000..4d7a432 --- /dev/null +++ b/app/src/App.ts @@ -0,0 +1,142 @@ +import { Event } from "core/events"; +import { Connection, type LibSqlCredentials, LibsqlConnection } from "data"; +import { + type InitialModuleConfigs, + ModuleManager, + type ModuleManagerOptions, + type Modules +} from "modules/ModuleManager"; +import * as SystemPermissions from "modules/permissions"; +import { SystemController } from "modules/server/SystemController"; + +export type AppPlugin = (app: App) => void; + +export class AppConfigUpdatedEvent extends Event<{ app: App }> { + static override slug = "app-config-updated"; +} +export class AppBuiltEvent extends Event<{ app: App }> { + static override slug = "app-built"; +} +export const AppEvents = { AppConfigUpdatedEvent, AppBuiltEvent } as const; + +export type CreateAppConfig = { + connection: + | Connection + | { + type: "libsql"; + config: LibSqlCredentials; + }; + initialConfig?: InitialModuleConfigs; + plugins?: AppPlugin[]; + options?: ModuleManagerOptions; +}; + +export type AppConfig = InitialModuleConfigs; + +export class App { + modules: ModuleManager; + static readonly Events = AppEvents; + + constructor( + private connection: Connection, + _initialConfig?: InitialModuleConfigs, + private plugins: AppPlugin[] = [], + moduleManagerOptions?: ModuleManagerOptions + ) { + this.modules = new ModuleManager(connection, { + ...moduleManagerOptions, + initial: _initialConfig, + onUpdated: async (key, config) => { + //console.log("[APP] config updated", key, config); + await this.build({ sync: true, save: true }); + await this.emgr.emit(new AppConfigUpdatedEvent({ app: this })); + } + }); + this.modules.ctx().emgr.registerEvents(AppEvents); + } + + static create(config: CreateAppConfig) { + let connection: Connection | undefined = undefined; + + if (config.connection instanceof Connection) { + connection = config.connection; + } else if (typeof config.connection === "object") { + switch (config.connection.type) { + case "libsql": + connection = new LibsqlConnection(config.connection.config); + break; + } + } + if (!connection) { + throw new Error("Invalid connection"); + } + + return new App(connection, config.initialConfig, config.plugins, config.options); + } + + get emgr() { + return this.modules.ctx().emgr; + } + + async build(options?: { sync?: boolean; drop?: boolean; save?: boolean }) { + //console.log("building"); + await this.modules.build(); + + if (options?.sync) { + const syncResult = await this.module.data.em + .schema() + .sync({ force: true, drop: options.drop }); + //console.log("syncing", syncResult); + } + + // load system controller + this.modules.ctx().guard.registerPermissions(Object.values(SystemPermissions)); + this.modules.server.route("/api/system", new SystemController(this).getController()); + + // load plugins + if (this.plugins.length > 0) { + this.plugins.forEach((plugin) => plugin(this)); + } + + //console.log("emitting built", options); + await this.emgr.emit(new AppBuiltEvent({ app: this })); + + // not found on any not registered api route + this.modules.server.all("/api/*", async (c) => c.notFound()); + + if (options?.save) { + await this.modules.save(); + } + } + + mutateConfig(module: Module) { + return this.modules.get(module).schema(); + } + + get fetch(): any { + return this.modules.server.fetch; + } + + get module() { + return new Proxy( + {}, + { + get: (_, module: keyof Modules) => { + return this.modules.get(module); + } + } + ) as Modules; + } + + getSchema() { + return this.modules.getSchema(); + } + + version() { + return this.modules.version(); + } + + toJSON(secrets?: boolean) { + return this.modules.toJSON(secrets); + } +} diff --git a/app/src/adapter/bun/bun.adapter.ts b/app/src/adapter/bun/bun.adapter.ts new file mode 100644 index 0000000..9c276ce --- /dev/null +++ b/app/src/adapter/bun/bun.adapter.ts @@ -0,0 +1,33 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { App, type CreateAppConfig } from "bknd"; +import { serveStatic } from "hono/bun"; + +let app: App; +export function serve(config: CreateAppConfig, distPath?: string) { + const root = path.resolve(distPath ?? "./node_modules/bknd/dist", "static"); + + return async (req: Request) => { + if (!app) { + app = App.create(config); + + app.emgr.on( + "app-built", + async () => { + app.modules.server.get( + "/assets/*", + serveStatic({ + root + }) + ); + app.module?.server?.setAdminHtml(await readFile(root + "/index.html", "utf-8")); + }, + "sync" + ); + + await app.build(); + } + + return app.modules.server.fetch(req); + }; +} diff --git a/app/src/adapter/bun/index.ts b/app/src/adapter/bun/index.ts new file mode 100644 index 0000000..c95cfb4 --- /dev/null +++ b/app/src/adapter/bun/index.ts @@ -0,0 +1 @@ +export * from "./bun.adapter"; diff --git a/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts new file mode 100644 index 0000000..8798bbe --- /dev/null +++ b/app/src/adapter/cloudflare/cloudflare-workers.adapter.ts @@ -0,0 +1,267 @@ +import { DurableObject } from "cloudflare:workers"; +import { App, type CreateAppConfig } from "bknd"; +import { Hono } from "hono"; +import { serveStatic } from "hono/cloudflare-workers"; +import type { BkndConfig, CfBkndModeCache } from "../index"; + +// @ts-ignore +//import manifest from "__STATIC_CONTENT_MANIFEST"; + +import _html from "../../static/index.html"; + +type Context = { + request: Request; + env: any; + ctx: ExecutionContext; + manifest: any; + html: string; +}; + +export function serve(_config: BkndConfig, manifest?: string, overrideHtml?: string) { + const html = overrideHtml ?? _html; + return { + async fetch(request: Request, env: any, ctx: ExecutionContext) { + const url = new URL(request.url); + + if (manifest) { + const pathname = url.pathname.slice(1); + const assetManifest = JSON.parse(manifest); + if (pathname && pathname in assetManifest) { + const hono = new Hono(); + + hono.all("*", async (c, next) => { + const res = await serveStatic({ + path: `./${pathname}`, + manifest, + onNotFound: (path) => console.log("not found", path) + })(c as any, next); + if (res instanceof Response) { + const ttl = pathname.startsWith("assets/") + ? 60 * 60 * 24 * 365 // 1 year + : 60 * 5; // 5 minutes + res.headers.set("Cache-Control", `public, max-age=${ttl}`); + return res; + } + + return c.notFound(); + }); + + return hono.fetch(request, env); + } + } + + const config = { + ..._config, + setAdminHtml: _config.setAdminHtml ?? !!manifest + }; + const context = { request, env, ctx, manifest, html }; + const mode = config.cloudflare?.mode?.(env); + + if (!mode) { + console.log("serving fresh..."); + const app = await getFresh(config, context); + return app.fetch(request, env); + } else if ("cache" in mode) { + console.log("serving cached..."); + const app = await getCached(config as any, context); + return app.fetch(request, env); + } else if ("durableObject" in mode) { + console.log("serving durable..."); + + if (config.onBuilt) { + console.log("onBuilt() is not supported with DurableObject mode"); + } + + const start = performance.now(); + + const durable = mode.durableObject; + const id = durable.idFromName(mode.key); + const stub = durable.get(id) as unknown as DurableBkndApp; + + const create_config = typeof config.app === "function" ? config.app(env) : config.app; + + const res = await stub.fire(request, { + config: create_config, + html, + keepAliveSeconds: mode.keepAliveSeconds, + setAdminHtml: config.setAdminHtml + }); + + const headers = new Headers(res.headers); + headers.set("X-TTDO", String(performance.now() - start)); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers + }); + } + } + }; +} + +async function getFresh(config: BkndConfig, { env, html }: Context) { + const create_config = typeof config.app === "function" ? config.app(env) : config.app; + const app = App.create(create_config); + + if (config.onBuilt) { + app.emgr.onEvent( + App.Events.AppBuiltEvent, + async ({ params: { app } }) => { + config.onBuilt!(app); + }, + "sync" + ); + } + + await app.build(); + + if (config?.setAdminHtml !== false) { + app.module.server.setAdminHtml(html); + } + + return app; +} + +async function getCached( + config: BkndConfig & { cloudflare: { mode: CfBkndModeCache } }, + { env, html, ctx }: Context +) { + const { cache, key } = config.cloudflare.mode(env) as ReturnType; + const create_config = typeof config.app === "function" ? config.app(env) : config.app; + + const cachedConfig = await cache.get(key); + const initialConfig = cachedConfig ? JSON.parse(cachedConfig) : undefined; + + const app = App.create({ ...create_config, initialConfig }); + + async function saveConfig(__config: any) { + ctx.waitUntil(cache.put(key, JSON.stringify(__config))); + } + + if (config.onBuilt) { + app.emgr.onEvent( + App.Events.AppBuiltEvent, + async ({ params: { app } }) => { + app.module.server.client.get("/__bknd/cache", async (c) => { + await cache.delete(key); + return c.json({ message: "Cache cleared" }); + }); + + config.onBuilt!(app); + }, + "sync" + ); + } + + app.emgr.onEvent( + App.Events.AppConfigUpdatedEvent, + async ({ params: { app } }) => { + saveConfig(app.toJSON(true)); + }, + "sync" + ); + + await app.build(); + if (!cachedConfig) { + saveConfig(app.toJSON(true)); + } + + //addAssetsRoute(app, manifest); + if (config?.setAdminHtml !== false) { + app.module.server.setAdminHtml(html); + } + + return app; +} + +export class DurableBkndApp extends DurableObject { + protected id = Math.random().toString(36).slice(2); + protected app?: App; + protected interval?: any; + + async fire( + request: Request, + options: { + config: CreateAppConfig; + html: string; + keepAliveSeconds?: number; + setAdminHtml?: boolean; + } + ) { + let buildtime = 0; + if (!this.app) { + const start = performance.now(); + const config = options.config; + + // change protocol to websocket if libsql + if ("type" in config.connection && config.connection.type === "libsql") { + config.connection.config.protocol = "wss"; + } + + this.app = App.create(config); + this.app.emgr.onEvent( + App.Events.AppBuiltEvent, + async ({ params: { app } }) => { + app.modules.server.get("/__do", async (c) => { + // @ts-ignore + const context: any = c.req.raw.cf ? c.req.raw.cf : c.env.cf; + return c.json({ + id: this.id, + keepAlive: options?.keepAliveSeconds, + colo: context.colo + }); + }); + + if (options?.setAdminHtml !== false) { + app.module.server.setAdminHtml(options.html); + } + }, + "sync" + ); + + await this.app.build(); + + buildtime = performance.now() - start; + } + + if (options?.keepAliveSeconds) { + this.keepAlive(options.keepAliveSeconds); + } + + console.log("id", this.id); + const res = await this.app!.fetch(request); + const headers = new Headers(res.headers); + headers.set("X-BuildTime", buildtime.toString()); + headers.set("X-DO-ID", this.id); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers + }); + } + + protected keepAlive(seconds: number) { + console.log("keep alive for", seconds); + if (this.interval) { + console.log("clearing, there is a new"); + clearInterval(this.interval); + } + + let i = 0; + this.interval = setInterval(() => { + i += 1; + //console.log("keep-alive", i); + if (i === seconds) { + console.log("cleared"); + clearInterval(this.interval); + + // ping every 30 seconds + } else if (i % 30 === 0) { + console.log("ping"); + this.app?.modules.ctx().connection.ping(); + } + }, 1000); + } +} diff --git a/app/src/adapter/cloudflare/index.ts b/app/src/adapter/cloudflare/index.ts new file mode 100644 index 0000000..f2d3cdd --- /dev/null +++ b/app/src/adapter/cloudflare/index.ts @@ -0,0 +1 @@ +export * from "./cloudflare-workers.adapter"; diff --git a/app/src/adapter/index.ts b/app/src/adapter/index.ts new file mode 100644 index 0000000..04a9bb8 --- /dev/null +++ b/app/src/adapter/index.ts @@ -0,0 +1,36 @@ +import type { App, CreateAppConfig } from "bknd"; + +export type CfBkndModeCache = (env: Env) => { + cache: KVNamespace; + key: string; +}; + +export type CfBkndModeDurableObject = (env: Env) => { + durableObject: DurableObjectNamespace; + key: string; + keepAliveSeconds?: number; +}; + +export type CloudflareBkndConfig = { + mode?: CfBkndModeCache | CfBkndModeDurableObject; + forceHttps?: boolean; +}; + +export type BkndConfig = { + app: CreateAppConfig | ((env: Env) => CreateAppConfig); + setAdminHtml?: boolean; + server?: { + port?: number; + platform?: "node" | "bun"; + }; + cloudflare?: CloudflareBkndConfig; + onBuilt?: (app: App) => Promise; +}; + +export type BkndConfigJson = { + app: CreateAppConfig; + setAdminHtml?: boolean; + server?: { + port?: number; + }; +}; diff --git a/app/src/adapter/nextjs/index.ts b/app/src/adapter/nextjs/index.ts new file mode 100644 index 0000000..957fa9e --- /dev/null +++ b/app/src/adapter/nextjs/index.ts @@ -0,0 +1 @@ +export * from "./nextjs.adapter"; diff --git a/app/src/adapter/nextjs/nextjs.adapter.ts b/app/src/adapter/nextjs/nextjs.adapter.ts new file mode 100644 index 0000000..d533d94 --- /dev/null +++ b/app/src/adapter/nextjs/nextjs.adapter.ts @@ -0,0 +1,25 @@ +import { App, type CreateAppConfig } from "bknd"; +import { isDebug } from "bknd/core"; + +function getCleanRequest(req: Request) { + // clean search params from "route" attribute + const url = new URL(req.url); + url.searchParams.delete("route"); + return new Request(url.toString(), { + method: req.method, + headers: req.headers, + body: req.body + }); +} + +let app: App; +export function serve(config: CreateAppConfig) { + return async (req: Request) => { + if (!app || isDebug()) { + app = App.create(config); + await app.build(); + } + const request = getCleanRequest(req); + return app.fetch(request, process.env); + }; +} diff --git a/app/src/adapter/remix/index.ts b/app/src/adapter/remix/index.ts new file mode 100644 index 0000000..77b0812 --- /dev/null +++ b/app/src/adapter/remix/index.ts @@ -0,0 +1 @@ +export * from "./remix.adapter"; diff --git a/app/src/adapter/remix/remix.adapter.ts b/app/src/adapter/remix/remix.adapter.ts new file mode 100644 index 0000000..bf978da --- /dev/null +++ b/app/src/adapter/remix/remix.adapter.ts @@ -0,0 +1,12 @@ +import { App, type CreateAppConfig } from "../../App"; + +let app: App; +export function serve(config: CreateAppConfig) { + return async (args: { request: Request }) => { + if (!app) { + app = App.create(config); + await app.build(); + } + return app.fetch(args.request); + }; +} diff --git a/app/src/adapter/vite/index.ts b/app/src/adapter/vite/index.ts new file mode 100644 index 0000000..832bcde --- /dev/null +++ b/app/src/adapter/vite/index.ts @@ -0,0 +1 @@ +export * from "./vite.adapter"; diff --git a/app/src/adapter/vite/vite.adapter.ts b/app/src/adapter/vite/vite.adapter.ts new file mode 100644 index 0000000..d2fbe10 --- /dev/null +++ b/app/src/adapter/vite/vite.adapter.ts @@ -0,0 +1,82 @@ +import { readFile } from "node:fs/promises"; +import { serveStatic } from "@hono/node-server/serve-static"; +import { App } from "../../App"; +import type { BkndConfig } from "../index"; + +async function getHtml() { + return readFile("index.html", "utf8"); +} +function addViteScripts(html: string) { + return html.replace( + "", + ` + +` + ); +} + +function createApp(config: BkndConfig, env: any) { + const create_config = typeof config.app === "function" ? config.app(env) : config.app; + return App.create(create_config); +} + +function setAppBuildListener(app: App, config: BkndConfig, html: string) { + app.emgr.on( + "app-built", + async () => { + await config.onBuilt?.(app); + app.module.server.setAdminHtml(html); + app.module.server.client.get("/assets/!*", serveStatic({ root: "./" })); + }, + "sync" + ); +} + +export async function serveFresh(config: BkndConfig, _html?: string) { + let html = _html; + if (!html) { + html = await getHtml(); + } + + html = addViteScripts(html); + + return { + async fetch(request: Request, env: any, ctx: ExecutionContext) { + const app = createApp(config, env); + + setAppBuildListener(app, config, html); + await app.build(); + + //console.log("routes", app.module.server.client.routes); + return app.fetch(request, env, ctx); + } + }; +} + +let app: App; +export async function serveCached(config: BkndConfig, _html?: string) { + let html = _html; + if (!html) { + html = await getHtml(); + } + + html = addViteScripts(html); + + return { + async fetch(request: Request, env: any, ctx: ExecutionContext) { + if (!app) { + app = createApp(config, env); + setAppBuildListener(app, config, html); + await app.build(); + } + + return app.fetch(request, env, ctx); + } + }; +} diff --git a/app/src/auth/AppAuth.ts b/app/src/auth/AppAuth.ts new file mode 100644 index 0000000..73bb829 --- /dev/null +++ b/app/src/auth/AppAuth.ts @@ -0,0 +1,269 @@ +import { type AuthAction, Authenticator, type ProfileExchange, Role, type Strategy } from "auth"; +import { Exception } from "core"; +import { Const, StringRecord, Type, transformObject } from "core/utils"; +import { + type Entity, + EntityIndex, + type EntityManager, + EnumField, + type Field, + type Mutator +} from "data"; +import { type FieldSchema, entity, enumm, make, text } from "data/prototype"; +import { cloneDeep, mergeWith, omit, pick } from "lodash-es"; +import { Module } from "modules/Module"; +import { AuthController } from "./api/AuthController"; +import { type AppAuthSchema, STRATEGIES, authConfigSchema } from "./auth-schema"; + +export type UserFieldSchema = FieldSchema; +declare global { + interface DB { + users: UserFieldSchema; + } +} + +export class AppAuth extends Module { + private _authenticator?: Authenticator; + cache: Record = {}; + + override async build() { + if (!this.config.enabled) { + this.setBuilt(); + return; + } + + // register roles + const roles = transformObject(this.config.roles ?? {}, (role, name) => { + //console.log("role", role, name); + return Role.create({ name, ...role }); + }); + this.ctx.guard.setRoles(Object.values(roles)); + this.ctx.guard.setConfig(this.config.guard ?? {}); + + // build strategies + const strategies = transformObject(this.config.strategies ?? {}, (strategy, name) => { + try { + return new STRATEGIES[strategy.type].cls(strategy.config as any); + } catch (e) { + throw new Error( + `Could not build strategy ${String(name)} with config ${JSON.stringify(strategy.config)}` + ); + } + }); + + const { fields, ...jwt } = this.config.jwt; + this._authenticator = new Authenticator(strategies, this.resolveUser.bind(this), { + jwt + }); + + this.registerEntities(); + super.setBuilt(); + + const controller = new AuthController(this); + //this.ctx.server.use(controller.getMiddleware); + this.ctx.server.route(this.config.basepath, controller.getController()); + } + + getMiddleware() { + if (!this.config.enabled) { + return; + } + + return new AuthController(this).getMiddleware; + } + + getSchema() { + return authConfigSchema; + } + + get authenticator(): Authenticator { + this.throwIfNotBuilt(); + return this._authenticator!; + } + + get em(): EntityManager { + return this.ctx.em as any; + } + + private async resolveUser( + action: AuthAction, + strategy: Strategy, + identifier: string, + profile: ProfileExchange + ): Promise { + console.log("***** AppAuth:resolveUser", { + action, + strategy: strategy.getName(), + identifier, + profile + }); + + const fields = this.getUsersEntity() + .getFillableFields("create") + .map((f) => f.name); + const filteredProfile = Object.fromEntries( + Object.entries(profile).filter(([key]) => fields.includes(key)) + ); + + switch (action) { + case "login": + return this.login(strategy, identifier, filteredProfile); + case "register": + return this.register(strategy, identifier, filteredProfile); + } + } + + private filterUserData(user: any) { + console.log( + "--filterUserData", + user, + this.config.jwt.fields, + pick(user, this.config.jwt.fields) + ); + return pick(user, this.config.jwt.fields); + } + + private async login(strategy: Strategy, identifier: string, profile: ProfileExchange) { + console.log("--- trying to login", { strategy: strategy.getName(), identifier, profile }); + if (!("email" in profile)) { + throw new Exception("Profile must have email"); + } + if (typeof identifier !== "string" || identifier.length === 0) { + throw new Exception("Identifier must be a string"); + } + + const users = this.getUsersEntity(); + this.toggleStrategyValueVisibility(true); + const result = await this.em.repo(users).findOne({ email: profile.email! }); + this.toggleStrategyValueVisibility(false); + if (!result.data) { + throw new Exception("User not found", 404); + } + console.log("---login data", result.data, result); + + // compare strategy and identifier + console.log("strategy comparison", result.data.strategy, strategy.getName()); + if (result.data.strategy !== strategy.getName()) { + console.log("!!! User registered with different strategy"); + throw new Exception("User registered with different strategy"); + } + + console.log("identifier comparison", result.data.strategy_value, identifier); + if (result.data.strategy_value !== identifier) { + console.log("!!! Invalid credentials"); + throw new Exception("Invalid credentials"); + } + + return this.filterUserData(result.data); + } + + private async register(strategy: Strategy, identifier: string, profile: ProfileExchange) { + if (!("email" in profile)) { + throw new Exception("Profile must have an email"); + } + if (typeof identifier !== "string" || identifier.length === 0) { + throw new Exception("Identifier must be a string"); + } + + const users = this.getUsersEntity(); + const { data } = await this.em.repo(users).findOne({ email: profile.email! }); + if (data) { + throw new Exception("User already exists"); + } + + const payload = { + ...profile, + strategy: strategy.getName(), + strategy_value: identifier + }; + + const mutator = this.em.mutator(users); + mutator.__unstable_toggleSystemEntityCreation(false); + this.toggleStrategyValueVisibility(true); + const createResult = await mutator.insertOne(payload); + mutator.__unstable_toggleSystemEntityCreation(true); + this.toggleStrategyValueVisibility(false); + if (!createResult.data) { + throw new Error("Could not create user"); + } + + return this.filterUserData(createResult.data); + } + + private toggleStrategyValueVisibility(visible: boolean) { + const field = this.getUsersEntity().field("strategy_value")!; + + field.config.hidden = !visible; + field.config.fillable = visible; + // @todo: think about a PasswordField that automatically hashes on save? + } + + getUsersEntity(forceCreate?: boolean): Entity<"users", typeof AppAuth.usersFields> { + const entity_name = this.config.entity_name; + if (forceCreate || !this.em.hasEntity(entity_name)) { + return entity(entity_name as "users", AppAuth.usersFields, undefined, "system"); + } + + return this.em.entity(entity_name) as any; + } + + static usersFields = { + email: text().required(), + strategy: text({ fillable: ["create"], hidden: ["form"] }).required(), + strategy_value: text({ + fillable: ["create"], + hidden: ["read", "table", "update", "form"] + }).required(), + role: text() + }; + + registerEntities() { + const users = this.getUsersEntity(); + + if (!this.em.hasEntity(users.name)) { + this.em.addEntity(users); + } else { + // if exists, check all fields required are there + // @todo: add to context: "needs sync" flag + const _entity = this.getUsersEntity(true); + for (const field of _entity.fields) { + const _field = users.field(field.name); + if (!_field) { + users.addField(field); + } + } + } + + const indices = [ + new EntityIndex(users, [users.field("email")!], true), + new EntityIndex(users, [users.field("strategy")!]), + new EntityIndex(users, [users.field("strategy_value")!]) + ]; + indices.forEach((index) => { + if (!this.em.hasIndex(index)) { + this.em.addIndex(index); + } + }); + + try { + const roles = Object.keys(this.config.roles ?? {}); + const field = make("role", enumm({ enum: roles })); + this.em.entity(users.name).__experimental_replaceField("role", field); + } catch (e) {} + + try { + const strategies = Object.keys(this.config.strategies ?? {}); + const field = make("strategy", enumm({ enum: strategies })); + this.em.entity(users.name).__experimental_replaceField("strategy", field); + } catch (e) {} + } + + override toJSON(secrets?: boolean): AppAuthSchema { + if (!this.config.enabled) { + return this.configDefault; + } + + // fixes freezed config object + return mergeWith({ ...this.config }, this.authenticator.toJSON(secrets)); + } +} diff --git a/app/src/auth/api/AuthApi.ts b/app/src/auth/api/AuthApi.ts new file mode 100644 index 0000000..6b6ef0d --- /dev/null +++ b/app/src/auth/api/AuthApi.ts @@ -0,0 +1,41 @@ +import type { AppAuthSchema, AppAuthStrategies } from "auth/auth-schema"; +import type { AuthResponse, SafeUser, Strategy } from "auth/authenticate/Authenticator"; +import { type BaseModuleApiOptions, ModuleApi } from "modules/ModuleApi"; + +export type AuthApiOptions = BaseModuleApiOptions & { + onTokenUpdate?: (token: string) => void | Promise; +}; + +export class AuthApi extends ModuleApi { + protected override getDefaultOptions(): Partial { + return { + basepath: "/api/auth" + }; + } + + async loginWithPassword(input: any) { + const res = await this.post(["password", "login"], input); + if (res.res.ok && res.body.token) { + await this.options.onTokenUpdate?.(res.body.token); + } + return res; + } + + async registerWithPassword(input: any) { + const res = await this.post(["password", "register"], input); + if (res.res.ok && res.body.token) { + await this.options.onTokenUpdate?.(res.body.token); + } + return res; + } + + async me() { + return this.get<{ user: SafeUser | null }>(["me"]); + } + + async strategies() { + return this.get<{ strategies: AppAuthSchema["strategies"] }>(["strategies"]); + } + + async logout() {} +} diff --git a/app/src/auth/api/AuthController.ts b/app/src/auth/api/AuthController.ts new file mode 100644 index 0000000..b9eb02f --- /dev/null +++ b/app/src/auth/api/AuthController.ts @@ -0,0 +1,57 @@ +import type { AppAuth } from "auth"; +import type { ClassController } from "core"; +import { Hono, type MiddlewareHandler } from "hono"; + +export class AuthController implements ClassController { + constructor(private auth: AppAuth) {} + + getMiddleware: MiddlewareHandler = async (c, next) => { + // @todo: consider adding app name to the payload, because user is not refetched + + //try { + if (c.req.raw.headers.has("Authorization")) { + const bearerHeader = String(c.req.header("Authorization")); + const token = bearerHeader.replace("Bearer ", ""); + const verified = await this.auth.authenticator.verify(token); + + // @todo: don't extract user from token, but from the database or cache + this.auth.ctx.guard.setUserContext(this.auth.authenticator.getUser()); + /*console.log("jwt verified?", { + verified, + auth: this.auth.authenticator.isUserLoggedIn() + });*/ + } else { + this.auth.authenticator.__setUserNull(); + } + /* } catch (e) { + this.auth.authenticator.__setUserNull(); + }*/ + + await next(); + }; + + getController(): Hono { + const hono = new Hono(); + const strategies = this.auth.authenticator.getStrategies(); + //console.log("strategies", strategies); + + for (const [name, strategy] of Object.entries(strategies)) { + //console.log("registering", name, "at", `/${name}`); + hono.route(`/${name}`, strategy.getController(this.auth.authenticator)); + } + + hono.get("/me", async (c) => { + if (this.auth.authenticator.isUserLoggedIn()) { + return c.json({ user: await this.auth.authenticator.getUser() }); + } + + return c.json({ user: null }, 403); + }); + + hono.get("/strategies", async (c) => { + return c.json({ strategies: this.auth.toJSON(false).strategies }); + }); + + return hono; + } +} diff --git a/app/src/auth/auth-schema.ts b/app/src/auth/auth-schema.ts new file mode 100644 index 0000000..19e5581 --- /dev/null +++ b/app/src/auth/auth-schema.ts @@ -0,0 +1,85 @@ +import { jwtConfig } from "auth/authenticate/Authenticator"; +import { CustomOAuthStrategy, OAuthStrategy, PasswordStrategy } from "auth/authenticate/strategies"; +import { type Static, StringRecord, Type, objectTransform } from "core/utils"; + +export const Strategies = { + password: { + cls: PasswordStrategy, + schema: PasswordStrategy.prototype.getSchema() + }, + oauth: { + cls: OAuthStrategy, + schema: OAuthStrategy.prototype.getSchema() + }, + custom_oauth: { + cls: CustomOAuthStrategy, + schema: CustomOAuthStrategy.prototype.getSchema() + } +} as const; + +export const STRATEGIES = Strategies; +const strategiesSchemaObject = objectTransform(STRATEGIES, (strategy, name) => { + return Type.Object( + { + type: Type.Const(name, { default: name, readOnly: true }), + config: strategy.schema + }, + { + title: name, + additionalProperties: false + } + ); +}); +const strategiesSchema = Type.Union(Object.values(strategiesSchemaObject)); +export type AppAuthStrategies = Static; +export type AppAuthOAuthStrategy = Static; + +const guardConfigSchema = Type.Object({ + enabled: Type.Optional(Type.Boolean({ default: false })) +}); +export const guardRoleSchema = Type.Object( + { + permissions: Type.Optional(Type.Array(Type.String())), + is_default: Type.Optional(Type.Boolean()), + implicit_allow: Type.Optional(Type.Boolean()) + }, + { additionalProperties: false } +); + +export const authConfigSchema = Type.Object( + { + enabled: Type.Boolean({ default: false }), + basepath: Type.String({ default: "/api/auth" }), + entity_name: Type.String({ default: "users" }), + jwt: Type.Composite( + [ + jwtConfig, + Type.Object({ + fields: Type.Array(Type.String(), { default: ["id", "email", "role"] }) + }) + ], + { default: {}, additionalProperties: false } + ), + strategies: Type.Optional( + StringRecord(strategiesSchema, { + title: "Strategies", + default: { + password: { + type: "password", + config: { + hashing: "sha256" + } + } + } + }) + ), + guard: Type.Optional(guardConfigSchema), + roles: Type.Optional(StringRecord(guardRoleSchema, { default: {} })) + }, + { + title: "Authentication", + additionalProperties: false + } +); + +export type AppAuthSchema = Static; diff --git a/app/src/auth/authenticate/Authenticator.ts b/app/src/auth/authenticate/Authenticator.ts new file mode 100644 index 0000000..b335623 --- /dev/null +++ b/app/src/auth/authenticate/Authenticator.ts @@ -0,0 +1,190 @@ +import { type Static, type TSchema, Type, parse, randomString, transformObject } from "core/utils"; +import type { Hono } from "hono"; +import { type JWTVerifyOptions, SignJWT, jwtVerify } from "jose"; + +type Input = any; // workaround + +// @todo: add schema to interface to ensure proper inference +export interface Strategy { + getController: (auth: Authenticator) => Hono; + getType: () => string; + getMode: () => "form" | "external"; + getName: () => string; + toJSON: (secrets?: boolean) => any; +} + +export type User = { + id: number; + email: string; + username: string; + password: string; + role: string; +}; + +export type ProfileExchange = { + email?: string; + username?: string; + sub?: string; + password?: string; + [key: string]: any; +}; + +export type SafeUser = Omit; +export type CreateUser = Pick & { [key: string]: any }; +export type AuthResponse = { user: SafeUser; token: string }; + +export interface UserPool { + findBy: (prop: Fields, value: string | number) => Promise; + create: (user: CreateUser) => Promise; +} + +export const jwtConfig = Type.Object( + { + // @todo: autogenerate a secret if not present. But it must be persisted from AppAuth + secret: Type.String({ default: "secret" }), + alg: Type.Optional(Type.String({ enum: ["HS256"], default: "HS256" })), + expiresIn: Type.Optional(Type.String()), + issuer: Type.Optional(Type.String()) + }, + { + default: {}, + additionalProperties: false + } +); +export const authenticatorConfig = Type.Object({ + jwt: jwtConfig +}); + +type AuthConfig = Static; +export type AuthAction = "login" | "register"; +export type AuthUserResolver = ( + action: AuthAction, + strategy: Strategy, + identifier: string, + profile: ProfileExchange +) => Promise; + +export class Authenticator = Record> { + private readonly strategies: Strategies; + private readonly config: AuthConfig; + private _user: SafeUser | undefined; + private readonly userResolver: AuthUserResolver; + + constructor(strategies: Strategies, userResolver?: AuthUserResolver, config?: AuthConfig) { + this.userResolver = userResolver ?? (async (a, s, i, p) => p as any); + this.strategies = strategies as Strategies; + this.config = parse(authenticatorConfig, config ?? {}); + + /*const secret = String(this.config.jwt.secret); + if (secret === "secret" || secret.length === 0) { + this.config.jwt.secret = randomString(64, true); + }*/ + } + + async resolve( + action: AuthAction, + strategy: Strategy, + identifier: string, + profile: ProfileExchange + ) { + //console.log("resolve", { action, strategy: strategy.getName(), profile }); + const user = await this.userResolver(action, strategy, identifier, profile); + + if (user) { + return { + user, + token: await this.jwt(user) + }; + } + + throw new Error("User could not be resolved"); + } + + getStrategies(): Strategies { + return this.strategies; + } + + isUserLoggedIn(): boolean { + return this._user !== undefined; + } + + getUser() { + return this._user; + } + + // @todo: determine what to do exactly + __setUserNull() { + this._user = undefined; + } + + strategy< + StrategyName extends keyof Strategies, + Strat extends Strategy = Strategies[StrategyName] + >(strategy: StrategyName): Strat { + try { + return this.strategies[strategy] as unknown as Strat; + } catch (e) { + throw new Error(`Strategy "${String(strategy)}" not found`); + } + } + + async jwt(user: Omit): Promise { + const prohibited = ["password"]; + for (const prop of prohibited) { + if (prop in user) { + throw new Error(`Property "${prop}" is prohibited`); + } + } + + const jwt = new SignJWT(user) + .setProtectedHeader({ alg: this.config.jwt?.alg ?? "HS256" }) + .setIssuedAt(); + + if (this.config.jwt?.issuer) { + jwt.setIssuer(this.config.jwt.issuer); + } + + if (this.config.jwt?.expiresIn) { + jwt.setExpirationTime(this.config.jwt.expiresIn); + } + + return jwt.sign(new TextEncoder().encode(this.config.jwt?.secret ?? "")); + } + + async verify(jwt: string): Promise { + const options: JWTVerifyOptions = { + algorithms: [this.config.jwt?.alg ?? "HS256"] + }; + + if (this.config.jwt?.issuer) { + options.issuer = this.config.jwt.issuer; + } + + if (this.config.jwt?.expiresIn) { + options.maxTokenAge = this.config.jwt.expiresIn; + } + + try { + const { payload } = await jwtVerify( + jwt, + new TextEncoder().encode(this.config.jwt?.secret ?? ""), + options + ); + this._user = payload; + return true; + } catch (e) { + this._user = undefined; + //console.error(e); + } + + return false; + } + + toJSON(secrets?: boolean) { + return { + ...this.config, + jwt: secrets ? this.config.jwt : undefined, + strategies: transformObject(this.getStrategies(), (s) => s.toJSON(secrets)) + }; + } +} diff --git a/app/src/auth/authenticate/strategies/PasswordStrategy.ts b/app/src/auth/authenticate/strategies/PasswordStrategy.ts new file mode 100644 index 0000000..36af6ec --- /dev/null +++ b/app/src/auth/authenticate/strategies/PasswordStrategy.ts @@ -0,0 +1,98 @@ +import type { Authenticator, Strategy } from "auth"; +import { type Static, StringEnum, Type, parse } from "core/utils"; +import { hash } from "core/utils"; +import { Hono } from "hono"; + +type LoginSchema = { username: string; password: string } | { email: string; password: string }; +type RegisterSchema = { email: string; password: string; [key: string]: any }; + +const schema = Type.Object({ + hashing: StringEnum(["plain", "sha256" /*, "bcrypt"*/] as const, { default: "sha256" }) +}); + +export type PasswordStrategyOptions = Static; +/*export type PasswordStrategyOptions2 = { + hashing?: "plain" | "bcrypt" | "sha256"; +};*/ + +export class PasswordStrategy implements Strategy { + private options: PasswordStrategyOptions; + + constructor(options: Partial = {}) { + this.options = parse(schema, options); + } + + async hash(password: string) { + switch (this.options.hashing) { + case "sha256": + return hash.sha256(password); + default: + return password; + } + } + + async login(input: LoginSchema) { + if (!("email" in input) || !("password" in input)) { + throw new Error("Invalid input: Email and password must be provided"); + } + + const hashedPassword = await this.hash(input.password); + return { ...input, password: hashedPassword }; + } + + async register(input: RegisterSchema) { + if (!input.email || !input.password) { + throw new Error("Invalid input: Email and password must be provided"); + } + + return { + ...input, + password: await this.hash(input.password) + }; + } + + getController(authenticator: Authenticator): Hono { + const hono = new Hono(); + + return hono + .post("/login", async (c) => { + const body = (await c.req.json()) ?? {}; + + const payload = await this.login(body); + const data = await authenticator.resolve("login", this, payload.password, payload); + + return c.json(data); + }) + .post("/register", async (c) => { + const body = (await c.req.json()) ?? {}; + + const payload = await this.register(body); + const data = await authenticator.resolve("register", this, payload.password, payload); + + return c.json(data); + }); + } + + getSchema() { + return schema; + } + + getType() { + return "password"; + } + + getMode() { + return "form" as const; + } + + getName() { + return "password" as const; + } + + toJSON(secrets?: boolean) { + return { + type: this.getType(), + config: secrets ? this.options : undefined + }; + } +} diff --git a/app/src/auth/authenticate/strategies/index.ts b/app/src/auth/authenticate/strategies/index.ts new file mode 100644 index 0000000..97ea311 --- /dev/null +++ b/app/src/auth/authenticate/strategies/index.ts @@ -0,0 +1,13 @@ +import { CustomOAuthStrategy } from "auth/authenticate/strategies/oauth/CustomOAuthStrategy"; +import { PasswordStrategy, type PasswordStrategyOptions } from "./PasswordStrategy"; +import { OAuthCallbackException, OAuthStrategy } from "./oauth/OAuthStrategy"; + +export * as issuers from "./oauth/issuers"; + +export { + PasswordStrategy, + type PasswordStrategyOptions, + OAuthStrategy, + OAuthCallbackException, + CustomOAuthStrategy +}; diff --git a/app/src/auth/authenticate/strategies/oauth/CustomOAuthStrategy.ts b/app/src/auth/authenticate/strategies/oauth/CustomOAuthStrategy.ts new file mode 100644 index 0000000..2ff54c9 --- /dev/null +++ b/app/src/auth/authenticate/strategies/oauth/CustomOAuthStrategy.ts @@ -0,0 +1,77 @@ +import { type Static, StringEnum, Type } from "core/utils"; +import type * as oauth from "oauth4webapi"; +import { OAuthStrategy } from "./OAuthStrategy"; + +type SupportedTypes = "oauth2" | "oidc"; + +type RequireKeys = Required> & Omit; + +const UrlString = Type.String({ pattern: "^(https?|wss?)://[^\\s/$.?#].[^\\s]*$" }); +const oauthSchemaCustom = Type.Object( + { + type: StringEnum(["oidc", "oauth2"] as const, { default: "oidc" }), + name: Type.String(), + client: Type.Object( + { + client_id: Type.String(), + client_secret: Type.String(), + token_endpoint_auth_method: StringEnum(["client_secret_basic"]) + }, + { + additionalProperties: false + } + ), + as: Type.Object( + { + issuer: Type.String(), + code_challenge_methods_supported: Type.Optional(StringEnum(["S256"])), + scopes_supported: Type.Optional(Type.Array(Type.String())), + scope_separator: Type.Optional(Type.String({ default: " " })), + authorization_endpoint: Type.Optional(UrlString), + token_endpoint: Type.Optional(UrlString), + userinfo_endpoint: Type.Optional(UrlString) + }, + { + additionalProperties: false + } + ) + // @todo: profile mapping + }, + { title: "Custom OAuth", additionalProperties: false } +); + +type OAuthConfigCustom = Static; + +export type UserProfile = { + sub: string; + email: string; + [key: string]: any; +}; + +export type IssuerConfig = { + type: SupportedTypes; + client: RequireKeys; + as: oauth.AuthorizationServer & { + scope_separator?: string; + }; + profile: ( + info: UserInfo, + config: Omit, + tokenResponse: any + ) => Promise; +}; + +export class CustomOAuthStrategy extends OAuthStrategy { + override getIssuerConfig(): IssuerConfig { + return { ...this.config, profile: async (info) => info } as any; + } + + // @ts-ignore + override getSchema() { + return oauthSchemaCustom; + } + + override getType() { + return "custom_oauth"; + } +} diff --git a/app/src/auth/authenticate/strategies/oauth/OAuthStrategy.ts b/app/src/auth/authenticate/strategies/oauth/OAuthStrategy.ts new file mode 100644 index 0000000..0214059 --- /dev/null +++ b/app/src/auth/authenticate/strategies/oauth/OAuthStrategy.ts @@ -0,0 +1,431 @@ +import type { AuthAction, Authenticator, Strategy } from "auth"; +import { Exception } from "core"; +import { type Static, StringEnum, type TSchema, Type, filterKeys, parse } from "core/utils"; +import { type Context, Hono } from "hono"; +import { getSignedCookie, setSignedCookie } from "hono/cookie"; +import * as oauth from "oauth4webapi"; +import * as issuers from "./issuers"; + +type ConfiguredIssuers = keyof typeof issuers; +type SupportedTypes = "oauth2" | "oidc"; + +type RequireKeys = Required> & Omit; + +const schemaProvided = Type.Object( + { + //type: StringEnum(["oidc", "oauth2"] as const, { default: "oidc" }), + name: StringEnum(Object.keys(issuers) as ConfiguredIssuers[]), + client: Type.Object( + { + client_id: Type.String(), + client_secret: Type.String() + }, + { + additionalProperties: false + } + ) + }, + { title: "OAuth" } +); +type ProvidedOAuthConfig = Static; + +export type CustomOAuthConfig = { + type: SupportedTypes; + name: string; +} & IssuerConfig & { + client: RequireKeys< + oauth.Client, + "client_id" | "client_secret" | "token_endpoint_auth_method" + >; + }; + +type OAuthConfig = ProvidedOAuthConfig | CustomOAuthConfig; + +export type UserProfile = { + sub: string; + email: string; + [key: string]: any; +}; + +export type IssuerConfig = { + type: SupportedTypes; + client: RequireKeys; + as: oauth.AuthorizationServer & { + scope_separator?: string; + }; + profile: ( + info: UserInfo, + config: Omit, + tokenResponse: any + ) => Promise; +}; + +export class OAuthCallbackException extends Exception { + override name = "OAuthCallbackException"; + + constructor( + public error: any, + public step: string + ) { + super("OAuthCallbackException on " + step); + } +} + +export class OAuthStrategy implements Strategy { + constructor(private _config: OAuthConfig) {} + + get config() { + return this._config; + } + + getIssuerConfig(): IssuerConfig { + return issuers[this.config.name]; + } + + async getConfig(): Promise< + IssuerConfig & { + client: { + client_id: string; + client_secret: string; + }; + } + > { + const info = this.getIssuerConfig(); + + if (info.type === "oidc") { + const issuer = new URL(info.as.issuer); + const request = await oauth.discoveryRequest(issuer); + info.as = await oauth.processDiscoveryResponse(issuer, request); + } + + return { + ...info, + type: info.type, + client: { + ...info.client, + ...this._config.client + } + }; + } + + async getCodeChallenge(as: oauth.AuthorizationServer, state: string, method: "S256" = "S256") { + const challenge_supported = as.code_challenge_methods_supported?.includes(method); + let challenge: string | undefined; + let challenge_method: string | undefined; + if (challenge_supported) { + challenge = await oauth.calculatePKCECodeChallenge(state); + challenge_method = method; + } + + return { challenge_supported, challenge, challenge_method }; + } + + async request(options: { redirect_uri: string; state: string; scopes?: string[] }): Promise<{ + url: string; + endpoint: string; + params: Record; + }> { + const { client, as } = await this.getConfig(); + + const { challenge_supported, challenge, challenge_method } = await this.getCodeChallenge( + as, + options.state + ); + + if (!as.authorization_endpoint) { + throw new Error("authorization_endpoint is not provided"); + } + + const scopes = options.scopes ?? as.scopes_supported; + if (!Array.isArray(scopes) || scopes.length === 0) { + throw new Error("No scopes provided"); + } + + if (scopes.every((scope) => !as.scopes_supported?.includes(scope))) { + throw new Error("Invalid scopes provided"); + } + + const endpoint = as.authorization_endpoint!; + const params: any = { + client_id: client.client_id, + redirect_uri: options.redirect_uri, + response_type: "code", + scope: scopes.join(as.scope_separator ?? " ") + }; + if (challenge_supported) { + params.code_challenge = challenge; + params.code_challenge_method = challenge_method; + } else { + params.nonce = options.state; + } + + return { + url: new URL(endpoint) + "?" + new URLSearchParams(params).toString(), + endpoint, + params + }; + } + + private async oidc( + callbackParams: URL | URLSearchParams, + options: { redirect_uri: string; state: string; scopes?: string[] } + ) { + const config = await this.getConfig(); + const { client, as, type } = config; + //console.log("config", config); + //console.log("callbackParams", callbackParams, options); + const parameters = oauth.validateAuthResponse( + as, + client, // no client_secret required + callbackParams, + oauth.expectNoState + ); + if (oauth.isOAuth2Error(parameters)) { + //console.log("callback.error", parameters); + throw new OAuthCallbackException(parameters, "validateAuthResponse"); + } + /*console.log( + "callback.parameters", + JSON.stringify(Object.fromEntries(parameters.entries()), null, 2), + );*/ + const response = await oauth.authorizationCodeGrantRequest( + as, + client, + parameters, + options.redirect_uri, + options.state + ); + //console.log("callback.response", response); + + const challenges = oauth.parseWwwAuthenticateChallenges(response); + if (challenges) { + for (const challenge of challenges) { + //console.log("callback.challenge", challenge); + } + // @todo: Handle www-authenticate challenges as needed + throw new OAuthCallbackException(challenges, "www-authenticate"); + } + + const { challenge_supported, challenge } = await this.getCodeChallenge(as, options.state); + + const expectedNonce = challenge_supported ? undefined : challenge; + const result = await oauth.processAuthorizationCodeOpenIDResponse( + as, + client, + response, + expectedNonce + ); + if (oauth.isOAuth2Error(result)) { + //console.log("callback.error", result); + // @todo: Handle OAuth 2.0 response body error + throw new OAuthCallbackException(result, "processAuthorizationCodeOpenIDResponse"); + } + + //console.log("callback.result", result); + + const claims = oauth.getValidatedIdTokenClaims(result); + //console.log("callback.IDTokenClaims", claims); + + const infoRequest = await oauth.userInfoRequest(as, client, result.access_token!); + + const resultUser = await oauth.processUserInfoResponse(as, client, claims.sub, infoRequest); + //console.log("callback.resultUser", resultUser); + + return await config.profile(resultUser, config, claims); // @todo: check claims + } + + private async oauth2( + callbackParams: URL | URLSearchParams, + options: { redirect_uri: string; state: string; scopes?: string[] } + ) { + const config = await this.getConfig(); + const { client, type, as, profile } = config; + console.log("config", { client, as, type }); + console.log("callbackParams", callbackParams, options); + const parameters = oauth.validateAuthResponse( + as, + client, // no client_secret required + callbackParams, + oauth.expectNoState + ); + if (oauth.isOAuth2Error(parameters)) { + console.log("callback.error", parameters); + throw new OAuthCallbackException(parameters, "validateAuthResponse"); + } + console.log( + "callback.parameters", + JSON.stringify(Object.fromEntries(parameters.entries()), null, 2) + ); + const response = await oauth.authorizationCodeGrantRequest( + as, + client, + parameters, + options.redirect_uri, + options.state + ); + + const challenges = oauth.parseWwwAuthenticateChallenges(response); + if (challenges) { + for (const challenge of challenges) { + //console.log("callback.challenge", challenge); + } + // @todo: Handle www-authenticate challenges as needed + throw new OAuthCallbackException(challenges, "www-authenticate"); + } + + // slack does not return valid "token_type"... + const copy = response.clone(); + let result: any = {}; + try { + result = await oauth.processAuthorizationCodeOAuth2Response(as, client, response); + if (oauth.isOAuth2Error(result)) { + console.log("error", result); + throw new Error(); // Handle OAuth 2.0 response body error + } + } catch (e) { + result = (await copy.json()) as any; + console.log("failed", result); + } + + const res2 = await oauth.userInfoRequest(as, client, result.access_token!); + const user = await res2.json(); + console.log("res2", res2, user); + + console.log("result", result); + return await config.profile(user, config, result); + } + + async callback( + callbackParams: URL | URLSearchParams, + options: { redirect_uri: string; state: string; scopes?: string[] } + ): Promise { + const type = this.getIssuerConfig().type; + + console.log("type", type); + switch (type) { + case "oidc": + return await this.oidc(callbackParams, options); + case "oauth2": + return await this.oauth2(callbackParams, options); + default: + throw new Error("Unsupported type"); + } + } + + getController(auth: Authenticator): Hono { + const hono = new Hono(); + const secret = "secret"; + const cookie_name = "_challenge"; + + const setState = async ( + c: Context, + config: { state: string; action: AuthAction; redirect?: string } + ): Promise => { + await setSignedCookie(c, cookie_name, JSON.stringify(config), secret, { + secure: true, + httpOnly: true, + sameSite: "Lax", + maxAge: 60 * 5 // 5 minutes + }); + }; + + const getState = async ( + c: Context + ): Promise<{ state: string; action: AuthAction; redirect?: string }> => { + const state = await getSignedCookie(c, secret, cookie_name); + try { + return JSON.parse(state as string); + } catch (e) { + throw new Error("Invalid state"); + } + }; + + hono.get("/callback", async (c) => { + const url = new URL(c.req.url); + const params = new URLSearchParams(url.search); + + const state = await getState(c); + console.log("url", url); + + const profile = await this.callback(params, { + redirect_uri: url.origin + url.pathname, + state: state.state + }); + + const { user, token } = await auth.resolve(state.action, this, profile.sub, profile); + console.log("******** RESOLVED ********", { user, token }); + + if (state.redirect) { + console.log("redirect to", state.redirect + "?token=" + token); + return c.redirect(state.redirect + "?token=" + token); + } + + return c.json({ user, token }); + }); + + hono.get("/:action", async (c) => { + const action = c.req.param("action") as AuthAction; + if (!["login", "register"].includes(action)) { + return c.notFound(); + } + + const url = new URL(c.req.url); + const path = url.pathname.replace(`/${action}`, ""); + const redirect_uri = url.origin + path + "/callback"; + const q_redirect = (c.req.query("redirect") as string) ?? undefined; + + const state = await oauth.generateRandomCodeVerifier(); + const response = await this.request({ + redirect_uri, + state + }); + //console.log("_state", state); + + await setState(c, { state, action, redirect: q_redirect }); + + if (c.req.header("Accept") === "application/json") { + return c.json({ + url: response.url, + redirect_uri, + challenge: state, + params: response.params + }); + } + + //return c.text(response.url); + console.log("--redirecting to", response.url); + + return c.redirect(response.url); + }); + + return hono; + } + + getType() { + return "oauth"; + } + + getMode() { + return "external" as const; + } + + getName() { + return this.config.name; + } + + getSchema() { + return schemaProvided; + } + + toJSON(secrets?: boolean) { + const config = secrets ? this.config : filterKeys(this.config, ["secret", "client_id"]); + + return { + type: this.getType(), + config: { + type: this.getIssuerConfig().type, + ...config + } + }; + } +} diff --git a/app/src/auth/authenticate/strategies/oauth/issuers/github.ts b/app/src/auth/authenticate/strategies/oauth/issuers/github.ts new file mode 100644 index 0000000..4f98b9f --- /dev/null +++ b/app/src/auth/authenticate/strategies/oauth/issuers/github.ts @@ -0,0 +1,63 @@ +import type { IssuerConfig } from "../OAuthStrategy"; + +type GithubUserInfo = { + id: number; + sub: string; + name: string; + email: null; + avatar_url: string; +}; + +type GithubUserEmailResponse = { + email: string; + primary: boolean; + verified: boolean; + visibility: string; +}[]; + +export const github: IssuerConfig = { + type: "oauth2", + client: { + token_endpoint_auth_method: "client_secret_basic", + }, + as: { + code_challenge_methods_supported: ["S256"], + issuer: "https://github.com", + scopes_supported: ["read:user", "user:email"], + scope_separator: " ", + authorization_endpoint: "https://github.com/login/oauth/authorize", + token_endpoint: "https://github.com/login/oauth/access_token", + userinfo_endpoint: "https://api.github.com/user", + }, + profile: async ( + info: GithubUserInfo, + config: Omit, + tokenResponse: any, + ) => { + console.log("github info", info, config, tokenResponse); + + try { + const res = await fetch("https://api.github.com/user/emails", { + headers: { + "User-Agent": "bknd", // this is mandatory... *smh* + Accept: "application/json", + Authorization: `Bearer ${tokenResponse.access_token}`, + }, + }); + const data = (await res.json()) as GithubUserEmailResponse; + console.log("data", data); + const email = data.find((e: any) => e.primary)?.email; + if (!email) { + throw new Error("No primary email found"); + } + + return { + ...info, + sub: String(info.id), + email: email, + }; + } catch (e) { + throw new Error("Couldn't retrive github email"); + } + }, +}; diff --git a/app/src/auth/authenticate/strategies/oauth/issuers/google.ts b/app/src/auth/authenticate/strategies/oauth/issuers/google.ts new file mode 100644 index 0000000..280c99b --- /dev/null +++ b/app/src/auth/authenticate/strategies/oauth/issuers/google.ts @@ -0,0 +1,29 @@ +import type { IssuerConfig } from "../OAuthStrategy"; + +type GoogleUserInfo = { + sub: string; + name: string; + given_name: string; + family_name: string; + picture: string; + email: string; + email_verified: boolean; + locale: string; +}; + +export const google: IssuerConfig = { + type: "oidc", + client: { + token_endpoint_auth_method: "client_secret_basic", + }, + as: { + issuer: "https://accounts.google.com", + }, + profile: async (info) => { + return { + ...info, + sub: info.sub, + email: info.email, + }; + }, +}; diff --git a/app/src/auth/authenticate/strategies/oauth/issuers/index.ts b/app/src/auth/authenticate/strategies/oauth/issuers/index.ts new file mode 100644 index 0000000..ab5ace1 --- /dev/null +++ b/app/src/auth/authenticate/strategies/oauth/issuers/index.ts @@ -0,0 +1,2 @@ +export { google } from "./google"; +export { github } from "./github"; diff --git a/app/src/auth/authorize/Guard.ts b/app/src/auth/authorize/Guard.ts new file mode 100644 index 0000000..caae555 --- /dev/null +++ b/app/src/auth/authorize/Guard.ts @@ -0,0 +1,160 @@ +import { Exception, Permission } from "core"; +import { type Static, Type, objectTransform } from "core/utils"; +import { Role } from "./Role"; + +export type GuardUserContext = { + role: string | null | undefined; + [key: string]: any; +}; + +export type GuardConfig = { + enabled?: boolean; +}; + +export class Guard { + permissions: Permission[]; + user?: GuardUserContext; + roles?: Role[]; + config?: GuardConfig; + + constructor(permissions: Permission[] = [], roles: Role[] = [], config?: GuardConfig) { + this.permissions = permissions; + this.roles = roles; + this.config = config; + } + + static create( + permissionNames: string[], + roles?: Record< + string, + { + permissions?: string[]; + is_default?: boolean; + implicit_allow?: boolean; + } + >, + config?: GuardConfig + ) { + const _roles = roles + ? objectTransform(roles, ({ permissions = [], is_default, implicit_allow }, name) => { + return Role.createWithPermissionNames(name, permissions, is_default, implicit_allow); + }) + : {}; + const _permissions = permissionNames.map((name) => new Permission(name)); + return new Guard(_permissions, Object.values(_roles), config); + } + + getPermissionNames(): string[] { + return this.permissions.map((permission) => permission.name); + } + + getPermissions(): Permission[] { + return this.permissions; + } + + permissionExists(permissionName: string): boolean { + return !!this.permissions.find((p) => p.name === permissionName); + } + + setRoles(roles: Role[]) { + this.roles = roles; + return this; + } + + getRoles() { + return this.roles; + } + + setConfig(config: Partial) { + this.config = { ...this.config, ...config }; + return this; + } + + registerPermission(permission: Permission) { + if (this.permissions.find((p) => p.name === permission.name)) { + throw new Error(`Permission ${permission.name} already exists`); + } + + this.permissions.push(permission); + return this; + } + + registerPermissions(permissions: Permission[]) { + for (const permission of permissions) { + this.registerPermission(permission); + } + + return this; + } + + setUserContext(user: GuardUserContext | undefined) { + this.user = user; + return this; + } + + getUserRole(): Role | undefined { + if (this.user && typeof this.user.role === "string") { + const role = this.roles?.find((role) => role.name === this.user?.role); + if (role) { + console.log("guard: role found", this.user.role); + return role; + } + } + + console.log("guard: role not found", this.user, this.user?.role); + return this.getDefaultRole(); + } + + getDefaultRole(): Role | undefined { + return this.roles?.find((role) => role.is_default); + } + + hasPermission(permission: Permission): boolean; + hasPermission(name: string): boolean; + hasPermission(permissionOrName: Permission | string): boolean { + if (this.config?.enabled !== true) { + //console.log("guard not enabled, allowing"); + return true; + } + + const name = typeof permissionOrName === "string" ? permissionOrName : permissionOrName.name; + const exists = this.permissionExists(name); + if (!exists) { + throw new Error(`Permission ${name} does not exist`); + } + + const role = this.getUserRole(); + + if (!role) { + console.log("guard: role not found, denying"); + return false; + } else if (role.implicit_allow === true) { + console.log("guard: role implicit allow, allowing"); + return true; + } + + const rolePermission = role.permissions.find( + (rolePermission) => rolePermission.permission.name === name + ); + + console.log("guard: rolePermission, allowing?", { + permission: name, + role: role.name, + allowing: !!rolePermission + }); + return !!rolePermission; + } + + granted(permission: Permission | string): boolean { + return this.hasPermission(permission as any); + } + + throwUnlessGranted(permission: Permission | string) { + if (!this.granted(permission)) { + throw new Exception( + `Permission "${typeof permission === "string" ? permission : permission.name}" not granted`, + 403 + ); + } + } +} diff --git a/app/src/auth/authorize/Role.ts b/app/src/auth/authorize/Role.ts new file mode 100644 index 0000000..6d1e0c5 --- /dev/null +++ b/app/src/auth/authorize/Role.ts @@ -0,0 +1,45 @@ +import { Permission } from "core"; + +export class RolePermission { + constructor( + public permission: Permission, + public config?: any + ) {} +} + +export class Role { + constructor( + public name: string, + public permissions: RolePermission[] = [], + public is_default: boolean = false, + public implicit_allow: boolean = false + ) {} + + static createWithPermissionNames( + name: string, + permissionNames: string[], + is_default: boolean = false, + implicit_allow: boolean = false + ) { + return new Role( + name, + permissionNames.map((name) => new RolePermission(new Permission(name))), + is_default, + implicit_allow + ); + } + + static create(config: { + name: string; + permissions?: string[]; + is_default?: boolean; + implicit_allow?: boolean; + }) { + return new Role( + config.name, + config.permissions?.map((name) => new RolePermission(new Permission(name))) ?? [], + config.is_default, + config.implicit_allow + ); + } +} diff --git a/app/src/auth/errors.ts b/app/src/auth/errors.ts new file mode 100644 index 0000000..e10266e --- /dev/null +++ b/app/src/auth/errors.ts @@ -0,0 +1,28 @@ +import { Exception } from "core"; + +export class UserExistsException extends Exception { + override name = "UserExistsException"; + override code = 422; + + constructor() { + super("User already exists"); + } +} + +export class UserNotFoundException extends Exception { + override name = "UserNotFoundException"; + override code = 404; + + constructor() { + super("User not found"); + } +} + +export class InvalidCredentialsException extends Exception { + override name = "InvalidCredentialsException"; + override code = 401; + + constructor() { + super("Invalid credentials"); + } +} diff --git a/app/src/auth/index.ts b/app/src/auth/index.ts new file mode 100644 index 0000000..fbb47fb --- /dev/null +++ b/app/src/auth/index.ts @@ -0,0 +1,21 @@ +export { UserExistsException, UserNotFoundException, InvalidCredentialsException } from "./errors"; +export { sha256 } from "./utils/hash"; +export { + type ProfileExchange, + type Strategy, + type User, + type SafeUser, + type CreateUser, + type AuthResponse, + type UserPool, + type AuthAction, + type AuthUserResolver, + Authenticator, + authenticatorConfig, + jwtConfig +} from "./authenticate/Authenticator"; + +export { AppAuth, type UserFieldSchema } from "./AppAuth"; + +export { Guard, type GuardUserContext, type GuardConfig } from "./authorize/Guard"; +export { Role } from "./authorize/Role"; diff --git a/app/src/auth/utils/hash.ts b/app/src/auth/utils/hash.ts new file mode 100644 index 0000000..5056938 --- /dev/null +++ b/app/src/auth/utils/hash.ts @@ -0,0 +1,13 @@ +// @deprecated: moved to @bknd/core +export async function sha256(password: string, salt?: string) { + // 1. Convert password to Uint8Array + const encoder = new TextEncoder(); + const data = encoder.encode((salt ?? "") + password); + + // 2. Hash the data using SHA-256 + const hashBuffer = await crypto.subtle.digest("SHA-256", data); + + // 3. Convert hash to hex string for easier display + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} diff --git a/app/src/cli/commands/config.ts b/app/src/cli/commands/config.ts new file mode 100644 index 0000000..461ee78 --- /dev/null +++ b/app/src/cli/commands/config.ts @@ -0,0 +1,12 @@ +import { getDefaultConfig } from "modules/ModuleManager"; +import type { CliCommand } from "../types"; + +export const config: CliCommand = (program) => { + program + .command("config") + .description("get default config") + .option("--pretty", "pretty print") + .action((options) => { + console.log(getDefaultConfig(options.pretty)); + }); +}; diff --git a/app/src/cli/commands/debug.ts b/app/src/cli/commands/debug.ts new file mode 100644 index 0000000..124d7d2 --- /dev/null +++ b/app/src/cli/commands/debug.ts @@ -0,0 +1,20 @@ +import path from "node:path"; +import url from "node:url"; +import { getDistPath, getRelativeDistPath, getRootPath } from "cli/utils/sys"; +import type { CliCommand } from "../types"; + +export const debug: CliCommand = (program) => { + program + .command("debug") + .description("debug path resolution") + .action(() => { + console.log("paths", { + rootpath: getRootPath(), + distPath: getDistPath(), + relativeDistPath: getRelativeDistPath(), + cwd: process.cwd(), + dir: path.dirname(url.fileURLToPath(import.meta.url)), + resolvedPkg: path.resolve(getRootPath(), "package.json") + }); + }); +}; diff --git a/app/src/cli/commands/index.ts b/app/src/cli/commands/index.ts new file mode 100644 index 0000000..dfe7189 --- /dev/null +++ b/app/src/cli/commands/index.ts @@ -0,0 +1,5 @@ +export { config } from "./config"; +export { schema } from "./schema"; +export { run } from "./run"; +export { debug } from "./debug"; +export { user } from "./user"; diff --git a/app/src/cli/commands/run/index.ts b/app/src/cli/commands/run/index.ts new file mode 100644 index 0000000..11d8ba3 --- /dev/null +++ b/app/src/cli/commands/run/index.ts @@ -0,0 +1 @@ +export * from "./run"; diff --git a/app/src/cli/commands/run/platform.ts b/app/src/cli/commands/run/platform.ts new file mode 100644 index 0000000..76c033e --- /dev/null +++ b/app/src/cli/commands/run/platform.ts @@ -0,0 +1,96 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { ServeStaticOptions } from "@hono/node-server/serve-static"; +import { type Config, createClient } from "@libsql/client/node"; +import { Connection, LibsqlConnection, SqliteLocalConnection } from "data"; +import type { MiddlewareHandler } from "hono"; +import { fileExists, getDistPath, getRelativeDistPath } from "../../utils/sys"; + +export const PLATFORMS = ["node", "bun"] as const; +export type Platform = (typeof PLATFORMS)[number]; + +export async function serveStatic(server: Platform): Promise { + switch (server) { + case "node": { + const m = await import("@hono/node-server/serve-static"); + return m.serveStatic({ + // somehow different for node + root: getRelativeDistPath() + "/static" + }); + } + case "bun": { + const m = await import("hono/bun"); + return m.serveStatic({ + root: path.resolve(getRelativeDistPath(), "static") + }); + } + } +} + +export async function attachServeStatic(app: any, platform: Platform) { + app.module.server.client.get("/assets/*", await serveStatic(platform)); +} + +export async function startServer(server: Platform, app: any, options: { port: number }) { + const port = options.port; + console.log("running on", server, port); + switch (server) { + case "node": { + // https://github.com/honojs/node-server/blob/main/src/response.ts#L88 + const serve = await import("@hono/node-server").then((m) => m.serve); + serve({ + fetch: (req) => app.fetch(req), + port + }); + break; + } + case "bun": { + Bun.serve({ + fetch: (req) => app.fetch(req), + port + }); + break; + } + } + + console.log("Server listening on", "http://localhost:" + port); +} + +export async function getHtml() { + return await readFile(path.resolve(getDistPath(), "static/index.html"), "utf-8"); +} + +export function getConnection(connectionOrConfig?: Connection | Config): Connection { + if (connectionOrConfig) { + if (connectionOrConfig instanceof Connection) { + return connectionOrConfig; + } + + if ("url" in connectionOrConfig) { + return new LibsqlConnection(createClient(connectionOrConfig)); + } + } + + console.log("Using in-memory database"); + return new LibsqlConnection(createClient({ url: ":memory:" })); + //return new SqliteLocalConnection(new Database(":memory:")); +} + +export async function getConfigPath(filePath?: string) { + if (filePath) { + const config_path = path.resolve(process.cwd(), filePath); + if (await fileExists(config_path)) { + return config_path; + } + } + + const paths = ["./bknd.config", "./bknd.config.ts", "./bknd.config.js"]; + for (const p of paths) { + const _p = path.resolve(process.cwd(), p); + if (await fileExists(_p)) { + return _p; + } + } + + return; +} diff --git a/app/src/cli/commands/run/run.ts b/app/src/cli/commands/run/run.ts new file mode 100644 index 0000000..14730f0 --- /dev/null +++ b/app/src/cli/commands/run/run.ts @@ -0,0 +1,115 @@ +import type { Config } from "@libsql/client/node"; +import { App } from "App"; +import type { BkndConfig } from "adapter"; +import { Option } from "commander"; +import type { Connection } from "data"; +import type { CliCommand } from "../../types"; +import { + PLATFORMS, + type Platform, + attachServeStatic, + getConfigPath, + getConnection, + getHtml, + startServer +} from "./platform"; + +const isBun = typeof Bun !== "undefined"; + +export const run: CliCommand = (program) => { + program + .command("run") + .addOption( + new Option("-p, --port ", "port to run on") + .env("PORT") + .default(1337) + .argParser((v) => Number.parseInt(v)) + ) + .addOption(new Option("-c, --config ", "config file")) + .addOption( + new Option("--db-url ", "database url, can be any valid libsql url").conflicts( + "config" + ) + ) + .addOption(new Option("--db-token ", "database token").conflicts("config")) + .addOption( + new Option("--server ", "server type") + .choices(PLATFORMS) + .default(isBun ? "bun" : "node") + ) + .action(action); +}; + +type MakeAppConfig = { + connection: Connection; + server?: { platform?: Platform }; + setAdminHtml?: boolean; + onBuilt?: (app: App) => Promise; +}; + +async function makeApp(config: MakeAppConfig) { + const html = await getHtml(); + const app = new App(config.connection); + + app.emgr.on( + "app-built", + async () => { + await attachServeStatic(app, config.server?.platform ?? "node"); + app.module.server.setAdminHtml(html); + + if (config.onBuilt) { + await config.onBuilt(app); + } + }, + "sync" + ); + + await app.build(); + return app; +} + +export async function makeConfigApp(config: BkndConfig, platform?: Platform) { + const appConfig = typeof config.app === "function" ? config.app(process.env) : config.app; + const html = await getHtml(); + const app = App.create(appConfig); + + app.emgr.on( + "app-built", + async () => { + await attachServeStatic(app, platform ?? "node"); + app.module.server.setAdminHtml(html); + + if (config.onBuilt) { + await config.onBuilt(app); + } + }, + "sync" + ); + + await app.build(); + return app; +} + +async function action(options: { + port: number; + config?: string; + dbUrl?: string; + dbToken?: string; + server: Platform; +}) { + const configFilePath = await getConfigPath(options.config); + + let app: App; + if (options.dbUrl || !configFilePath) { + const connection = getConnection( + options.dbUrl ? { url: options.dbUrl, authToken: options.dbToken } : undefined + ); + app = await makeApp({ connection, server: { platform: options.server } }); + } else { + console.log("Using config from:", configFilePath); + const config = (await import(configFilePath).then((m) => m.default)) as BkndConfig; + app = await makeConfigApp(config, options.server); + } + + await startServer(options.server, app, { port: options.port }); +} diff --git a/app/src/cli/commands/schema.ts b/app/src/cli/commands/schema.ts new file mode 100644 index 0000000..8c59d7e --- /dev/null +++ b/app/src/cli/commands/schema.ts @@ -0,0 +1,12 @@ +import { getDefaultSchema } from "modules/ModuleManager"; +import type { CliCommand } from "../types"; + +export const schema: CliCommand = (program) => { + program + .command("schema") + .description("get schema") + .option("--pretty", "pretty print") + .action((options) => { + console.log(getDefaultSchema(options.pretty)); + }); +}; diff --git a/app/src/cli/commands/user.ts b/app/src/cli/commands/user.ts new file mode 100644 index 0000000..06b5296 --- /dev/null +++ b/app/src/cli/commands/user.ts @@ -0,0 +1,144 @@ +import { password as $password, text as $text } from "@clack/prompts"; +import type { PasswordStrategy } from "auth/authenticate/strategies"; +import type { App, BkndConfig } from "bknd"; +import { makeConfigApp } from "cli/commands/run"; +import { getConfigPath } from "cli/commands/run/platform"; +import type { CliCommand } from "cli/types"; +import { Argument } from "commander"; + +export const user: CliCommand = (program) => { + program + .command("user") + .description("create and update user (auth)") + .addArgument(new Argument("", "action to perform").choices(["create", "update"])) + .action(action); +}; + +async function action(action: "create" | "update", options: any) { + const configFilePath = await getConfigPath(); + if (!configFilePath) { + console.error("config file not found"); + return; + } + + const config = (await import(configFilePath).then((m) => m.default)) as BkndConfig; + const app = await makeConfigApp(config, options.server); + + switch (action) { + case "create": + await create(app, options); + break; + case "update": + await update(app, options); + break; + } +} + +async function create(app: App, options: any) { + const config = app.module.auth.toJSON(true); + const strategy = app.module.auth.authenticator.strategy("password") as PasswordStrategy; + const users_entity = config.entity_name; + + const email = await $text({ + message: "Enter email", + validate: (v) => { + if (!v.includes("@")) { + return "Invalid email"; + } + return; + } + }); + + const password = await $password({ + message: "Enter password", + validate: (v) => { + if (v.length < 3) { + return "Invalid password"; + } + return; + } + }); + + if (typeof email !== "string" || typeof password !== "string") { + console.log("Cancelled"); + process.exit(0); + } + + try { + const mutator = app.modules.ctx().em.mutator(users_entity); + mutator.__unstable_toggleSystemEntityCreation(true); + const res = await mutator.insertOne({ + email, + strategy: "password", + strategy_value: await strategy.hash(password as string) + }); + mutator.__unstable_toggleSystemEntityCreation(false); + + console.log("Created:", res.data); + } catch (e) { + console.error("Error"); + } +} + +async function update(app: App, options: any) { + const config = app.module.auth.toJSON(true); + const strategy = app.module.auth.authenticator.strategy("password") as PasswordStrategy; + const users_entity = config.entity_name; + const em = app.modules.ctx().em; + + const email = (await $text({ + message: "Which user? Enter email", + validate: (v) => { + if (!v.includes("@")) { + return "Invalid email"; + } + return; + } + })) as string; + if (typeof email !== "string") { + console.log("Cancelled"); + process.exit(0); + } + + const { data: user } = await em.repository(users_entity).findOne({ email }); + if (!user) { + console.log("User not found"); + process.exit(0); + } + console.log("User found:", user); + + const password = await $password({ + message: "New Password?", + validate: (v) => { + if (v.length < 3) { + return "Invalid password"; + } + return; + } + }); + if (typeof password !== "string") { + console.log("Cancelled"); + process.exit(0); + } + + try { + function togglePw(visible: boolean) { + const field = em.entity(users_entity).field("strategy_value")!; + + field.config.hidden = !visible; + field.config.fillable = visible; + } + togglePw(true); + await app.modules + .ctx() + .em.mutator(users_entity) + .updateOne(user.id, { + strategy_value: await strategy.hash(password as string) + }); + togglePw(false); + + console.log("Updated:", user); + } catch (e) { + console.error("Error", e); + } +} diff --git a/app/src/cli/index.ts b/app/src/cli/index.ts new file mode 100644 index 0000000..d7c7ef5 --- /dev/null +++ b/app/src/cli/index.ts @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import { Command } from "commander"; +import * as commands from "./commands"; +import { getVersion } from "./utils/sys"; +const program = new Command(); + +export async function main() { + program + .name("bknd") + .description("bknd cli") + .version(await getVersion()); + + // register commands + for (const command of Object.values(commands)) { + command(program); + } + + program.parse(); +} + +main().then(null).catch(console.error); diff --git a/app/src/cli/types.d.ts b/app/src/cli/types.d.ts new file mode 100644 index 0000000..6dd97aa --- /dev/null +++ b/app/src/cli/types.d.ts @@ -0,0 +1,3 @@ +import type { Command } from "commander"; + +export type CliCommand = (program: Command) => void; diff --git a/app/src/cli/utils/sys.ts b/app/src/cli/utils/sys.ts new file mode 100644 index 0000000..ab61fdd --- /dev/null +++ b/app/src/cli/utils/sys.ts @@ -0,0 +1,40 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import url from "node:url"; + +export function getRootPath() { + const _path = path.dirname(url.fileURLToPath(import.meta.url)); + // because of "src", local needs one more level up + return path.resolve(_path, process.env.LOCAL ? "../../../" : "../../"); +} + +export function getDistPath() { + return path.resolve(getRootPath(), "dist"); +} + +export function getRelativeDistPath() { + return path.relative(process.cwd(), getDistPath()); +} + +export async function getVersion() { + try { + const resolved = path.resolve(getRootPath(), "package.json"); + const pkg = await readFile(resolved, "utf-8"); + if (pkg) { + return JSON.parse(pkg).version ?? "preview"; + } + } catch (e) { + console.error("Failed to resolve version"); + } + + return "unknown"; +} + +export async function fileExists(filePath: string) { + try { + await readFile(path.resolve(process.cwd(), filePath)); + return true; + } catch { + return false; + } +} diff --git a/app/src/core/cache/adapters/CloudflareKvCache.ts b/app/src/core/cache/adapters/CloudflareKvCache.ts new file mode 100644 index 0000000..61b71f3 --- /dev/null +++ b/app/src/core/cache/adapters/CloudflareKvCache.ts @@ -0,0 +1,127 @@ +import type { ICacheItem, ICachePool } from "../cache-interface"; + +export class CloudflareKVCachePool implements ICachePool { + constructor(private namespace: KVNamespace) {} + + supports = () => ({ + metadata: true, + clear: false, + }); + + async get(key: string): Promise> { + const result = await this.namespace.getWithMetadata(key); + const hit = result.value !== null && typeof result.value !== "undefined"; + // Assuming metadata is not supported directly; + // you may adjust if Cloudflare KV supports it in future. + return new CloudflareKVCacheItem(key, result.value ?? undefined, hit, result.metadata) as any; + } + + async getMany(keys: string[] = []): Promise>> { + const items = new Map>(); + await Promise.all( + keys.map(async (key) => { + const item = await this.get(key); + items.set(key, item); + }), + ); + return items; + } + + async has(key: string): Promise { + const data = await this.namespace.get(key); + return data !== null; + } + + async clear(): Promise { + // Cloudflare KV does not support clearing all keys in one operation + return false; + } + + async delete(key: string): Promise { + await this.namespace.delete(key); + return true; + } + + async deleteMany(keys: string[]): Promise { + const results = await Promise.all(keys.map((key) => this.delete(key))); + return results.every((result) => result); + } + + async save(item: CloudflareKVCacheItem): Promise { + await this.namespace.put(item.key(), (await item.value()) as string, { + expirationTtl: item._expirationTtl, + metadata: item.metadata(), + }); + + return true; + } + + async put( + key: string, + value: any, + options?: { ttl?: number; expiresAt?: Date; metadata?: Record }, + ): Promise { + const item = new CloudflareKVCacheItem(key, value, true, options?.metadata); + + if (options?.expiresAt) item.expiresAt(options.expiresAt); + if (options?.ttl) item.expiresAfter(options.ttl); + + return await this.save(item); + } +} + +export class CloudflareKVCacheItem implements ICacheItem { + _expirationTtl: number | undefined; + + constructor( + private _key: string, + private data: Data | undefined, + private _hit: boolean = false, + private _metadata: Record = {}, + ) {} + + key(): string { + return this._key; + } + + value(): Data | undefined { + if (this.data) { + try { + return JSON.parse(this.data as string); + } catch (e) {} + } + + return this.data ?? undefined; + } + + metadata(): Record { + return this._metadata; + } + + hit(): boolean { + return this._hit; + } + + set(value: Data, metadata: Record = {}): this { + this.data = value; + this._metadata = metadata; + return this; + } + + expiresAt(expiration: Date | null): this { + // Cloudflare KV does not support specific date expiration; calculate ttl instead. + if (expiration) { + const now = new Date(); + const ttl = (expiration.getTime() - now.getTime()) / 1000; + return this.expiresAfter(Math.max(0, Math.floor(ttl))); + } + return this.expiresAfter(null); + } + + expiresAfter(time: number | null): this { + // Dummy implementation as Cloudflare KV requires setting expiration during PUT operation. + // This method will be effectively implemented in the Cache Pool save methods. + this._expirationTtl = time ?? undefined; + return this; + } +} diff --git a/app/src/core/cache/adapters/MemoryCache.ts b/app/src/core/cache/adapters/MemoryCache.ts new file mode 100644 index 0000000..75e0df7 --- /dev/null +++ b/app/src/core/cache/adapters/MemoryCache.ts @@ -0,0 +1,139 @@ +import type { ICacheItem, ICachePool } from "../cache-interface"; + +export class MemoryCache implements ICachePool { + private cache: Map> = new Map(); + private maxSize?: number; + + constructor(options?: { maxSize?: number }) { + this.maxSize = options?.maxSize; + } + + supports = () => ({ + metadata: true, + clear: true + }); + + async get(key: string): Promise> { + if (!this.cache.has(key)) { + // use undefined to denote a miss initially + return new MemoryCacheItem(key, undefined!); + } + return this.cache.get(key)!; + } + + async getMany(keys: string[] = []): Promise>> { + const items = new Map>(); + for (const key of keys) { + items.set(key, await this.get(key)); + } + return items; + } + + async has(key: string): Promise { + return this.cache.has(key) && this.cache.get(key)!.hit(); + } + + async clear(): Promise { + this.cache.clear(); + return true; + } + + async delete(key: string): Promise { + return this.cache.delete(key); + } + + async deleteMany(keys: string[]): Promise { + let success = true; + for (const key of keys) { + if (!this.delete(key)) { + success = false; + } + } + return success; + } + + async save(item: MemoryCacheItem): Promise { + this.checkSizeAndPurge(); + this.cache.set(item.key(), item); + return true; + } + + async put( + key: string, + value: Data, + options: { expiresAt?: Date; ttl?: number; metadata?: Record } = {} + ): Promise { + const item = await this.get(key); + item.set(value, options.metadata || {}); + if (options.expiresAt) { + item.expiresAt(options.expiresAt); + } else if (typeof options.ttl === "number") { + item.expiresAfter(options.ttl); + } + return this.save(item); + } + + private checkSizeAndPurge(): void { + if (!this.maxSize) return; + + if (this.cache.size >= this.maxSize) { + // Implement logic to purge items, e.g., LRU (Least Recently Used) + // For simplicity, clear the oldest item inserted + const keyToDelete = this.cache.keys().next().value; + this.cache.delete(keyToDelete!); + } + } +} + +export class MemoryCacheItem implements ICacheItem { + private _key: string; + private _value: Data | undefined; + private expiration: Date | null = null; + private _metadata: Record = {}; + + constructor(key: string, value: Data, metadata: Record = {}) { + this._key = key; + this.set(value, metadata); + } + + key(): string { + return this._key; + } + + metadata(): Record { + return this._metadata; + } + + value(): Data | undefined { + return this._value; + } + + hit(): boolean { + if (this.expiration !== null && new Date() > this.expiration) { + return false; + } + return this.value() !== undefined; + } + + set(value: Data, metadata: Record = {}): this { + this._value = value; + this._metadata = metadata; + return this; + } + + expiresAt(expiration: Date | null): this { + this.expiration = expiration; + return this; + } + + expiresAfter(time: number | null): this { + if (typeof time === "number") { + const expirationDate = new Date(); + expirationDate.setSeconds(expirationDate.getSeconds() + time); + this.expiration = expirationDate; + } else { + this.expiration = null; + } + return this; + } +} diff --git a/app/src/core/cache/cache-interface.ts b/app/src/core/cache/cache-interface.ts new file mode 100644 index 0000000..c6e099e --- /dev/null +++ b/app/src/core/cache/cache-interface.ts @@ -0,0 +1,178 @@ +/** + * CacheItem defines an interface for interacting with objects inside a cache. + * based on https://www.php-fig.org/psr/psr-6/ + */ +export interface ICacheItem { + /** + * Returns the key for the current cache item. + * + * The key is loaded by the Implementing Library, but should be available to + * the higher level callers when needed. + * + * @returns The key string for this cache item. + */ + key(): string; + + /** + * Retrieves the value of the item from the cache associated with this object's key. + * + * The value returned must be identical to the value originally stored by set(). + * + * If isHit() returns false, this method MUST return null. Note that null + * is a legitimate cached value, so the isHit() method SHOULD be used to + * differentiate between "null value was found" and "no value was found." + * + * @returns The value corresponding to this cache item's key, or undefined if not found. + */ + value(): Data | undefined; + + /** + * Retrieves the metadata of the item from the cache associated with this object's key. + */ + metadata(): Record; + + /** + * Confirms if the cache item lookup resulted in a cache hit. + * + * Note: This method MUST NOT have a race condition between calling isHit() + * and calling get(). + * + * @returns True if the request resulted in a cache hit. False otherwise. + */ + hit(): boolean; + + /** + * Sets the value represented by this cache item. + * + * The value argument may be any item that can be serialized by PHP, + * although the method of serialization is left up to the Implementing + * Library. + * + * @param value The serializable value to be stored. + * @param metadata The metadata to be associated with the item. + * @returns The invoked object. + */ + set(value: Data, metadata?: Record): this; + + /** + * Sets the expiration time for this cache item. + * + * @param expiration The point in time after which the item MUST be considered expired. + * If null is passed explicitly, a default value MAY be used. If none is set, + * the value should be stored permanently or for as long as the + * implementation allows. + * @returns The called object. + */ + expiresAt(expiration: Date | null): this; + + /** + * Sets the expiration time for this cache item. + * + * @param time The period of time from the present after which the item MUST be considered + * expired. An integer parameter is understood to be the time in seconds until + * expiration. If null is passed explicitly, a default value MAY be used. + * If none is set, the value should be stored permanently or for as long as the + * implementation allows. + * @returns The called object. + */ + expiresAfter(time: number | null): this; +} + +/** + * CachePool generates CacheItem objects. + * based on https://www.php-fig.org/psr/psr-6/ + */ +export interface ICachePool { + supports(): { + metadata: boolean; + clear: boolean; + }; + + /** + * Returns a Cache Item representing the specified key. + * This method must always return a CacheItemInterface object, even in case of + * a cache miss. It MUST NOT return null. + * + * @param key The key for which to return the corresponding Cache Item. + * @throws Error If the key string is not a legal value an Error MUST be thrown. + * @returns The corresponding Cache Item. + */ + get(key: string): Promise>; + + /** + * Returns a traversable set of cache items. + * + * @param keys An indexed array of keys of items to retrieve. + * @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown. + * @returns A traversable collection of Cache Items keyed by the cache keys of + * each item. A Cache item will be returned for each key, even if that + * key is not found. However, if no keys are specified then an empty + * traversable MUST be returned instead. + */ + getMany(keys?: string[]): Promise>>; + + /** + * Confirms if the cache contains specified cache item. + * + * Note: This method MAY avoid retrieving the cached value for performance reasons. + * This could result in a race condition with CacheItemInterface.get(). To avoid + * such situation use CacheItemInterface.isHit() instead. + * + * @param key The key for which to check existence. + * @throws Error If the key string is not a legal value an Error MUST be thrown. + * @returns True if item exists in the cache, false otherwise. + */ + has(key: string): Promise; + + /** + * Deletes all items in the pool. + * @returns True if the pool was successfully cleared. False if there was an error. + */ + clear(): Promise; + + /** + * Removes the item from the pool. + * + * @param key The key to delete. + * @throws Error If the key string is not a legal value an Error MUST be thrown. + * @returns True if the item was successfully removed. False if there was an error. + */ + delete(key: string): Promise; + + /** + * Removes multiple items from the pool. + * + * @param keys An array of keys that should be removed from the pool. + * @throws Error If any of the keys in keys are not a legal value an Error MUST be thrown. + * @returns True if the items were successfully removed. False if there was an error. + */ + deleteMany(keys: string[]): Promise; + + /** + * Persists a cache item immediately. + * + * @param item The cache item to save. + * @returns True if the item was successfully persisted. False if there was an error. + */ + save(item: ICacheItem): Promise; + + /** + * Persists any deferred cache items. + * @returns True if all not-yet-saved items were successfully saved or there were none. False otherwise. + */ + put( + key: string, + value: any, + options?: { expiresAt?: Date; metadata?: Record }, + ): Promise; + put( + key: string, + value: any, + options?: { ttl?: number; metadata?: Record }, + ): Promise; + put( + key: string, + value: any, + options?: ({ ttl?: number } | { expiresAt?: Date }) & { metadata?: Record }, + ): Promise; +} diff --git a/app/src/core/clients/aws/AwsClient.ts b/app/src/core/clients/aws/AwsClient.ts new file mode 100644 index 0000000..838367f --- /dev/null +++ b/app/src/core/clients/aws/AwsClient.ts @@ -0,0 +1,96 @@ +import { AwsClient as Aws4fetchClient } from "aws4fetch"; +import { objectKeysPascalToKebab } from "../../utils/objects"; +import { xmlToObject } from "../../utils/xml"; + +type Aws4fetchClientConfig = ConstructorParameters[0]; +type AwsClientConfig = { + responseType?: "xml" | "json"; + responseKeysToUpper?: boolean; + convertParams?: "pascalToKebab"; +}; + +export class AwsClient extends Aws4fetchClient { + readonly #options: AwsClientConfig; + + constructor(aws4fetchConfig: Aws4fetchClientConfig, options?: AwsClientConfig) { + super(aws4fetchConfig); + this.#options = options ?? { + responseType: "json", + }; + } + + protected convertParams(params: Record): Record { + switch (this.#options.convertParams) { + case "pascalToKebab": + return objectKeysPascalToKebab(params); + default: + return params; + } + } + + getUrl(path: string = "/", searchParamsObj: Record = {}): string { + //console.log("super:getUrl", path, searchParamsObj); + const url = new URL(path); + const converted = this.convertParams(searchParamsObj); + Object.entries(converted).forEach(([key, value]) => { + url.searchParams.append(key, value as any); + }); + return url.toString(); + } + + protected updateKeysRecursively(obj: any, direction: "toUpperCase" | "toLowerCase") { + if (obj === null || obj === undefined) return obj; + + if (Array.isArray(obj)) { + return obj.map((item) => this.updateKeysRecursively(item, direction)); + } + + if (typeof obj === "object") { + return Object.keys(obj).reduce( + (acc, key) => { + // only if key doesn't have any whitespaces + let newKey = key; + if (key.indexOf(" ") === -1) { + newKey = key.charAt(0)[direction]() + key.slice(1); + } + acc[newKey] = this.updateKeysRecursively(obj[key], direction); + return acc; + }, + {} as { [key: string]: any }, + ); + } + + return obj; + } + + async fetchJson>( + input: RequestInfo, + init?: RequestInit, + ): Promise { + const response = await this.fetch(input, init); + + if (this.#options.responseType === "xml") { + if (!response.ok) { + const body = await response.text(); + throw new Error(body); + } + + const raw = await response.text(); + //console.log("raw", raw); + //console.log(JSON.stringify(xmlToObject(raw), null, 2)); + return xmlToObject(raw) as T; + } + + if (!response.ok) { + const body = await response.json<{ message: string }>(); + throw new Error(body.message); + } + + const raw = (await response.json()) as T; + if (this.#options.responseKeysToUpper) { + return this.updateKeysRecursively(raw, "toUpperCase"); + } + + return raw; + } +} diff --git a/app/src/core/config.ts b/app/src/core/config.ts new file mode 100644 index 0000000..445b67c --- /dev/null +++ b/app/src/core/config.ts @@ -0,0 +1,12 @@ +/** + * These are package global defaults. + */ +import type { Generated } from "kysely"; + +export type PrimaryFieldType = number | Generated; + +export const config = { + data: { + default_primary_field: "id" + } +} as const; diff --git a/app/src/core/env.ts b/app/src/core/env.ts new file mode 100644 index 0000000..386ec9d --- /dev/null +++ b/app/src/core/env.ts @@ -0,0 +1,27 @@ +type TURSO_DB = { + url: string; + authToken: string; +}; + +export type Env = { + __STATIC_CONTENT: Fetcher; + ENVIRONMENT: string; + CACHE: KVNamespace; + + // db + DB_DATA: TURSO_DB; + DB_SCHEMA: TURSO_DB; + + // storage + STORAGE: { access_key: string; secret_access_key: string; url: string }; + BUCKET: R2Bucket; +}; + +export function isDebug(): boolean { + try { + // @ts-expect-error - this is a global variable in dev + return __isDev === "1" || __isDev === 1; + } catch (e) { + return false; + } +} diff --git a/app/src/core/errors.ts b/app/src/core/errors.ts new file mode 100644 index 0000000..ce63ed9 --- /dev/null +++ b/app/src/core/errors.ts @@ -0,0 +1,37 @@ +export class Exception extends Error { + code = 400; + override name = "Exception"; + + constructor(message: string, code?: number) { + super(message); + if (code) { + this.code = code; + } + } + + toJSON() { + return { + error: this.message, + type: this.name + //message: this.message + }; + } +} + +export class BkndError extends Error { + constructor( + message: string, + public details?: Record, + public type?: string + ) { + super(message); + } + + toJSON() { + return { + type: this.type ?? "unknown", + message: this.message, + details: this.details + }; + } +} diff --git a/app/src/core/events/Event.ts b/app/src/core/events/Event.ts new file mode 100644 index 0000000..247c7a5 --- /dev/null +++ b/app/src/core/events/Event.ts @@ -0,0 +1,21 @@ +export abstract class Event { + /** + * Unique event slug + * Must be static, because registering events is done by class + */ + static slug: string = "untitled-event"; + params: Params; + + constructor(params: Params) { + this.params = params; + } +} + +// @todo: current workaround: potentially there is none and that's the way +export class NoParamEvent extends Event { + static override slug: string = "noparam-event"; + + constructor() { + super(null); + } +} diff --git a/app/src/core/events/EventListener.ts b/app/src/core/events/EventListener.ts new file mode 100644 index 0000000..951fce8 --- /dev/null +++ b/app/src/core/events/EventListener.ts @@ -0,0 +1,22 @@ +import type { Event } from "./Event"; +import type { EventClass } from "./EventManager"; + +export const ListenerModes = ["sync", "async"] as const; +export type ListenerMode = (typeof ListenerModes)[number]; + +export type ListenerHandler = ( + event: E, + slug: string, +) => Promise | void; + +export class EventListener { + mode: ListenerMode = "async"; + event: EventClass; + handler: ListenerHandler; + + constructor(event: EventClass, handler: ListenerHandler, mode: ListenerMode = "async") { + this.event = event; + this.handler = handler; + this.mode = mode; + } +} diff --git a/app/src/core/events/EventManager.ts b/app/src/core/events/EventManager.ts new file mode 100644 index 0000000..3b85d23 --- /dev/null +++ b/app/src/core/events/EventManager.ts @@ -0,0 +1,151 @@ +import type { Event } from "./Event"; +import { EventListener, type ListenerHandler, type ListenerMode } from "./EventListener"; + +export interface EmitsEvents { + emgr: EventManager; +} + +export type EventClass = { + new (params: any): Event; + slug: string; +}; + +export class EventManager< + RegisteredEvents extends Record = Record +> { + protected events: EventClass[] = []; + protected listeners: EventListener[] = []; + + constructor(events?: RegisteredEvents, listeners?: EventListener[]) { + if (events) { + this.registerEvents(events); + } + + if (listeners) { + for (const listener of listeners) { + this.addListener(listener); + } + } + } + + clearEvents() { + this.events = []; + return this; + } + + clearAll() { + this.clearEvents(); + this.listeners = []; + return this; + } + + get Events(): { [K in keyof RegisteredEvents]: RegisteredEvents[K] } { + // proxy class to access events + return new Proxy(this, { + get: (_, prop: string) => { + return this.events.find((e) => e.slug === prop); + } + }) as any; + } + + eventExists(slug: string): boolean; + eventExists(event: EventClass | Event): boolean; + eventExists(eventOrSlug: EventClass | Event | string): boolean { + let slug: string; + + if (typeof eventOrSlug === "string") { + slug = eventOrSlug; + } else { + // @ts-expect-error + slug = eventOrSlug.constructor?.slug ?? eventOrSlug.slug; + /*eventOrSlug instanceof Event + ? // @ts-expect-error slug is static + eventOrSlug.constructor.slug + : eventOrSlug.slug;*/ + } + + return !!this.events.find((e) => slug === e.slug); + } + + protected throwIfEventNotRegistered(event: EventClass) { + if (!this.eventExists(event)) { + throw new Error(`Event "${event.slug}" not registered`); + } + } + + registerEvent(event: EventClass, silent: boolean = false) { + if (this.eventExists(event)) { + if (silent) { + return this; + } + + throw new Error(`Event "${event.name}" already registered.`); + } + + this.events.push(event); + return this; + } + + registerEvents(eventObjects: Record): this; + registerEvents(eventArray: EventClass[]): this; + registerEvents(objectOrArray: Record | EventClass[]): this { + const events = + typeof objectOrArray === "object" ? Object.values(objectOrArray) : objectOrArray; + events.forEach((event) => this.registerEvent(event, true)); + return this; + } + + addListener(listener: EventListener) { + this.throwIfEventNotRegistered(listener.event); + + this.listeners.push(listener); + return this; + } + + onEvent>( + event: ActualEvent, + handler: ListenerHandler, + mode: ListenerMode = "async" + ) { + this.throwIfEventNotRegistered(event); + + const listener = new EventListener(event, handler, mode); + this.addListener(listener as any); + } + + on( + slug: string, + handler: ListenerHandler>, + mode: ListenerMode = "async" + ) { + const event = this.events.find((e) => e.slug === slug); + if (!event) { + throw new Error(`Event "${slug}" not registered`); + } + + this.onEvent(event, handler, mode); + } + + onAny(handler: ListenerHandler>, mode: ListenerMode = "async") { + this.events.forEach((event) => this.onEvent(event, handler, mode)); + } + + async emit(event: Event) { + // @ts-expect-error slug is static + const slug = event.constructor.slug; + if (!this.eventExists(event)) { + throw new Error(`Event "${slug}" not registered`); + } + + const listeners = this.listeners.filter((listener) => listener.event.slug === slug); + //console.log("---!-- emitting", slug, listeners.length); + + for (const listener of listeners) { + if (listener.mode === "sync") { + await listener.handler(event, listener.event.slug); + } else { + listener.handler(event, listener.event.slug); + } + } + } +} diff --git a/app/src/core/events/index.ts b/app/src/core/events/index.ts new file mode 100644 index 0000000..b823edf --- /dev/null +++ b/app/src/core/events/index.ts @@ -0,0 +1,8 @@ +export { Event, NoParamEvent } from "./Event"; +export { + EventListener, + ListenerModes, + type ListenerMode, + type ListenerHandler, +} from "./EventListener"; +export { EventManager, type EmitsEvents, type EventClass } from "./EventManager"; diff --git a/app/src/core/index.ts b/app/src/core/index.ts new file mode 100644 index 0000000..5229bd9 --- /dev/null +++ b/app/src/core/index.ts @@ -0,0 +1,28 @@ +export { Endpoint, type RequestResponse, type Middleware } from "./server/Endpoint"; +export { zValidator } from "./server/lib/zValidator"; +export { tbValidator } from "./server/lib/tbValidator"; +export { Exception, BkndError } from "./errors"; +export { isDebug } from "./env"; +export { type PrimaryFieldType, config } from "./config"; +export { AwsClient } from "./clients/aws/AwsClient"; +export { + SimpleRenderer, + type TemplateObject, + type TemplateTypes, + type SimpleRendererOptions +} from "./template/SimpleRenderer"; +export { Controller, type ClassController } from "./server/Controller"; +export { SchemaObject } from "./object/SchemaObject"; +export { DebugLogger } from "./utils/DebugLogger"; +export { Permission } from "./security/Permission"; +export { + exp, + makeValidator, + type FilterQuery, + type Primitive, + isPrimitive, + type TExpression, + type BooleanLike, + isBooleanLike +} from "./object/query/query"; +export { Registry, type Constructor } from "./registry/Registry"; diff --git a/app/src/core/object/SchemaObject.ts b/app/src/core/object/SchemaObject.ts new file mode 100644 index 0000000..8865c50 --- /dev/null +++ b/app/src/core/object/SchemaObject.ts @@ -0,0 +1,199 @@ +import { cloneDeep, get, has, mergeWith, omit, set } from "lodash-es"; +import { + Default, + type Static, + type TObject, + getFullPathKeys, + mark, + parse, + stripMark +} from "../utils"; + +export type SchemaObjectOptions = { + onUpdate?: (config: Static) => void | Promise; + restrictPaths?: string[]; + overwritePaths?: (RegExp | string)[]; + forceParse?: boolean; +}; + +export class SchemaObject { + private readonly _default: Partial>; + private _value: Static; + private _config: Static; + private _restriction_bypass: boolean = false; + + constructor( + private _schema: Schema, + initial?: Partial>, + private options?: SchemaObjectOptions + ) { + this._default = Default(_schema, {} as any) as any; + this._value = initial + ? parse(_schema, cloneDeep(initial as any), { + forceParse: this.isForceParse(), + skipMark: this.isForceParse() + }) + : this._default; + this._config = Object.freeze(this._value); + } + + protected isForceParse(): boolean { + return this.options?.forceParse ?? true; + } + + default(): Static { + return this._default; + } + + get(options?: { stripMark?: boolean }): Static { + if (options?.stripMark) { + return stripMark(this._config); + } + + return this._config; + } + + async set(config: Static, noEmit?: boolean): Promise> { + const valid = parse(this._schema, cloneDeep(config) as any, { + forceParse: true, + skipMark: this.isForceParse() + }); + this._value = valid; + this._config = Object.freeze(valid); + + if (noEmit !== true) { + await this.options?.onUpdate?.(this._config); + } + + return this._config; + } + + bypass() { + this._restriction_bypass = true; + return this; + } + + throwIfRestricted(object: object): void; + throwIfRestricted(path: string): void; + throwIfRestricted(pathOrObject: string | object): void { + // only bypass once + if (this._restriction_bypass) { + this._restriction_bypass = false; + return; + } + + const paths = this.options?.restrictPaths ?? []; + if (Array.isArray(paths) && paths.length > 0) { + for (const path of paths) { + const restricted = + typeof pathOrObject === "string" + ? pathOrObject.startsWith(path) + : has(pathOrObject, path); + + if (restricted) { + throw new Error(`Path "${path}" is restricted`); + } + } + } + + return; + } + + async patch(path: string, value: any): Promise<[Partial>, Static]> { + const current = cloneDeep(this._config); + const partial = path.length > 0 ? (set({}, path, value) as Partial>) : value; + + this.throwIfRestricted(partial); + //console.log(getFullPathKeys(value).map((k) => path + "." + k)); + + // overwrite arrays and primitives, only deep merge objects + // @ts-ignore + const config = mergeWith(current, partial, (objValue, srcValue) => { + if (Array.isArray(objValue) && Array.isArray(srcValue)) { + return srcValue; + } + }); + + //console.log("overwritePaths", this.options?.overwritePaths); + if (this.options?.overwritePaths) { + const keys = getFullPathKeys(value).map((k) => path + "." + k); + const overwritePaths = keys.filter((k) => { + return this.options?.overwritePaths?.some((p) => { + if (typeof p === "string") { + return k === p; + } else { + return p.test(k); + } + }); + }); + //console.log("overwritePaths", keys, overwritePaths); + + if (overwritePaths.length > 0) { + // filter out less specific paths (but only if more than 1) + const specific = + overwritePaths.length > 1 + ? overwritePaths.filter((k) => + overwritePaths.some((k2) => { + console.log("keep?", { k, k2 }, k2 !== k && k2.startsWith(k)); + return k2 !== k && k2.startsWith(k); + }) + ) + : overwritePaths; + //console.log("specific", specific); + + for (const p of specific) { + set(config, p, get(partial, p)); + } + } + } + + //console.log("patch", { path, value, partial, config, current }); + + const newConfig = await this.set(config); + return [partial, newConfig]; + } + + async overwrite(path: string, value: any): Promise<[Partial>, Static]> { + const current = cloneDeep(this._config); + const partial = path.length > 0 ? (set({}, path, value) as Partial>) : value; + + this.throwIfRestricted(partial); + //console.log(getFullPathKeys(value).map((k) => path + "." + k)); + + // overwrite arrays and primitives, only deep merge objects + // @ts-ignore + const config = set(current, path, value); + + //console.log("overwrite", { path, value, partial, config, current }); + + const newConfig = await this.set(config); + return [partial, newConfig]; + } + + has(path: string): boolean { + const p = path.split("."); + if (p.length > 1) { + const parent = p.slice(0, -1).join("."); + if (!has(this._config, parent)) { + console.log("parent", parent, JSON.stringify(this._config, null, 2)); + throw new Error(`Parent path "${parent}" does not exist`); + } + } + + return has(this._config, path); + } + + async remove(path: string): Promise<[Partial>, Static]> { + this.throwIfRestricted(path); + + if (!this.has(path)) { + throw new Error(`Path "${path}" does not exist`); + } + + const current = cloneDeep(this._config); + const removed = get(current, path) as Partial>; + const config = omit(current, path); + const newConfig = await this.set(config); + return [removed, newConfig]; + } +} diff --git a/app/src/core/object/query/object-query.ts b/app/src/core/object/query/object-query.ts new file mode 100644 index 0000000..c4c802e --- /dev/null +++ b/app/src/core/object/query/object-query.ts @@ -0,0 +1,96 @@ +import { type FilterQuery, type Primitive, exp, isPrimitive, makeValidator } from "./query"; + +const expressions = [ + exp( + "$eq", + (v: Primitive) => isPrimitive(v), + (e, a) => e === a + ), + exp( + "$ne", + (v: Primitive) => isPrimitive(v), + (e, a) => e !== a + ), + exp( + "$like", + (v: Primitive) => isPrimitive(v), + (e, a) => { + switch (typeof a) { + case "string": + return (a as string).includes(e as string); + case "number": + return (a as number) === Number(e); + case "boolean": + return (a as boolean) === Boolean(e); + default: + return false; + } + } + ), + exp( + "$regex", + (v: RegExp | string) => (v instanceof RegExp ? true : typeof v === "string"), + (e: any, a: any) => { + if (e instanceof RegExp) { + return e.test(a); + } + if (typeof e === "string") { + const regex = new RegExp(e); + return regex.test(a); + } + return false; + } + ), + exp( + "$isnull", + (v: boolean | 1 | 0) => true, + (e, a) => (e ? a === null : a !== null) + ), + exp( + "$notnull", + (v: boolean | 1 | 0) => true, + (e, a) => (e ? a !== null : a === null) + ), + exp( + "$in", + (v: (string | number)[]) => Array.isArray(v), + (e: any, a: any) => e.includes(a) + ), + exp( + "$notin", + (v: (string | number)[]) => Array.isArray(v), + (e: any, a: any) => !e.includes(a) + ), + exp( + "$gt", + (v: number) => typeof v === "number", + (e: any, a: any) => a > e + ), + exp( + "$gte", + (v: number) => typeof v === "number", + (e: any, a: any) => a >= e + ), + exp( + "$lt", + (v: number) => typeof v === "number", + (e: any, a: any) => a < e + ), + exp( + "$lte", + (v: number) => typeof v === "number", + (e: any, a: any) => a <= e + ), + exp( + "$between", + (v: [number, number]) => + Array.isArray(v) && v.length === 2 && v.every((n) => typeof n === "number"), + (e: any, a: any) => e[0] <= a && a <= e[1] + ) +]; + +export type ObjectQuery = FilterQuery; +const validator = makeValidator(expressions); +export const convert = (query: ObjectQuery) => validator.convert(query); +export const validate = (query: ObjectQuery, object: Record) => + validator.validate(query, { object, convert: true }); diff --git a/app/src/core/object/query/query.ts b/app/src/core/object/query/query.ts new file mode 100644 index 0000000..07a4c3b --- /dev/null +++ b/app/src/core/object/query/query.ts @@ -0,0 +1,209 @@ +export type Primitive = string | number | boolean; +export function isPrimitive(value: any): value is Primitive { + return ["string", "number", "boolean"].includes(typeof value); +} +export type BooleanLike = boolean | 0 | 1; +export function isBooleanLike(value: any): value is boolean { + return [true, false, 0, 1].includes(value); +} + +export class Expression { + expect!: Expect; + + constructor( + public key: Key, + public valid: (v: Expect) => boolean, + public validate: (e: any, a: any, ctx: CTX) => any + ) {} +} +export type TExpression = Expression; + +export function exp( + key: Key, + valid: (v: Expect) => boolean, + validate: (e: Expect, a: unknown, ctx: CTX) => any +): Expression { + return new Expression(key, valid, validate); +} + +type Expressions = Expression[]; +type ExpressionMap = { + [K in Exps[number]["key"]]: Extract extends Expression + ? E + : never; +}; +type ExpressionCondition = { + [K in keyof ExpressionMap]: { [P in K]: ExpressionMap[K] }; +}[keyof ExpressionMap]; + +function getExpression( + expressions: Exps, + key: string +): Expression { + const exp = expressions.find((e) => e.key === key); + if (!exp) throw new Error(`Expression does not exist: "${key}"`); + return exp as any; +} + +type LiteralExpressionCondition = { + [key: string]: Primitive | ExpressionCondition; +}; + +const OperandOr = "$or"; +type OperandCondition = { + [OperandOr]?: LiteralExpressionCondition | ExpressionCondition; +}; + +export type FilterQuery = + | LiteralExpressionCondition + | OperandCondition; + +function _convert( + $query: FilterQuery, + expressions: Exps, + path: string[] = [] +): FilterQuery { + //console.log("-----------------"); + const ExpressionConditionKeys = expressions.map((e) => e.key); + const keys = Object.keys($query); + const operands = [OperandOr] as const; + const newQuery: FilterQuery = {}; + + if (keys.some((k) => k.startsWith("$") && !operands.includes(k as any))) { + throw new Error(`Invalid key '${keys}'. Keys must not start with '$'.`); + } + + if (path.length > 0 && keys.some((k) => operands.includes(k as any))) { + throw new Error(`Operand ${OperandOr} can only appear at the top level.`); + } + + function validate(key: string, value: any, path: string[] = []) { + const exp = getExpression(expressions, key as any); + if (exp.valid(value) === false) { + throw new Error(`Invalid value at "${[...path, key].join(".")}": ${value}`); + } + } + + for (const [key, value] of Object.entries($query)) { + // if $or, convert each value + if (key === "$or") { + newQuery.$or = _convert(value, expressions, [...path, key]); + + // if primitive, assume $eq + } else if (isPrimitive(value)) { + validate("$eq", value, path); + newQuery[key] = { $eq: value }; + + // if object, check for expressions + } else if (typeof value === "object") { + // when object is given, check if all keys are expressions + const invalid = Object.keys(value).filter( + (f) => !ExpressionConditionKeys.includes(f as any) + ); + if (invalid.length === 0) { + newQuery[key] = {}; + // validate each expression + for (const [k, v] of Object.entries(value)) { + validate(k, v, [...path, key]); + newQuery[key][k] = v; + } + } else { + throw new Error( + `Invalid key(s) at "${key}": ${invalid.join(", ")}. Expected expressions.` + ); + } + } + } + + return newQuery; +} + +type ValidationResults = { $and: any[]; $or: any[]; keys: Set }; +type BuildOptions = { + object?: any; + exp_ctx?: any; + convert?: boolean; + value_is_kv?: boolean; +}; +function _build( + _query: FilterQuery, + expressions: Exps, + options: BuildOptions +): ValidationResults { + const $query = options.convert ? _convert(_query, expressions) : _query; + + //console.log("-----------------", { $query }); + //const keys = Object.keys($query); + const result: ValidationResults = { + $and: [], + $or: [], + keys: new Set() + }; + + const { $or, ...$and } = $query; + + function __validate($op: string, expected: any, actual: any, path: string[] = []) { + const exp = getExpression(expressions, $op as any); + if (!exp) { + throw new Error(`Expression does not exist: "${$op}"`); + } + if (!exp.valid(expected)) { + throw new Error(`Invalid expected value at "${[...path, $op].join(".")}": ${expected}`); + } + //console.log("found exp", { key: exp.key, expected, actual }); + return exp.validate(expected, actual, options.exp_ctx); + } + + // check $and + //console.log("$and entries", Object.entries($and)); + for (const [key, value] of Object.entries($and)) { + //console.log("$op/$v", Object.entries(value)); + for (const [$op, $v] of Object.entries(value)) { + const objValue = options.value_is_kv ? key : options.object[key]; + //console.log("--check $and", { key, value, objValue, v_i_kv: options.value_is_kv }); + //console.log("validate", { $op, $v, objValue, key }); + result.$and.push(__validate($op, $v, objValue, [key])); + result.keys.add(key); + } + //console.log("-", { key, value }); + } + + // check $or + for (const [key, value] of Object.entries($or ?? {})) { + const objValue = options.value_is_kv ? key : options.object[key]; + + for (const [$op, $v] of Object.entries(value)) { + //console.log("validate", { $op, $v, objValue }); + result.$or.push(__validate($op, $v, objValue, [key])); + result.keys.add(key); + } + //console.log("-", { key, value }); + } + + //console.log("matches", matches); + return result; +} + +function _validate(results: ValidationResults): boolean { + const matches: { $and?: boolean; $or?: boolean } = { + $and: undefined, + $or: undefined + }; + + matches.$and = results.$and.every((r) => Boolean(r)); + matches.$or = results.$or.some((r) => Boolean(r)); + + return !!matches.$and || !!matches.$or; +} + +export function makeValidator(expressions: Exps) { + return { + convert: (query: FilterQuery) => _convert(query, expressions), + build: (query: FilterQuery, options: BuildOptions) => + _build(query, expressions, options), + validate: (query: FilterQuery, options: BuildOptions) => { + const fns = _build(query, expressions, options); + return _validate(fns); + } + }; +} diff --git a/app/src/core/registry/Registry.ts b/app/src/core/registry/Registry.ts new file mode 100644 index 0000000..12209ca --- /dev/null +++ b/app/src/core/registry/Registry.ts @@ -0,0 +1,30 @@ +export type Constructor = new (...args: any[]) => T; +export class Registry = Record> { + private is_set: boolean = false; + private items: Items = {} as Items; + + set>(items: Actual) { + if (this.is_set) { + throw new Error("Registry is already set"); + } + // @ts-ignore + this.items = items; + this.is_set = true; + + return this as unknown as Registry; + } + + add(name: string, item: Item) { + // @ts-ignore + this.items[name] = item; + return this; + } + + get(name: Name): Items[Name] { + return this.items[name]; + } + + all() { + return this.items; + } +} diff --git a/app/src/core/security/Permission.ts b/app/src/core/security/Permission.ts new file mode 100644 index 0000000..eb72ccb --- /dev/null +++ b/app/src/core/security/Permission.ts @@ -0,0 +1,11 @@ +export class Permission { + constructor(public name: Name) { + this.name = name; + } + + toJSON() { + return { + name: this.name + }; + } +} diff --git a/app/src/core/server/ContextHelper.ts b/app/src/core/server/ContextHelper.ts new file mode 100644 index 0000000..fb51347 --- /dev/null +++ b/app/src/core/server/ContextHelper.ts @@ -0,0 +1,29 @@ +import type { Context } from "hono"; + +export class ContextHelper { + constructor(protected c: Context) {} + + contentTypeMime(): string { + const contentType = this.c.res.headers.get("Content-Type"); + if (contentType) { + return String(contentType.split(";")[0]); + } + return ""; + } + + isHtml(): boolean { + return this.contentTypeMime() === "text/html"; + } + + url(): URL { + return new URL(this.c.req.url); + } + + headersObject() { + const headers = {}; + for (const [k, v] of this.c.res.headers.entries()) { + headers[k] = v; + } + return headers; + } +} diff --git a/app/src/core/server/Controller.ts b/app/src/core/server/Controller.ts new file mode 100644 index 0000000..9c74571 --- /dev/null +++ b/app/src/core/server/Controller.ts @@ -0,0 +1,155 @@ +import { Hono, type MiddlewareHandler, type ValidationTargets } from "hono"; +import type { H } from "hono/types"; +import { safelyParseObjectValues } from "../utils"; +import type { Endpoint, Middleware } from "./Endpoint"; +import { zValidator } from "./lib/zValidator"; + +type RouteProxy = { + [K in keyof Endpoints]: Endpoints[K]; +}; + +export interface ClassController { + getController: () => Hono; + getMiddleware?: MiddlewareHandler; +} + +/** + * @deprecated + */ +export class Controller< + Endpoints extends Record = Record, + Middlewares extends Record = Record +> { + protected endpoints: Endpoints = {} as Endpoints; + protected middlewares: Middlewares = {} as Middlewares; + + public prefix: string = "/"; + public routes: RouteProxy; + + constructor( + prefix: string = "/", + endpoints: Endpoints = {} as Endpoints, + middlewares: Middlewares = {} as Middlewares + ) { + this.prefix = prefix; + this.endpoints = endpoints; + this.middlewares = middlewares; + + this.routes = new Proxy( + {}, + { + get: (_, name: string) => { + return this.endpoints[name]; + } + } + ) as RouteProxy; + } + + add( + this: Controller, + name: Name, + endpoint: E + ): Controller> { + const newEndpoints = { + ...this.endpoints, + [name]: endpoint + } as Endpoints & Record; + const newController: Controller> = new Controller< + Endpoints & Record + >(); + newController.endpoints = newEndpoints; + newController.middlewares = this.middlewares; + return newController; + } + + get(name: Name): Endpoints[Name] { + return this.endpoints[name]; + } + + honoify(_hono: Hono = new Hono()) { + const hono = _hono.basePath(this.prefix); + + // apply middlewares + for (const m_name in this.middlewares) { + const middleware = this.middlewares[m_name]; + + if (typeof middleware === "function") { + //if (isDebug()) console.log("+++ appyling middleware", m_name, middleware); + hono.use(middleware); + } + } + + // apply endpoints + for (const name in this.endpoints) { + const endpoint = this.endpoints[name]; + if (!endpoint) continue; + + const handlers: H[] = []; + + const supportedValidations: Array = ["param", "query", "json"]; + + // if validations are present, add them to the handlers + for (const validation of supportedValidations) { + if (endpoint.validation[validation]) { + handlers.push(async (c, next) => { + // @todo: potentially add "strict" to all schemas? + const res = await zValidator( + validation, + endpoint.validation[validation] as any, + (target, value, c) => { + if (["query", "param"].includes(target)) { + return safelyParseObjectValues(value); + } + //console.log("preprocess", target, value, c.req.raw.url); + return value; + } + )(c, next); + + if (res instanceof Response && res.status === 400) { + const error = await res.json(); + return c.json( + { + error: "Validation error", + target: validation, + message: error + }, + 400 + ); + } + + return res; + }); + } + } + + // add actual handler + handlers.push(endpoint.toHandler()); + + const method = endpoint.method.toLowerCase() as + | "get" + | "post" + | "put" + | "delete" + | "patch"; + + //if (isDebug()) console.log("--- adding", method, endpoint.path); + hono[method](endpoint.path, ...handlers); + } + + return hono; + } + + toJSON() { + const endpoints: any = {}; + for (const name in this.endpoints) { + const endpoint = this.endpoints[name]; + if (!endpoint) continue; + + endpoints[name] = { + method: endpoint.method, + path: (this.prefix + endpoint.path).replace("//", "/") + }; + } + return endpoints; + } +} diff --git a/app/src/core/server/Endpoint.ts b/app/src/core/server/Endpoint.ts new file mode 100644 index 0000000..c45b644 --- /dev/null +++ b/app/src/core/server/Endpoint.ts @@ -0,0 +1,147 @@ +import type { Context, MiddlewareHandler, Next, ValidationTargets } from "hono"; +import type { Handler } from "hono/types"; +import { encodeSearch, replaceUrlParam } from "../utils"; +import type { Prettify } from "../utils"; + +type ZodSchema = { [key: string]: any }; + +type Method = "GET" | "POST" | "PUT" | "DELETE" | "PATCH"; +type Validation = { + [K in keyof ValidationTargets]?: any; +} & { + param?: P extends ZodSchema ? P : undefined; + query?: Q extends ZodSchema ? Q : undefined; + json?: B extends ZodSchema ? B : undefined; +}; + +type ValidationInput = { + param?: P extends ZodSchema ? P["_input"] : undefined; + query?: Q extends ZodSchema ? Q["_input"] : undefined; + json?: B extends ZodSchema ? B["_input"] : undefined; +}; + +type HonoEnv = any; + +export type Middleware = MiddlewareHandler; + +type HandlerFunction

= (c: Context, next: Next) => R; +export type RequestResponse = { + status: number; + ok: boolean; + response: Awaited; +}; + +/** + * @deprecated + */ +export class Endpoint< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any +> { + constructor( + readonly method: Method, + readonly path: Path, + readonly handler: HandlerFunction, + readonly validation: Validation = {} + ) {} + + // @todo: typing is not ideal + async $request( + args?: ValidationInput, + baseUrl: string = "http://localhost:28623" + ): Promise>> { + let path = this.path as string; + if (args?.param) { + path = replaceUrlParam(path, args.param); + } + + if (args?.query) { + path += "?" + encodeSearch(args.query); + } + + const url = [baseUrl, path].join("").replace(/\/$/, ""); + const options: RequestInit = { + method: this.method, + headers: {} as any + }; + + if (!["GET", "HEAD"].includes(this.method)) { + if (args?.json) { + options.body = JSON.stringify(args.json); + options.headers!["Content-Type"] = "application/json"; + } + } + + const res = await fetch(url, options); + return { + status: res.status, + ok: res.ok, + response: (await res.json()) as any + }; + } + + toHandler(): Handler { + return async (c, next) => { + const res = await this.handler(c, next); + //console.log("toHandler:isResponse", res instanceof Response); + //return res; + if (res instanceof Response) { + return res; + } + return c.json(res as any) as unknown as Handler; + }; + } + + static get< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any + >(path: Path, handler: HandlerFunction, validation?: Validation) { + return new Endpoint("GET", path, handler, validation); + } + + static post< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any + >(path: Path, handler: HandlerFunction, validation?: Validation) { + return new Endpoint("POST", path, handler, validation); + } + + static patch< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any + >(path: Path, handler: HandlerFunction, validation?: Validation) { + return new Endpoint("PATCH", path, handler, validation); + } + + static put< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any + >(path: Path, handler: HandlerFunction, validation?: Validation) { + return new Endpoint("PUT", path, handler, validation); + } + + static delete< + Path extends string = any, + P extends ZodSchema = any, + Q extends ZodSchema = any, + B extends ZodSchema = any, + R = any + >(path: Path, handler: HandlerFunction, validation?: Validation) { + return new Endpoint("DELETE", path, handler, validation); + } +} diff --git a/app/src/core/server/lib/tbValidator.ts b/app/src/core/server/lib/tbValidator.ts new file mode 100644 index 0000000..2118b18 --- /dev/null +++ b/app/src/core/server/lib/tbValidator.ts @@ -0,0 +1,37 @@ +import type { StaticDecode, TSchema } from "@sinclair/typebox"; +import { Value, type ValueError } from "@sinclair/typebox/value"; +import type { Context, Env, MiddlewareHandler, ValidationTargets } from "hono"; +import { validator } from "hono/validator"; + +type Hook = ( + result: { success: true; data: T } | { success: false; errors: ValueError[] }, + c: Context +) => Response | Promise | void; + +export function tbValidator< + T extends TSchema, + Target extends keyof ValidationTargets, + E extends Env, + P extends string, + V extends { in: { [K in Target]: StaticDecode }; out: { [K in Target]: StaticDecode } } +>(target: Target, schema: T, hook?: Hook, E, P>): MiddlewareHandler { + // Compile the provided schema once rather than per validation. This could be optimized further using a shared schema + // compilation pool similar to the Fastify implementation. + + // @ts-expect-error not typed well + return validator(target, (data, c) => { + if (Value.Check(schema, data)) { + // always decode + const decoded = Value.Decode(schema, data); + + if (hook) { + const hookResult = hook({ success: true, data: decoded }, c); + if (hookResult instanceof Response || hookResult instanceof Promise) { + return hookResult; + } + } + return decoded; + } + return c.json({ success: false, errors: [...Value.Errors(schema, data)] }, 400); + }); +} diff --git a/app/src/core/server/lib/zValidator.ts b/app/src/core/server/lib/zValidator.ts new file mode 100644 index 0000000..4d16a59 --- /dev/null +++ b/app/src/core/server/lib/zValidator.ts @@ -0,0 +1,75 @@ +import type { + Context, + Env, + Input, + MiddlewareHandler, + TypedResponse, + ValidationTargets, +} from "hono"; +import { validator } from "hono/validator"; +import type { ZodError, ZodSchema, z } from "zod"; + +export type Hook = ( + result: { success: true; data: T } | { success: false; error: ZodError; data: T }, + c: Context, +) => Response | void | TypedResponse | Promise>; + +type HasUndefined = undefined extends T ? true : false; + +export const zValidator = < + T extends ZodSchema, + Target extends keyof ValidationTargets, + E extends Env, + P extends string, + In = z.input, + Out = z.output, + I extends Input = { + in: HasUndefined extends true + ? { + [K in Target]?: K extends "json" + ? In + : HasUndefined extends true + ? { [K2 in keyof In]?: ValidationTargets[K][K2] } + : { [K2 in keyof In]: ValidationTargets[K][K2] }; + } + : { + [K in Target]: K extends "json" + ? In + : HasUndefined extends true + ? { [K2 in keyof In]?: ValidationTargets[K][K2] } + : { [K2 in keyof In]: ValidationTargets[K][K2] }; + }; + out: { [K in Target]: Out }; + }, + V extends I = I, +>( + target: Target, + schema: T, + preprocess?: (target: string, value: In, c: Context) => V, // <-- added + hook?: Hook, E, P>, +): MiddlewareHandler => + // @ts-expect-error not typed well + validator(target, async (value, c) => { + // added: preprocess value first if given + const _value = preprocess ? preprocess(target, value, c) : (value as any); + const result = await schema.safeParseAsync(_value); + + if (hook) { + const hookResult = await hook({ data: value, ...result }, c); + if (hookResult) { + if (hookResult instanceof Response) { + return hookResult; + } + + if ("response" in hookResult) { + return hookResult.response; + } + } + } + + if (!result.success) { + return c.json(result, 400); + } + + return result.data as z.infer; + }); diff --git a/app/src/core/template/SimpleRenderer.ts b/app/src/core/template/SimpleRenderer.ts new file mode 100644 index 0000000..0e90c8e --- /dev/null +++ b/app/src/core/template/SimpleRenderer.ts @@ -0,0 +1,96 @@ +import { Liquid, LiquidError } from "liquidjs"; +import type { RenderOptions } from "liquidjs/dist/liquid-options"; +import { BkndError } from "../errors"; + +export type TemplateObject = Record>; +export type TemplateTypes = string | TemplateObject; + +export type SimpleRendererOptions = RenderOptions & { + renderKeys?: boolean; +}; + +export class SimpleRenderer { + private engine = new Liquid(); + + constructor( + private variables: Record = {}, + private options: SimpleRendererOptions = {} + ) {} + + another() { + return 1; + } + + static hasMarkup(template: string | object): boolean { + //console.log("has markup?", template); + let flat: string = ""; + + if (Array.isArray(template) || typeof template === "object") { + // only plain arrays and objects + if (!["Array", "Object"].includes(template.constructor.name)) return false; + + flat = JSON.stringify(template); + } else { + flat = String(template); + } + + //console.log("** flat", flat); + + const checks = ["{{", "{%", "{#", "{:"]; + const hasMarkup = checks.some((check) => flat.includes(check)); + //console.log("--has markup?", hasMarkup); + return hasMarkup; + } + + async render(template: Given): Promise { + try { + if (typeof template === "string") { + return (await this.renderString(template)) as unknown as Given; + } else if (Array.isArray(template)) { + return (await Promise.all( + template.map((item) => this.render(item)) + )) as unknown as Given; + } else if (typeof template === "object") { + return (await this.renderObject(template)) as unknown as Given; + } + } catch (e) { + if (e instanceof LiquidError) { + const details = { + name: e.name, + token: { + kind: e.token.kind, + input: e.token.input, + begin: e.token.begin, + end: e.token.end + } + }; + + throw new BkndError(e.message, details, "liquid"); + } + + throw e; + } + + throw new Error("Invalid template type"); + } + + async renderString(template: string): Promise { + //console.log("*** renderString", template, this.variables); + return this.engine.parseAndRender(template, this.variables, this.options); + } + + async renderObject(template: TemplateObject): Promise { + const result: TemplateObject = {}; + + for (const [key, value] of Object.entries(template)) { + let resultKey = key; + if (this.options.renderKeys) { + resultKey = await this.renderString(key); + } + + result[resultKey] = await this.render(value); + } + + return result; + } +} diff --git a/app/src/core/types.ts b/app/src/core/types.ts new file mode 100644 index 0000000..a3bf623 --- /dev/null +++ b/app/src/core/types.ts @@ -0,0 +1,4 @@ +export interface Serializable { + toJSON(): Json; + fromJSON(json: Json): Class; +} \ No newline at end of file diff --git a/app/src/core/utils/DebugLogger.ts b/app/src/core/utils/DebugLogger.ts new file mode 100644 index 0000000..aada58d --- /dev/null +++ b/app/src/core/utils/DebugLogger.ts @@ -0,0 +1,36 @@ +export class DebugLogger { + public _context: string[] = []; + _enabled: boolean = true; + private readonly id = Math.random().toString(36).substr(2, 9); + private last: number = 0; + + constructor(enabled: boolean = true) { + this._enabled = enabled; + } + + context(context: string) { + //console.log("[ settings context ]", context, this._context); + this._context.push(context); + return this; + } + + clear() { + //console.log("[ clear context ]", this._context.pop(), this._context); + this._context.pop(); + return this; + } + + log(...args: any[]) { + if (!this._enabled) return this; + + const now = performance.now(); + const time = Number.parseInt(String(now - this.last)); + const indents = " ".repeat(this._context.length); + const context = + this._context.length > 0 ? `[${this._context[this._context.length - 1]}]` : ""; + console.log(indents, context, time, ...args); + + this.last = now; + return this; + } +} diff --git a/app/src/core/utils/browser.ts b/app/src/core/utils/browser.ts new file mode 100644 index 0000000..afba7ba --- /dev/null +++ b/app/src/core/utils/browser.ts @@ -0,0 +1,20 @@ +export type TBrowser = "Opera" | "Edge" | "Chrome" | "Safari" | "Firefox" | "IE" | "unknown"; +export function getBrowser(): TBrowser { + if ((navigator.userAgent.indexOf("Opera") || navigator.userAgent.indexOf("OPR")) !== -1) { + return "Opera"; + } else if (navigator.userAgent.indexOf("Edg") !== -1) { + return "Edge"; + } else if (navigator.userAgent.indexOf("Chrome") !== -1) { + return "Chrome"; + } else if (navigator.userAgent.indexOf("Safari") !== -1) { + return "Safari"; + } else if (navigator.userAgent.indexOf("Firefox") !== -1) { + return "Firefox"; + // @ts-ignore + } else if (navigator.userAgent.indexOf("MSIE") !== -1 || !!document.documentMode === true) { + //IF IE > 10 + return "IE"; + } else { + return "unknown"; + } +} diff --git a/app/src/core/utils/crypto.ts b/app/src/core/utils/crypto.ts new file mode 100644 index 0000000..6996d1c --- /dev/null +++ b/app/src/core/utils/crypto.ts @@ -0,0 +1,29 @@ +export const HashAlgorithms = ["SHA-1", "SHA-256", "SHA-384", "SHA-512"] as const; +export type HashAlgorithm = (typeof HashAlgorithms)[number]; +export async function digest(alg: HashAlgorithm, input: string, salt?: string, pepper?: string) { + if (!HashAlgorithms.includes(alg)) { + throw new Error(`Invalid hash algorithm: ${alg}`); + } + + // convert to Uint8Array + const data = new TextEncoder().encode((salt ?? "") + input + (pepper ?? "")); + + // hash to alg + const hashBuffer = await crypto.subtle.digest(alg, data); + + // convert hash to hex string + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export const hash = { + sha256: async (input: string, salt?: string, pepper?: string) => + digest("SHA-256", input, salt, pepper), + sha1: async (input: string, salt?: string, pepper?: string) => + digest("SHA-1", input, salt, pepper) +}; + +export async function checksum(s: any) { + const o = typeof s === "string" ? s : JSON.stringify(s); + return await digest("SHA-1", o); +} diff --git a/app/src/core/utils/dates.ts b/app/src/core/utils/dates.ts new file mode 100644 index 0000000..c33b496 --- /dev/null +++ b/app/src/core/utils/dates.ts @@ -0,0 +1,14 @@ +import dayjs from "dayjs"; +import weekOfYear from "dayjs/plugin/weekOfYear.js"; + +declare module "dayjs" { + interface Dayjs { + week(): number; + + week(value: number): dayjs.Dayjs; + } +} + +dayjs.extend(weekOfYear); + +export { dayjs }; diff --git a/app/src/core/utils/index.ts b/app/src/core/utils/index.ts new file mode 100644 index 0000000..dcd5aff --- /dev/null +++ b/app/src/core/utils/index.ts @@ -0,0 +1,13 @@ +export * from "./browser"; +export * from "./objects"; +export * from "./strings"; +export * from "./perf"; +export * from "./reqres"; +export * from "./xml"; +export type { Prettify, PrettifyRec } from "./types"; +export * from "./typebox"; +export * from "./dates"; +export * from "./crypto"; +export * from "./uuid"; +export { FromSchema } from "./typebox/from-schema"; +export * from "./test"; diff --git a/app/src/core/utils/objects.ts b/app/src/core/utils/objects.ts new file mode 100644 index 0000000..16bffe9 --- /dev/null +++ b/app/src/core/utils/objects.ts @@ -0,0 +1,198 @@ +import { pascalToKebab } from "./strings"; + +export function _jsonp(obj: any, space = 2): string { + return JSON.stringify(obj, null, space); +} + +export function safelyParseObjectValues(obj: T): T { + return Object.entries(obj).reduce((acc, [key, value]) => { + try { + // @ts-ignore + acc[key] = JSON.parse(value); + } catch (error) { + // @ts-ignore + acc[key] = value; + } + return acc; + }, {} as T); +} + +export function keepChanged(origin: T, updated: T): Partial { + return Object.keys(updated).reduce( + (acc, key) => { + if (updated[key] !== origin[key]) { + acc[key] = updated[key]; + } + return acc; + }, + {} as Partial + ); +} + +export function objectKeysPascalToKebab(obj: any, ignoreKeys: string[] = []): any { + if (obj === null || typeof obj !== "object") { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map((item) => objectKeysPascalToKebab(item, ignoreKeys)); + } + + return Object.keys(obj).reduce( + (acc, key) => { + const kebabKey = ignoreKeys.includes(key) ? key : pascalToKebab(key); + acc[kebabKey] = objectKeysPascalToKebab(obj[key], ignoreKeys); + return acc; + }, + {} as Record + ); +} + +export function filterKeys( + obj: Object, + keysToFilter: string[] +): Object { + const result = {} as Object; + + for (const key in obj) { + const shouldFilter = keysToFilter.some((filterKey) => key.includes(filterKey)); + if (!shouldFilter) { + if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) { + result[key] = filterKeys(obj[key], keysToFilter); + } else { + result[key] = obj[key]; + } + } + } + + return result; +} + +export function transformObject, U>( + object: T, + transform: (value: T[keyof T], key: keyof T) => U | undefined +): { [K in keyof T]: U } { + return Object.entries(object).reduce( + (acc, [key, value]) => { + const t = transform(value, key as keyof T); + if (typeof t !== "undefined") { + acc[key as keyof T] = t; + } + return acc; + }, + {} as { [K in keyof T]: U } + ); +} +export const objectTransform = transformObject; + +export function objectEach, U>( + object: T, + each: (value: T[keyof T], key: keyof T) => U +): void { + Object.entries(object).forEach( + ([key, value]) => { + each(value, key); + }, + {} as { [K in keyof T]: U } + ); +} + +/** + * Simple object check. + * @param item + * @returns {boolean} + */ +export function isObject(item) { + return item && typeof item === "object" && !Array.isArray(item); +} + +/** + * Deep merge two objects. + * @param target + * @param ...sources + */ +export function mergeDeep(target, ...sources) { + if (!sources.length) return target; + const source = sources.shift(); + + if (isObject(target) && isObject(source)) { + for (const key in source) { + if (isObject(source[key])) { + if (!target[key]) Object.assign(target, { [key]: {} }); + mergeDeep(target[key], source[key]); + } else { + Object.assign(target, { [key]: source[key] }); + } + } + } + + return mergeDeep(target, ...sources); +} + +export function getFullPathKeys(obj: any, parentPath: string = ""): string[] { + let keys: string[] = []; + + for (const key in obj) { + const fullPath = parentPath ? `${parentPath}.${key}` : key; + keys.push(fullPath); + + if (typeof obj[key] === "object" && obj[key] !== null) { + keys = keys.concat(getFullPathKeys(obj[key], fullPath)); + } + } + + return keys; +} + +export function flattenObject(obj: any, parentKey = "", result: any = {}): any { + for (const key in obj) { + if (key in obj) { + const newKey = parentKey ? `${parentKey}.${key}` : key; + if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) { + flattenObject(obj[key], newKey, result); + } else if (Array.isArray(obj[key])) { + obj[key].forEach((item, index) => { + const arrayKey = `${newKey}.${index}`; + if (typeof item === "object" && item !== null) { + flattenObject(item, arrayKey, result); + } else { + result[arrayKey] = item; + } + }); + } else { + result[newKey] = obj[key]; + } + } + } + return result; +} + +export function objectDepth(object: object): number { + let level = 1; + for (const key in object) { + if (typeof object[key] === "object") { + const depth = objectDepth(object[key]) + 1; + level = Math.max(depth, level); + } + } + return level; +} + +export function objectCleanEmpty(obj: Obj): Obj { + return Object.entries(obj).reduce((acc, [key, value]) => { + if (value && Array.isArray(value) && value.some((v) => typeof v === "object")) { + const nested = value.map(objectCleanEmpty); + if (nested.length > 0) { + acc[key] = nested; + } + } else if (value && typeof value === "object" && !Array.isArray(value)) { + const nested = objectCleanEmpty(value); + if (Object.keys(nested).length > 0) { + acc[key] = nested; + } + } else if (value !== "" && value !== null && value !== undefined) { + acc[key] = value; + } + return acc; + }, {} as any); +} diff --git a/app/src/core/utils/perf.ts b/app/src/core/utils/perf.ts new file mode 100644 index 0000000..1104c58 --- /dev/null +++ b/app/src/core/utils/perf.ts @@ -0,0 +1,60 @@ +export class Perf { + private marks: { mark: string; time: number }[] = []; + private startTime: number; + private endTime: number | null = null; + + private constructor() { + this.startTime = performance.now(); + } + + static start(): Perf { + return new Perf(); + } + + mark(markName: string): void { + if (this.endTime !== null) { + throw new Error("Cannot add marks after perf measurement has been closed."); + } + + const currentTime = performance.now(); + const lastMarkTime = + this.marks.length > 0 ? this.marks[this.marks.length - 1]!.time : this.startTime; + const elapsedTimeSinceLastMark = currentTime - lastMarkTime; + + this.marks.push({ mark: markName, time: elapsedTimeSinceLastMark }); + } + + close(): void { + if (this.endTime !== null) { + throw new Error("Perf measurement has already been closed."); + } + this.endTime = performance.now(); + } + + result(): { total: number; marks: { mark: string; time: number }[] } { + if (this.endTime === null) { + throw new Error("Perf measurement has not been closed yet."); + } + + const totalTime = this.endTime - this.startTime; + return { + total: Number.parseFloat(totalTime.toFixed(2)), + marks: this.marks.map((mark) => ({ + mark: mark.mark, + time: Number.parseFloat(mark.time.toFixed(2)), + })), + }; + } + + static async execute(fn: () => Promise, times: number = 1): Promise { + const perf = Perf.start(); + + for (let i = 0; i < times; i++) { + await fn(); + perf.mark(`iteration-${i}`); + } + + perf.close(); + return perf.result(); + } +} diff --git a/app/src/core/utils/reqres.ts b/app/src/core/utils/reqres.ts new file mode 100644 index 0000000..21e0e28 --- /dev/null +++ b/app/src/core/utils/reqres.ts @@ -0,0 +1,84 @@ +export function headersToObject(headers: Headers): Record { + if (!headers) return {}; + return { ...Object.fromEntries(headers.entries()) }; +} + +export function pickHeaders(headers: Headers, keys: string[]): Record { + const obj = headersToObject(headers); + const res = {}; + for (const key of keys) { + if (obj[key]) { + res[key] = obj[key]; + } + } + return res; +} + +export const replaceUrlParam = (urlString: string, params: Record) => { + let newString = urlString; + for (const [k, v] of Object.entries(params)) { + const reg = new RegExp(`/:${k}(?:{[^/]+})?`); + newString = newString.replace(reg, `/${v}`); + } + return newString; +}; + +export function encodeSearch(obj, options?: { prefix?: string; encode?: boolean }) { + let str = ""; + function _encode(str) { + return options?.encode ? encodeURIComponent(str) : str; + } + + for (const k in obj) { + let tmp = obj[k]; + if (tmp !== void 0) { + if (Array.isArray(tmp)) { + for (let i = 0; i < tmp.length; i++) { + if (str.length > 0) str += "&"; + str += `${_encode(k)}=${_encode(tmp[i])}`; + } + } else { + if (typeof tmp === "object") { + tmp = JSON.stringify(tmp); + } + + if (str.length > 0) str += "&"; + str += `${_encode(k)}=${_encode(tmp)}`; + } + } + } + + return (options?.prefix || "") + str; +} + +export function decodeSearch(str) { + function toValue(mix) { + if (!mix) return ""; + const str = decodeURIComponent(mix); + if (str === "false") return false; + if (str === "true") return true; + try { + return JSON.parse(str); + } catch (e) { + return +str * 0 === 0 ? +str : str; + } + } + + let tmp: any; + let k: string; + const out = {}; + const arr = str.split("&"); + + // biome-ignore lint/suspicious/noAssignInExpressions: + while ((tmp = arr.shift())) { + tmp = tmp.split("="); + k = tmp.shift(); + if (out[k] !== void 0) { + out[k] = [].concat(out[k], toValue(tmp.shift())); + } else { + out[k] = toValue(tmp.shift()); + } + } + + return out; +} diff --git a/app/src/core/utils/sql.ts b/app/src/core/utils/sql.ts new file mode 100644 index 0000000..39dcbba --- /dev/null +++ b/app/src/core/utils/sql.ts @@ -0,0 +1,9 @@ +import { isDebug } from "../env"; + +export async function formatSql(sql: string): Promise { + if (isDebug()) { + const { format } = await import("sql-formatter"); + return format(sql); + } + return ""; +} diff --git a/app/src/core/utils/strings.ts b/app/src/core/utils/strings.ts new file mode 100644 index 0000000..adc68b7 --- /dev/null +++ b/app/src/core/utils/strings.ts @@ -0,0 +1,62 @@ +export function objectToKeyValueArray>(obj: T) { + return Object.keys(obj).map((key) => ({ key, value: obj[key as keyof T] })); +} + +export function ucFirst(str: string): string { + if (!str || str.length === 0) return str; + return str.charAt(0).toUpperCase() + str.slice(1); +} + +export function ucFirstAll(str: string, split: string = " "): string { + if (!str || str.length === 0) return str; + return str + .split(split) + .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) + .join(split); +} + +export function ucFirstAllSnakeToPascalWithSpaces(str: string, split: string = " "): string { + return ucFirstAll(snakeToPascalWithSpaces(str), split); +} + +export function randomString(length: number, includeSpecial = false): string { + const base = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const special = "!@#$%^&*()_+{}:\"<>?|[];',./`~"; + const chars = base + (includeSpecial ? special : ""); + let result = ""; + for (let i = 0; i < length; i++) { + result += chars[Math.floor(Math.random() * chars.length)]; + } + return result; +} + +/** + * Convert a string from snake_case to PascalCase with spaces + * Example: `snake_to_pascal` -> `Snake To Pascal` + * + * @param str + */ +export function snakeToPascalWithSpaces(str: string): string { + if (!str || str.length === 0) return str; + + return str + .split("_") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(" "); +} + +export function pascalToKebab(pascalStr: string): string { + return pascalStr.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase(); +} + +/** + * Replace simple mustache like {placeholders} in a string + * + * @param str + * @param vars + */ +export function replaceSimplePlaceholders(str: string, vars: Record): string { + return str.replace(/\{\$(\w+)\}/g, (match, key) => { + return key in vars ? vars[key] : match; + }); +} diff --git a/app/src/core/utils/test.ts b/app/src/core/utils/test.ts new file mode 100644 index 0000000..3753944 --- /dev/null +++ b/app/src/core/utils/test.ts @@ -0,0 +1,18 @@ +type ConsoleSeverity = "log" | "warn" | "error"; +const _oldConsoles = { + log: console.log, + warn: console.warn, + error: console.error +}; + +export function disableConsoleLog(severities: ConsoleSeverity[] = ["log"]) { + severities.forEach((severity) => { + console[severity] = () => null; + }); +} + +export function enableConsoleLog() { + Object.entries(_oldConsoles).forEach(([severity, fn]) => { + console[severity as ConsoleSeverity] = fn; + }); +} diff --git a/app/src/core/utils/typebox/from-schema.ts b/app/src/core/utils/typebox/from-schema.ts new file mode 100644 index 0000000..4c2663a --- /dev/null +++ b/app/src/core/utils/typebox/from-schema.ts @@ -0,0 +1,268 @@ +/*-------------------------------------------------------------------------- + +@sinclair/typebox/prototypes + +The MIT License (MIT) + +Copyright (c) 2017-2024 Haydn Paterson (sinclair) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Type from "@sinclair/typebox"; + +// ------------------------------------------------------------------ +// Schematics +// ------------------------------------------------------------------ +const IsExact = (value: unknown, expect: unknown) => value === expect; +const IsSValue = (value: unknown): value is SValue => + Type.ValueGuard.IsString(value) || + Type.ValueGuard.IsNumber(value) || + Type.ValueGuard.IsBoolean(value); +const IsSEnum = (value: unknown): value is SEnum => + Type.ValueGuard.IsObject(value) && + Type.ValueGuard.IsArray(value.enum) && + value.enum.every((value) => IsSValue(value)); +const IsSAllOf = (value: unknown): value is SAllOf => + Type.ValueGuard.IsObject(value) && Type.ValueGuard.IsArray(value.allOf); +const IsSAnyOf = (value: unknown): value is SAnyOf => + Type.ValueGuard.IsObject(value) && Type.ValueGuard.IsArray(value.anyOf); +const IsSOneOf = (value: unknown): value is SOneOf => + Type.ValueGuard.IsObject(value) && Type.ValueGuard.IsArray(value.oneOf); +const IsSTuple = (value: unknown): value is STuple => + Type.ValueGuard.IsObject(value) && + IsExact(value.type, "array") && + Type.ValueGuard.IsArray(value.items); +const IsSArray = (value: unknown): value is SArray => + Type.ValueGuard.IsObject(value) && + IsExact(value.type, "array") && + !Type.ValueGuard.IsArray(value.items) && + Type.ValueGuard.IsObject(value.items); +const IsSConst = (value: unknown): value is SConst => + Type.ValueGuard.IsObject(value) && Type.ValueGuard.IsObject(value["const"]); +const IsSString = (value: unknown): value is SString => + Type.ValueGuard.IsObject(value) && IsExact(value.type, "string"); +const IsSNumber = (value: unknown): value is SNumber => + Type.ValueGuard.IsObject(value) && IsExact(value.type, "number"); +const IsSInteger = (value: unknown): value is SInteger => + Type.ValueGuard.IsObject(value) && IsExact(value.type, "integer"); +const IsSBoolean = (value: unknown): value is SBoolean => + Type.ValueGuard.IsObject(value) && IsExact(value.type, "boolean"); +const IsSNull = (value: unknown): value is SBoolean => + Type.ValueGuard.IsObject(value) && IsExact(value.type, "null"); +const IsSProperties = (value: unknown): value is SProperties => Type.ValueGuard.IsObject(value); +// prettier-ignore +const IsSObject = (value: unknown): value is SObject => Type.ValueGuard.IsObject(value) && IsExact(value.type, 'object') && IsSProperties(value.properties) && (value.required === undefined || Type.ValueGuard.IsArray(value.required) && value.required.every((value: unknown) => Type.ValueGuard.IsString(value))) +type SValue = string | number | boolean; +type SEnum = Readonly<{ enum: readonly SValue[] }>; +type SAllOf = Readonly<{ allOf: readonly unknown[] }>; +type SAnyOf = Readonly<{ anyOf: readonly unknown[] }>; +type SOneOf = Readonly<{ oneOf: readonly unknown[] }>; +type SProperties = Record; +type SObject = Readonly<{ type: "object"; properties: SProperties; required?: readonly string[] }>; +type STuple = Readonly<{ type: "array"; items: readonly unknown[] }>; +type SArray = Readonly<{ type: "array"; items: unknown }>; +type SConst = Readonly<{ const: SValue }>; +type SString = Readonly<{ type: "string" }>; +type SNumber = Readonly<{ type: "number" }>; +type SInteger = Readonly<{ type: "integer" }>; +type SBoolean = Readonly<{ type: "boolean" }>; +type SNull = Readonly<{ type: "null" }>; +// ------------------------------------------------------------------ +// FromRest +// ------------------------------------------------------------------ +// prettier-ignore +type TFromRest = ( + T extends readonly [infer L extends unknown, ...infer R extends unknown[]] + ? TFromSchema extends infer S extends Type.TSchema + ? TFromRest + : TFromRest + : Acc +) +function FromRest(T: T): TFromRest { + return T.map((L) => FromSchema(L)) as never; +} +// ------------------------------------------------------------------ +// FromEnumRest +// ------------------------------------------------------------------ +// prettier-ignore +type TFromEnumRest = ( + T extends readonly [infer L extends SValue, ...infer R extends SValue[]] + ? TFromEnumRest]> + : Acc +) +function FromEnumRest(T: T): TFromEnumRest { + return T.map((L) => Type.Literal(L)) as never; +} +// ------------------------------------------------------------------ +// AllOf +// ------------------------------------------------------------------ +// prettier-ignore +type TFromAllOf = ( + TFromRest extends infer Rest extends Type.TSchema[] + ? Type.TIntersectEvaluated + : Type.TNever +) +function FromAllOf(T: T): TFromAllOf { + return Type.IntersectEvaluated(FromRest(T.allOf), T); +} +// ------------------------------------------------------------------ +// AnyOf +// ------------------------------------------------------------------ +// prettier-ignore +type TFromAnyOf = ( + TFromRest extends infer Rest extends Type.TSchema[] + ? Type.TUnionEvaluated + : Type.TNever +) +function FromAnyOf(T: T): TFromAnyOf { + return Type.UnionEvaluated(FromRest(T.anyOf), T); +} +// ------------------------------------------------------------------ +// OneOf +// ------------------------------------------------------------------ +// prettier-ignore +type TFromOneOf = ( + TFromRest extends infer Rest extends Type.TSchema[] + ? Type.TUnionEvaluated + : Type.TNever +) +function FromOneOf(T: T): TFromOneOf { + return Type.UnionEvaluated(FromRest(T.oneOf), T); +} +// ------------------------------------------------------------------ +// Enum +// ------------------------------------------------------------------ +// prettier-ignore +type TFromEnum = ( + TFromEnumRest extends infer Elements extends Type.TSchema[] + ? Type.TUnionEvaluated + : Type.TNever +) +function FromEnum(T: T): TFromEnum { + return Type.UnionEvaluated(FromEnumRest(T.enum)); +} +// ------------------------------------------------------------------ +// Tuple +// ------------------------------------------------------------------ +// prettier-ignore +type TFromTuple = ( + TFromRest extends infer Elements extends Type.TSchema[] + ? Type.TTuple + : Type.TTuple<[]> +) +// prettier-ignore +function FromTuple(T: T): TFromTuple { + return Type.Tuple(FromRest(T.items), T) as never +} +// ------------------------------------------------------------------ +// Array +// ------------------------------------------------------------------ +// prettier-ignore +type TFromArray = ( + TFromSchema extends infer Items extends Type.TSchema + ? Type.TArray + : Type.TArray +) +// prettier-ignore +function FromArray(T: T): TFromArray { + return Type.Array(FromSchema(T.items), T) as never +} +// ------------------------------------------------------------------ +// Const +// ------------------------------------------------------------------ +// prettier-ignore +type TFromConst = ( + Type.Ensure> +) +function FromConst(T: T) { + return Type.Literal(T.const, T); +} +// ------------------------------------------------------------------ +// Object +// ------------------------------------------------------------------ +type TFromPropertiesIsOptional< + K extends PropertyKey, + R extends string | unknown, +> = unknown extends R ? true : K extends R ? false : true; +// prettier-ignore +type TFromProperties = Type.Evaluate<{ + -readonly [K in keyof T]: TFromPropertiesIsOptional extends true + ? Type.TOptional> + : TFromSchema +}> +// prettier-ignore +type TFromObject = ( + TFromProperties[number]> extends infer Properties extends Type.TProperties + ? Type.TObject + : Type.TObject<{}> +) +function FromObject(T: T): TFromObject { + const properties = globalThis.Object.getOwnPropertyNames(T.properties).reduce((Acc, K) => { + return { + ...Acc, + [K]: + T.required && T.required.includes(K) + ? FromSchema(T.properties[K]) + : Type.Optional(FromSchema(T.properties[K])), + }; + }, {} as Type.TProperties); + return Type.Object(properties, T) as never; +} +// ------------------------------------------------------------------ +// FromSchema +// ------------------------------------------------------------------ +// prettier-ignore +export type TFromSchema = ( + T extends SAllOf ? TFromAllOf : + T extends SAnyOf ? TFromAnyOf : + T extends SOneOf ? TFromOneOf : + T extends SEnum ? TFromEnum : + T extends SObject ? TFromObject : + T extends STuple ? TFromTuple : + T extends SArray ? TFromArray : + T extends SConst ? TFromConst : + T extends SString ? Type.TString : + T extends SNumber ? Type.TNumber : + T extends SInteger ? Type.TInteger : + T extends SBoolean ? Type.TBoolean : + T extends SNull ? Type.TNull : + Type.TUnknown +) +/** Parses a TypeBox type from raw JsonSchema */ +export function FromSchema(T: T): TFromSchema { + // prettier-ignore + return ( + IsSAllOf(T) ? FromAllOf(T) : + IsSAnyOf(T) ? FromAnyOf(T) : + IsSOneOf(T) ? FromOneOf(T) : + IsSEnum(T) ? FromEnum(T) : + IsSObject(T) ? FromObject(T) : + IsSTuple(T) ? FromTuple(T) : + IsSArray(T) ? FromArray(T) : + IsSConst(T) ? FromConst(T) : + IsSString(T) ? Type.String(T) : + IsSNumber(T) ? Type.Number(T) : + IsSInteger(T) ? Type.Integer(T) : + IsSBoolean(T) ? Type.Boolean(T) : + IsSNull(T) ? Type.Null(T) : + Type.Unknown(T || {}) + ) as never +} diff --git a/app/src/core/utils/typebox/index.ts b/app/src/core/utils/typebox/index.ts new file mode 100644 index 0000000..2b923c1 --- /dev/null +++ b/app/src/core/utils/typebox/index.ts @@ -0,0 +1,206 @@ +import { + Kind, + type ObjectOptions, + type SchemaOptions, + type Static, + type StaticDecode, + type StringOptions, + type TLiteral, + type TLiteralValue, + type TObject, + type TRecord, + type TSchema, + type TString, + Type, + TypeRegistry +} from "@sinclair/typebox"; +import { + DefaultErrorFunction, + Errors, + SetErrorFunction, + type ValueErrorIterator +} from "@sinclair/typebox/errors"; +import { Check, Default, Value, type ValueError } from "@sinclair/typebox/value"; +import { cloneDeep } from "lodash-es"; + +export type RecursivePartial = { + [P in keyof T]?: T[P] extends (infer U)[] + ? RecursivePartial[] + : T[P] extends object | undefined + ? RecursivePartial + : T[P]; +}; + +type ParseOptions = { + useDefaults?: boolean; + decode?: boolean; + onError?: (errors: ValueErrorIterator) => void; + forceParse?: boolean; + skipMark?: boolean; +}; + +const validationSymbol = Symbol("tb-parse-validation"); + +export class TypeInvalidError extends Error { + errors: ValueError[]; + constructor( + public schema: TSchema, + public data: unknown, + message?: string + ) { + //console.warn("errored schema", JSON.stringify(schema, null, 2)); + super(message ?? `Invalid: ${JSON.stringify(data)}`); + this.errors = [...Errors(schema, data)]; + } + + first() { + return this.errors[0]!; + } + + firstToString() { + const first = this.first(); + return `${first.message} at "${first.path}"`; + } + + toJSON() { + return { + message: this.message, + schema: this.schema, + data: this.data, + errors: this.errors + }; + } +} + +export function stripMark(obj: any) { + const newObj = cloneDeep(obj); + mark(newObj, false); + return newObj; +} + +export function mark(obj: any, validated = true) { + if (typeof obj === "object" && obj !== null && !Array.isArray(obj)) { + if (validated) { + obj[validationSymbol] = true; + } else { + delete obj[validationSymbol]; + } + for (const key in obj) { + if (typeof obj[key] === "object" && obj[key] !== null) { + mark(obj[key], validated); + } + } + } +} + +export function parse( + schema: Schema, + data: RecursivePartial>, + options?: ParseOptions +): Static { + if (!options?.forceParse && typeof data === "object" && validationSymbol in data) { + if (options?.useDefaults === false) { + return data as Static; + } + + // this is important as defaults are expected + return Default(schema, data as any) as Static; + } + + const parsed = options?.useDefaults === false ? data : Default(schema, data); + + if (Check(schema, parsed)) { + options?.skipMark !== true && mark(parsed, true); + return parsed as Static; + } else if (options?.onError) { + options.onError(Errors(schema, data)); + } else { + throw new TypeInvalidError(schema, data); + } + + // @todo: check this + return undefined as any; +} + +export function parseDecode( + schema: Schema, + data: RecursivePartial> +): StaticDecode { + //console.log("parseDecode", schema, data); + const parsed = Default(schema, data); + + if (Check(schema, parsed)) { + return parsed as StaticDecode; + } + //console.log("errors", ...Errors(schema, data)); + + throw new TypeInvalidError(schema, data); +} + +export function strictParse( + schema: Schema, + data: Static, + options?: ParseOptions +): Static { + return parse(schema, data as any, options); +} + +export function registerCustomTypeboxKinds(registry: typeof TypeRegistry) { + registry.Set("StringEnum", (schema: any, value: any) => { + return typeof value === "string" && schema.enum.includes(value); + }); +} +registerCustomTypeboxKinds(TypeRegistry); + +export const StringEnum = (values: T, options?: StringOptions) => + Type.Unsafe({ + [Kind]: "StringEnum", + type: "string", + enum: values, + ...options + }); + +// key value record compatible with RJSF and typebox inference +// acting like a Record, but using an Object with additionalProperties +export const StringRecord = (properties: T, options?: ObjectOptions) => + Type.Object({}, { ...options, additionalProperties: properties }) as unknown as TRecord< + TString, + typeof properties + >; + +// fixed value that only be what is given + prefilled +export const Const = (value: T, options?: SchemaOptions) => + Type.Literal(value, { ...options, default: value, const: value, readOnly: true }) as TLiteral; + +export const StringIdentifier = Type.String({ + pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$", + minLength: 2, + maxLength: 150 +}); + +SetErrorFunction((error) => { + if (error?.schema?.errorMessage) { + return error.schema.errorMessage; + } + + if (error?.schema?.[Kind] === "StringEnum") { + return `Expected: ${error.schema.enum.map((e) => `"${e}"`).join(", ")}`; + } + + return DefaultErrorFunction(error); +}); + +export { + Type, + type Static, + type StaticDecode, + type TSchema, + Kind, + type TObject, + type ValueError, + type SchemaOptions, + Value, + Default, + Errors, + Check +}; diff --git a/app/src/core/utils/types.d.ts b/app/src/core/utils/types.d.ts new file mode 100644 index 0000000..7afb18f --- /dev/null +++ b/app/src/core/utils/types.d.ts @@ -0,0 +1,8 @@ +export type Prettify = { + [K in keyof T]: T[K]; +} & NonNullable; + +// prettify recursively +export type PrettifyRec = { + [K in keyof T]: T[K] extends object ? Prettify : T[K]; +} & NonNullable; diff --git a/app/src/core/utils/uuid.ts b/app/src/core/utils/uuid.ts new file mode 100644 index 0000000..32397e8 --- /dev/null +++ b/app/src/core/utils/uuid.ts @@ -0,0 +1,4 @@ +// generates v4 +export function uuid(): string { + return crypto.randomUUID(); +} diff --git a/app/src/core/utils/xml.ts b/app/src/core/utils/xml.ts new file mode 100644 index 0000000..fdad21c --- /dev/null +++ b/app/src/core/utils/xml.ts @@ -0,0 +1,6 @@ +import { XMLParser } from "fast-xml-parser"; + +export function xmlToObject(xml: string) { + const parser = new XMLParser(); + return parser.parse(xml); +} diff --git a/app/src/data/AppData.ts b/app/src/data/AppData.ts new file mode 100644 index 0000000..821a3c6 --- /dev/null +++ b/app/src/data/AppData.ts @@ -0,0 +1,122 @@ +import { transformObject } from "core/utils"; +import { DataPermissions, Entity, EntityIndex, type EntityManager, type Field } from "data"; +import { Module } from "modules/Module"; +import { DataController } from "./api/DataController"; +import { + type AppDataConfig, + FIELDS, + RELATIONS, + type TAppDataEntity, + type TAppDataRelation, + dataConfigSchema +} from "./data-schema"; + +export class AppData extends Module { + static constructEntity(name: string, entityConfig: TAppDataEntity) { + const fields = transformObject(entityConfig.fields ?? {}, (fieldConfig, name) => { + const { type } = fieldConfig; + if (!(type in FIELDS)) { + throw new Error(`Field type "${type}" not found`); + } + + const { field } = FIELDS[type as any]; + const returnal = new field(name, fieldConfig.config) as Field; + return returnal; + }); + + // @todo: entity must be migrated to typebox + return new Entity( + name, + Object.values(fields), + entityConfig.config as any, + entityConfig.type as any + ); + } + + static constructRelation( + relationConfig: TAppDataRelation, + resolver: (name: Entity | string) => Entity + ) { + return new RELATIONS[relationConfig.type].cls( + resolver(relationConfig.source), + resolver(relationConfig.target), + relationConfig.config + ); + } + + override async build() { + const entities = transformObject(this.config.entities ?? {}, (entityConfig, name) => { + return AppData.constructEntity(name, entityConfig); + }); + + const _entity = (_e: Entity | string): Entity => { + const name = typeof _e === "string" ? _e : _e.name; + const entity = entities[name]; + if (entity) return entity; + throw new Error(`Entity "${name}" not found`); + }; + + const relations = transformObject(this.config.relations ?? {}, (relation) => + AppData.constructRelation(relation, _entity) + ); + + const indices = transformObject(this.config.indices ?? {}, (index, name) => { + const entity = _entity(index.entity)!; + const fields = index.fields.map((f) => entity.field(f)!); + return new EntityIndex(entity, fields, index.unique, name); + }); + + for (const entity of Object.values(entities)) { + this.ctx.em.addEntity(entity); + } + + for (const relation of Object.values(relations)) { + this.ctx.em.addRelation(relation); + } + + for (const index of Object.values(indices)) { + this.ctx.em.addIndex(index); + } + + this.ctx.server.route( + this.basepath, + new DataController(this.ctx, this.config).getController() + ); + this.ctx.guard.registerPermissions(Object.values(DataPermissions)); + + this.setBuilt(); + } + + getSchema() { + return dataConfigSchema; + } + + get em(): EntityManager { + this.throwIfNotBuilt(); + return this.ctx.em; + } + + private get basepath() { + return this.config.basepath ?? "/api/data"; + } + + override getOverwritePaths() { + return [ + /^entities\..*\.config$/, + /^entities\..*\.fields\..*\.config$/ + ///^entities\..*\.fields\..*\.config\.schema$/ + ]; + } + + /*registerController(server: AppServer) { + console.log("adding data controller to", this.basepath); + server.add(this.basepath, new DataController(this.em)); + }*/ + + override toJSON(secrets?: boolean): AppDataConfig { + return { + ...this.config, + ...this.em.toJSON() + }; + } +} diff --git a/app/src/data/api/DataApi.ts b/app/src/data/api/DataApi.ts new file mode 100644 index 0000000..0fa6d3e --- /dev/null +++ b/app/src/data/api/DataApi.ts @@ -0,0 +1,63 @@ +import type { EntityData, RepoQuery, RepositoryResponse } from "data"; +import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules"; + +export type DataApiOptions = BaseModuleApiOptions & { + defaultQuery?: Partial; +}; + +export class DataApi extends ModuleApi { + protected override getDefaultOptions(): Partial { + return { + basepath: "/api/data", + defaultQuery: { + limit: 10 + } + }; + } + + async readOne( + entity: string, + id: PrimaryFieldType, + query: Partial> = {} + ) { + return this.get>([entity, id], query); + } + + async readMany(entity: string, query: Partial = {}) { + return this.get>( + [entity], + query ?? this.options.defaultQuery + ); + } + + async readManyByReference( + entity: string, + id: PrimaryFieldType, + reference: string, + query: Partial = {} + ) { + return this.get>( + [entity, id, reference], + query ?? this.options.defaultQuery + ); + } + + async createOne(entity: string, input: EntityData) { + return this.post>([entity], input); + } + + async updateOne(entity: string, id: PrimaryFieldType, input: EntityData) { + return this.patch>([entity, id], input); + } + + async deleteOne(entity: string, id: PrimaryFieldType) { + return this.delete>([entity, id]); + } + + async count(entity: string, where: RepoQuery["where"] = {}) { + return this.post>( + [entity, "fn", "count"], + where + ); + } +} diff --git a/app/src/data/api/DataController.ts b/app/src/data/api/DataController.ts new file mode 100644 index 0000000..418e8ae --- /dev/null +++ b/app/src/data/api/DataController.ts @@ -0,0 +1,384 @@ +import { type ClassController, isDebug, tbValidator as tb } from "core"; +import { Type, objectCleanEmpty, objectTransform } from "core/utils"; +import { + DataPermissions, + type EntityData, + type EntityManager, + FieldClassMap, + type MutatorResponse, + PrimaryField, + type RepoQuery, + type RepositoryResponse, + TextField, + querySchema +} from "data"; +import { Hono } from "hono"; +import type { Handler } from "hono/types"; +import type { ModuleBuildContext } from "modules"; +import { AppData } from "../AppData"; +import { type AppDataConfig, FIELDS } from "../data-schema"; + +export class DataController implements ClassController { + constructor( + private readonly ctx: ModuleBuildContext, + private readonly config: AppDataConfig + ) { + /*console.log( + "data controller", + this.em.entities.map((e) => e.name) + );*/ + } + + get em(): EntityManager { + return this.ctx.em; + } + + get guard() { + return this.ctx.guard; + } + + repoResult = RepositoryResponse>( + res: T + ): Pick { + let meta: Partial = {}; + + if ("meta" in res) { + const { query, ...rest } = res.meta; + meta = rest; + if (isDebug()) meta.query = query; + } + + const template = { data: res.data, meta }; + + // @todo: this works but it breaks in FE (need to improve DataTable) + //return objectCleanEmpty(template) as any; + // filter empty + return Object.fromEntries( + Object.entries(template).filter(([_, v]) => typeof v !== "undefined" && v !== null) + ) as any; + } + + mutatorResult(res: MutatorResponse | MutatorResponse) { + const template = { data: res.data }; + + // filter empty + //return objectCleanEmpty(template); + return Object.fromEntries(Object.entries(template).filter(([_, v]) => v !== undefined)); + } + + entityExists(entity: string) { + try { + return !!this.em.entity(entity); + } catch (e) { + return false; + } + } + + getController(): Hono { + const hono = new Hono(); + const definedEntities = this.em.entities.map((e) => e.name); + const tbNumber = Type.Transform(Type.String({ pattern: "^[1-9][0-9]{0,}$" })) + .Decode(Number.parseInt) + .Encode(String); + + // @todo: sample implementation how to augment handler with additional info + function handler(name: string, h: HH): any { + const func = h; + // @ts-ignore + func.description = name; + return func; + } + + // add timing + /*hono.use("*", async (c, next) => { + startTime(c, "data"); + await next(); + endTime(c, "data"); + });*/ + + // info + hono.get( + "/", + handler("data info", (c) => { + // sample implementation + return c.json(this.em.toJSON()); + }) + ); + + // sync endpoint + hono.get("/sync", async (c) => { + this.guard.throwUnlessGranted(DataPermissions.databaseSync); + + const force = c.req.query("force") === "1"; + const drop = c.req.query("drop") === "1"; + //console.log("force", force); + const tables = await this.em.schema().introspect(); + //console.log("tables", tables); + const changes = await this.em.schema().sync({ + force, + drop + }); + return c.json({ tables: tables.map((t) => t.name), changes }); + }); + + /** + * Function endpoints + */ + hono + // fn: count + .post( + "/:entity/fn/count", + tb("param", Type.Object({ entity: Type.String() })), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + const { entity } = c.req.valid("param"); + if (!this.entityExists(entity)) { + return c.notFound(); + } + + const where = c.req.json() as any; + const result = await this.em.repository(entity).count(where); + return c.json({ entity, count: result.count }); + } + ) + // fn: exists + .post( + "/:entity/fn/exists", + tb("param", Type.Object({ entity: Type.String() })), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + const { entity } = c.req.valid("param"); + if (!this.entityExists(entity)) { + return c.notFound(); + } + + const where = c.req.json() as any; + const result = await this.em.repository(entity).exists(where); + return c.json({ entity, exists: result.exists }); + } + ); + + /** + * Read endpoints + */ + hono + // read entity schema + .get("/schema.json", async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + const url = new URL(c.req.url); + const $id = `${url.origin}${this.config.basepath}/schema.json`; + const schemas = Object.fromEntries( + this.em.entities.map((e) => [ + e.name, + { + $ref: `schemas/${e.name}` + } + ]) + ); + return c.json({ + $schema: "https://json-schema.org/draft/2020-12/schema", + $id, + properties: schemas + }); + }) + // read schema + .get( + "/schemas/:entity", + tb("param", Type.Object({ entity: Type.String() })), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + //console.log("request", c.req.raw); + const { entity } = c.req.param(); + if (!this.entityExists(entity)) { + console.log("not found", entity, definedEntities); + return c.notFound(); + } + const _entity = this.em.entity(entity); + const schema = _entity.toSchema(); + const url = new URL(c.req.url); + const base = `${url.origin}${this.config.basepath}`; + const $id = `${base}/schemas/${entity}`; + return c.json({ + $schema: `${base}/schema.json`, + $id, + title: _entity.label, + $comment: _entity.config.description, + ...schema + }); + } + ) + // read many + .get( + "/:entity", + tb("param", Type.Object({ entity: Type.String() })), + tb("query", querySchema), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + //console.log("request", c.req.raw); + const { entity } = c.req.param(); + if (!this.entityExists(entity)) { + console.log("not found", entity, definedEntities); + return c.notFound(); + } + const options = c.req.valid("query") as RepoQuery; + //console.log("before", this.ctx.emgr.Events); + const result = await this.em.repository(entity).findMany(options); + + return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + } + ) + + // read one + .get( + "/:entity/:id", + tb( + "param", + Type.Object({ + entity: Type.String(), + id: tbNumber + }) + ), + tb("query", querySchema), + /*zValidator("param", z.object({ entity: z.string(), id: z.coerce.number() })), + zValidator("query", repoQuerySchema),*/ + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + const { entity, id } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const options = c.req.valid("query") as RepoQuery; + const result = await this.em.repository(entity).findId(Number(id), options); + + return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + } + ) + // read many by reference + .get( + "/:entity/:id/:reference", + tb( + "param", + Type.Object({ + entity: Type.String(), + id: tbNumber, + reference: Type.String() + }) + ), + tb("query", querySchema), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + const { entity, id, reference } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + + const options = c.req.valid("query") as RepoQuery; + const result = await this.em + .repository(entity) + .findManyByReference(Number(id), reference, options); + + return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + } + ) + // func query + .post( + "/:entity/query", + tb("param", Type.Object({ entity: Type.String() })), + tb("json", querySchema), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityRead); + + const { entity } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const options = (await c.req.valid("json")) as RepoQuery; + console.log("options", options); + const result = await this.em.repository(entity).findMany(options); + + return c.json(this.repoResult(result), { status: result.data ? 200 : 404 }); + } + ); + + /** + * Mutation endpoints + */ + // insert one + hono + .post("/:entity", tb("param", Type.Object({ entity: Type.String() })), async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityCreate); + + const { entity } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const body = (await c.req.json()) as EntityData; + const result = await this.em.mutator(entity).insertOne(body); + + return c.json(this.mutatorResult(result), 201); + }) + // update one + .patch( + "/:entity/:id", + tb("param", Type.Object({ entity: Type.String(), id: tbNumber })), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityUpdate); + + const { entity, id } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const body = (await c.req.json()) as EntityData; + const result = await this.em.mutator(entity).updateOne(Number(id), body); + + return c.json(this.mutatorResult(result)); + } + ) + // delete one + .delete( + "/:entity/:id", + tb("param", Type.Object({ entity: Type.String(), id: tbNumber })), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityDelete); + + const { entity, id } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const result = await this.em.mutator(entity).deleteOne(Number(id)); + + return c.json(this.mutatorResult(result)); + } + ) + + // delete many + .delete( + "/:entity", + tb("param", Type.Object({ entity: Type.String() })), + tb("json", querySchema.properties.where), + async (c) => { + this.guard.throwUnlessGranted(DataPermissions.entityDelete); + + //console.log("request", c.req.raw); + const { entity } = c.req.param(); + if (!this.entityExists(entity)) { + return c.notFound(); + } + const where = c.req.valid("json") as RepoQuery["where"]; + console.log("where", where); + + const result = await this.em.mutator(entity).deleteMany(where); + + return c.json(this.mutatorResult(result)); + } + ); + + return hono; + } +} diff --git a/app/src/data/connection/Connection.ts b/app/src/data/connection/Connection.ts new file mode 100644 index 0000000..b3f7e10 --- /dev/null +++ b/app/src/data/connection/Connection.ts @@ -0,0 +1,97 @@ +import { + type AliasableExpression, + type DatabaseIntrospector, + type Expression, + type Kysely, + type KyselyPlugin, + type RawBuilder, + type SelectQueryBuilder, + type SelectQueryNode, + type Simplify, + sql +} from "kysely"; + +export type QB = SelectQueryBuilder; + +export type IndexMetadata = { + name: string; + table: string; + isUnique: boolean; + columns: { name: string; order: number }[]; +}; + +export interface ConnectionIntrospector extends DatabaseIntrospector { + getIndices(tbl_name?: string): Promise; +} + +export interface SelectQueryBuilderExpression extends AliasableExpression { + get isSelectQueryBuilder(): true; + toOperationNode(): SelectQueryNode; +} + +export type DbFunctions = { + jsonObjectFrom(expr: SelectQueryBuilderExpression): RawBuilder | null>; + jsonArrayFrom(expr: SelectQueryBuilderExpression): RawBuilder[]>; + jsonBuildObject>>( + obj: O + ): RawBuilder< + Simplify<{ + [K in keyof O]: O[K] extends Expression ? V : never; + }> + >; +}; + +export abstract class Connection { + kysely: Kysely; + + constructor( + kysely: Kysely, + public fn: Partial = {}, + protected plugins: KyselyPlugin[] = [] + ) { + this.kysely = kysely; + } + + getIntrospector(): ConnectionIntrospector { + return this.kysely.introspection as ConnectionIntrospector; + } + + supportsBatching(): boolean { + return false; + } + + supportsIndices(): boolean { + return false; + } + + async ping(): Promise { + const res = await sql`SELECT 1`.execute(this.kysely); + return res.rows.length > 0; + } + + protected async batch( + queries: [...Queries] + ): Promise<{ + [K in keyof Queries]: Awaited>; + }> { + throw new Error("Batching not supported"); + } + + async batchQuery( + queries: [...Queries] + ): Promise<{ + [K in keyof Queries]: Awaited>; + }> { + // bypass if no client support + if (!this.supportsBatching()) { + const data: any = []; + for (const q of queries) { + const result = await q.execute(); + data.push(result); + } + return data; + } + + return await this.batch(queries); + } +} diff --git a/app/src/data/connection/LibsqlConnection.ts b/app/src/data/connection/LibsqlConnection.ts new file mode 100644 index 0000000..54202d3 --- /dev/null +++ b/app/src/data/connection/LibsqlConnection.ts @@ -0,0 +1,100 @@ +import { type Client, type InStatement, createClient } from "@libsql/client/web"; +import { LibsqlDialect } from "@libsql/kysely-libsql"; +import { type DatabaseIntrospector, Kysely, ParseJSONResultsPlugin, sql } from "kysely"; +import { FilterNumericKeysPlugin } from "../plugins/FilterNumericKeysPlugin"; +import { KyselyPluginRunner } from "../plugins/KyselyPluginRunner"; +import type { QB } from "./Connection"; +import { SqliteConnection } from "./SqliteConnection"; +import { SqliteIntrospector } from "./SqliteIntrospector"; + +export const LIBSQL_PROTOCOLS = ["wss", "https", "libsql"] as const; +export type LibSqlCredentials = { + url: string; + authToken?: string; + protocol?: (typeof LIBSQL_PROTOCOLS)[number]; +}; + +class CustomLibsqlDialect extends LibsqlDialect { + override createIntrospector(db: Kysely): DatabaseIntrospector { + return new SqliteIntrospector(db, { + excludeTables: ["libsql_wasm_func_table"] + }); + } +} + +export class LibsqlConnection extends SqliteConnection { + private client: Client; + + constructor(client: Client); + constructor(credentials: LibSqlCredentials); + constructor(clientOrCredentials: Client | LibSqlCredentials) { + const plugins = [new FilterNumericKeysPlugin(), new ParseJSONResultsPlugin()]; + let client: Client; + if ("url" in clientOrCredentials) { + let { url, authToken, protocol } = clientOrCredentials; + if (protocol && LIBSQL_PROTOCOLS.includes(protocol)) { + console.log("changing protocol to", protocol); + const [, rest] = url.split("://"); + url = `${protocol}://${rest}`; + } + + //console.log("using", url, { protocol }); + + client = createClient({ url, authToken }); + } else { + //console.log("-- client provided"); + client = clientOrCredentials; + } + + const kysely = new Kysely({ + // @ts-expect-error libsql has type issues + dialect: new CustomLibsqlDialect({ client }), + plugins + //log: ["query"], + }); + + super(kysely, {}, plugins); + this.client = client; + } + + override supportsBatching(): boolean { + return true; + } + + override supportsIndices(): boolean { + return true; + } + + getClient(): Client { + return this.client; + } + + protected override async batch( + queries: [...Queries] + ): Promise<{ + [K in keyof Queries]: Awaited>; + }> { + const stms: InStatement[] = queries.map((q) => { + const compiled = q.compile(); + //console.log("compiled", compiled.sql, compiled.parameters); + return { + sql: compiled.sql, + args: compiled.parameters as any[] + }; + }); + + const res = await this.client.batch(stms); + + // let it run through plugins + const kyselyPlugins = new KyselyPluginRunner(this.plugins); + + const data: any = []; + for (const r of res) { + const rows = await kyselyPlugins.transformResultRows(r.rows); + data.push(rows); + } + //console.log("data", data); + + return data; + } +} diff --git a/app/src/data/connection/SqliteConnection.ts b/app/src/data/connection/SqliteConnection.ts new file mode 100644 index 0000000..7a32093 --- /dev/null +++ b/app/src/data/connection/SqliteConnection.ts @@ -0,0 +1,22 @@ +import type { Kysely, KyselyPlugin } from "kysely"; +import { jsonArrayFrom, jsonBuildObject, jsonObjectFrom } from "kysely/helpers/sqlite"; +import { Connection, type DbFunctions } from "./Connection"; + +export class SqliteConnection extends Connection { + constructor(kysely: Kysely, fn: Partial = {}, plugins: KyselyPlugin[] = []) { + super( + kysely, + { + ...fn, + jsonArrayFrom, + jsonObjectFrom, + jsonBuildObject + }, + plugins + ); + } + + override supportsIndices(): boolean { + return true; + } +} diff --git a/app/src/data/connection/SqliteIntrospector.ts b/app/src/data/connection/SqliteIntrospector.ts new file mode 100644 index 0000000..cf68816 --- /dev/null +++ b/app/src/data/connection/SqliteIntrospector.ts @@ -0,0 +1,164 @@ +import type { + DatabaseIntrospector, + DatabaseMetadata, + DatabaseMetadataOptions, + ExpressionBuilder, + Kysely, + SchemaMetadata, + TableMetadata, +} from "kysely"; +import { DEFAULT_MIGRATION_LOCK_TABLE, DEFAULT_MIGRATION_TABLE, sql } from "kysely"; +import type { ConnectionIntrospector, IndexMetadata } from "./Connection"; + +export type SqliteIntrospectorConfig = { + excludeTables?: string[]; +}; + +export class SqliteIntrospector implements DatabaseIntrospector, ConnectionIntrospector { + readonly #db: Kysely; + readonly _excludeTables: string[] = []; + + constructor(db: Kysely, config: SqliteIntrospectorConfig = {}) { + this.#db = db; + this._excludeTables = config.excludeTables ?? []; + } + + async getSchemas(): Promise { + // Sqlite doesn't support schemas. + return []; + } + + async getIndices(tbl_name?: string): Promise { + const indices = await this.#db + .selectFrom("sqlite_master") + .where("type", "=", "index") + .$if(!!tbl_name, (eb) => eb.where("tbl_name", "=", tbl_name)) + .select("name") + .$castTo<{ name: string }>() + .execute(); + + return Promise.all(indices.map(({ name }) => this.#getIndexMetadata(name))); + } + + async #getIndexMetadata(index: string): Promise { + const db = this.#db; + + // Get the SQL that was used to create the index. + const indexDefinition = await db + .selectFrom("sqlite_master") + .where("name", "=", index) + .select(["sql", "tbl_name", "type"]) + .$castTo<{ sql: string | undefined; tbl_name: string; type: string }>() + .executeTakeFirstOrThrow(); + + //console.log("--indexDefinition--", indexDefinition, index); + + // check unique by looking for the word "unique" in the sql + const isUnique = indexDefinition.sql?.match(/unique/i) != null; + + const columns = await db + .selectFrom( + sql<{ + seqno: number; + cid: number; + name: string; + }>`pragma_index_info(${index})`.as("index_info"), + ) + .select(["seqno", "cid", "name"]) + .orderBy("cid") + .execute(); + + return { + name: index, + table: indexDefinition.tbl_name, + isUnique: isUnique, + columns: columns.map((col) => ({ + name: col.name, + order: col.seqno, + })), + }; + } + + private excludeTables(tables: string[] = []) { + return (eb: ExpressionBuilder) => { + const and = tables.map((t) => eb("name", "!=", t)); + return eb.and(and); + }; + } + + async getTables( + options: DatabaseMetadataOptions = { withInternalKyselyTables: false }, + ): Promise { + let query = this.#db + .selectFrom("sqlite_master") + .where("type", "in", ["table", "view"]) + .where("name", "not like", "sqlite_%") + .select("name") + .orderBy("name") + .$castTo<{ name: string }>(); + + if (!options.withInternalKyselyTables) { + query = query.where( + this.excludeTables([DEFAULT_MIGRATION_TABLE, DEFAULT_MIGRATION_LOCK_TABLE]), + ); + } + if (this._excludeTables.length > 0) { + query = query.where(this.excludeTables(this._excludeTables)); + } + + const tables = await query.execute(); + return Promise.all(tables.map(({ name }) => this.#getTableMetadata(name))); + } + + async getMetadata(options?: DatabaseMetadataOptions): Promise { + return { + tables: await this.getTables(options), + }; + } + + async #getTableMetadata(table: string): Promise { + const db = this.#db; + + // Get the SQL that was used to create the table. + const tableDefinition = await db + .selectFrom("sqlite_master") + .where("name", "=", table) + .select(["sql", "type"]) + .$castTo<{ sql: string | undefined; type: string }>() + .executeTakeFirstOrThrow(); + + // Try to find the name of the column that has `autoincrement` 🤦 + const autoIncrementCol = tableDefinition.sql + ?.split(/[\(\),]/) + ?.find((it) => it.toLowerCase().includes("autoincrement")) + ?.trimStart() + ?.split(/\s+/)?.[0] + ?.replace(/["`]/g, ""); + + const columns = await db + .selectFrom( + sql<{ + name: string; + type: string; + notnull: 0 | 1; + dflt_value: any; + }>`pragma_table_info(${table})`.as("table_info"), + ) + .select(["name", "type", "notnull", "dflt_value"]) + .orderBy("cid") + .execute(); + + return { + name: table, + isView: tableDefinition.type === "view", + columns: columns.map((col) => ({ + name: col.name, + dataType: col.type, + isNullable: !col.notnull, + isAutoIncrementing: col.name === autoIncrementCol, + hasDefaultValue: col.dflt_value != null, + comment: undefined, + })), + }; + } +} diff --git a/app/src/data/connection/SqliteLocalConnection.ts b/app/src/data/connection/SqliteLocalConnection.ts new file mode 100644 index 0000000..0b1a8c8 --- /dev/null +++ b/app/src/data/connection/SqliteLocalConnection.ts @@ -0,0 +1,31 @@ +import type { DatabaseIntrospector, SqliteDatabase } from "kysely"; +import { Kysely, SqliteDialect } from "kysely"; +import { DeserializeJsonValuesPlugin } from "../plugins/DeserializeJsonValuesPlugin"; +import { SqliteConnection } from "./SqliteConnection"; +import { SqliteIntrospector } from "./SqliteIntrospector"; + +class CustomSqliteDialect extends SqliteDialect { + override createIntrospector(db: Kysely): DatabaseIntrospector { + return new SqliteIntrospector(db, { + excludeTables: ["test_table"] + }); + } +} + +export class SqliteLocalConnection extends SqliteConnection { + constructor(private database: SqliteDatabase) { + const plugins = [new DeserializeJsonValuesPlugin()]; + const kysely = new Kysely({ + dialect: new CustomSqliteDialect({ database }), + plugins + //log: ["query"], + }); + + super(kysely); + this.plugins = plugins; + } + + override supportsIndices(): boolean { + return true; + } +} diff --git a/app/src/data/data-schema.ts b/app/src/data/data-schema.ts new file mode 100644 index 0000000..cc53bfd --- /dev/null +++ b/app/src/data/data-schema.ts @@ -0,0 +1,83 @@ +import { type Static, StringRecord, Type, objectTransform } from "core/utils"; +import { + FieldClassMap, + RelationClassMap, + RelationFieldClassMap, + entityConfigSchema, + entityTypes +} from "data"; +import { MediaField, mediaFieldConfigSchema } from "../media/MediaField"; + +export const FIELDS = { + ...FieldClassMap, + ...RelationFieldClassMap, + media: { schema: mediaFieldConfigSchema, field: MediaField } +}; +export type FieldType = keyof typeof FIELDS; + +export const RELATIONS = RelationClassMap; + +export const fieldsSchemaObject = objectTransform(FIELDS, (field, name) => { + return Type.Object( + { + type: Type.Const(name, { default: name, readOnly: true }), + config: Type.Optional(field.schema) + }, + { + title: name + } + ); +}); +export const fieldsSchema = Type.Union(Object.values(fieldsSchemaObject)); +export const entityFields = StringRecord(fieldsSchema); +export type TAppDataField = Static; +export type TAppDataEntityFields = Static; + +export const entitiesSchema = Type.Object({ + //name: Type.String(), + type: Type.Optional(Type.String({ enum: entityTypes, default: "regular", readOnly: true })), + config: Type.Optional(entityConfigSchema), + fields: Type.Optional(entityFields) +}); +export type TAppDataEntity = Static; + +export const relationsSchema = Object.entries(RelationClassMap).map(([name, relationClass]) => { + return Type.Object( + { + type: Type.Const(name, { default: name, readOnly: true }), + source: Type.String(), + target: Type.String(), + config: Type.Optional(relationClass.schema) + }, + { + title: name + } + ); +}); +export type TAppDataRelation = Static<(typeof relationsSchema)[number]>; + +export const indicesSchema = Type.Object( + { + entity: Type.String(), + fields: Type.Array(Type.String(), { minItems: 1 }), + //name: Type.Optional(Type.String()), + unique: Type.Optional(Type.Boolean({ default: false })) + }, + { + additionalProperties: false + } +); + +export const dataConfigSchema = Type.Object( + { + basepath: Type.Optional(Type.String({ default: "/api/data" })), + entities: Type.Optional(StringRecord(entitiesSchema, { default: {} })), + relations: Type.Optional(StringRecord(Type.Union(relationsSchema), { default: {} })), + indices: Type.Optional(StringRecord(indicesSchema, { default: {} })) + }, + { + additionalProperties: false + } +); + +export type AppDataConfig = Static; diff --git a/app/src/data/entities/Entity.ts b/app/src/data/entities/Entity.ts new file mode 100644 index 0000000..4b14f5d --- /dev/null +++ b/app/src/data/entities/Entity.ts @@ -0,0 +1,238 @@ +import { config } from "core"; +import { + type Static, + StringEnum, + Type, + parse, + snakeToPascalWithSpaces, + transformObject +} from "core/utils"; +import { type Field, PrimaryField, type TActionContext, type TRenderContext } from "../fields"; + +// @todo: entity must be migrated to typebox +export const entityConfigSchema = Type.Object( + { + name: Type.Optional(Type.String()), + name_singular: Type.Optional(Type.String()), + description: Type.Optional(Type.String()), + sort_field: Type.Optional(Type.String({ default: config.data.default_primary_field })), + sort_dir: Type.Optional(StringEnum(["asc", "desc"], { default: "asc" })) + }, + { + additionalProperties: false + } +); + +export type EntityConfig = Static; + +export type EntityData = Record; +export type EntityJSON = ReturnType; + +/** + * regular: normal defined entity + * system: generated by the system, e.g. "users" from auth + * generated: result of a relation, e.g. many-to-many relation's connection entity + */ +export const entityTypes = ["regular", "system", "generated"] as const; +export type TEntityType = (typeof entityTypes)[number]; + +/** + * @todo: add check for adding fields (primary and relation not allowed) + * @todo: add option to disallow api deletes (or api actions in general) + */ +export class Entity< + EntityName extends string = string, + Fields extends Record> = Record> +> { + readonly #_name!: EntityName; + readonly #_fields!: Fields; // only for types + + readonly name: string; + readonly fields: Field[]; + readonly config: EntityConfig; + protected data: EntityData[] | undefined; + readonly type: TEntityType = "regular"; + + constructor(name: string, fields?: Field[], config?: EntityConfig, type?: TEntityType) { + if (typeof name !== "string" || name.length === 0) { + throw new Error("Entity name must be a non-empty string"); + } + + this.name = name; + this.config = parse(entityConfigSchema, config || {}) as EntityConfig; + + // add id field if not given + // @todo: add test + const primary_count = fields?.filter((field) => field instanceof PrimaryField).length ?? 0; + if (primary_count > 1) { + throw new Error(`Entity "${name}" has more than one primary field`); + } + this.fields = primary_count === 1 ? [] : [new PrimaryField()]; + + if (fields) { + fields.forEach((field) => this.addField(field)); + } + + if (type) this.type = type; + } + + static create(args: { + name: string; + fields?: Field[]; + config?: EntityConfig; + type?: TEntityType; + }) { + return new Entity(args.name, args.fields, args.config, args.type); + } + + // @todo: add test + getType(): TEntityType { + return this.type; + } + + getSelect(alias?: string, context?: TActionContext | TRenderContext): string[] { + return this.getFields() + .filter((field) => !field.isHidden(context ?? "read")) + .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name)); + } + + getDefaultSort() { + return { + by: this.config.sort_field, + dir: this.config.sort_dir + }; + } + + getAliasedSelectFrom( + select: string[], + _alias?: string, + context?: TActionContext | TRenderContext + ): string[] { + const alias = _alias ?? this.name; + return this.getFields() + .filter( + (field) => + !field.isVirtual() && + !field.isHidden(context ?? "read") && + select.includes(field.name) + ) + .map((field) => (alias ? `${alias}.${field.name} as ${field.name}` : field.name)); + } + + getFillableFields(context?: TActionContext, include_virtual?: boolean): Field[] { + return this.getFields(include_virtual).filter((field) => field.isFillable(context)); + } + + getRequiredFields(): Field[] { + return this.getFields().filter((field) => field.isRequired()); + } + + getDefaultObject(): EntityData { + return this.getFields().reduce((acc, field) => { + if (field.hasDefault()) { + acc[field.name] = field.getDefault(); + } + return acc; + }, {} as EntityData); + } + + getField(name: string): Field | undefined { + return this.fields.find((field) => field.name === name); + } + + __experimental_replaceField(name: string, field: Field) { + const index = this.fields.findIndex((f) => f.name === name); + if (index === -1) { + throw new Error(`Field "${name}" not found on entity "${this.name}"`); + } + + this.fields[index] = field; + } + + getPrimaryField(): PrimaryField { + return this.fields[0] as PrimaryField; + } + + id(): PrimaryField { + return this.getPrimaryField(); + } + + get label(): string { + return snakeToPascalWithSpaces(this.config.name ?? this.name); + } + + field(name: string): Field | undefined { + return this.getField(name); + } + + getFields(include_virtual: boolean = false): Field[] { + if (include_virtual) return this.fields; + return this.fields.filter((f) => !f.isVirtual()); + } + + addField(field: Field) { + const existing = this.getField(field.name); + // make unique name check + if (existing) { + // @todo: for now adding a graceful method + if (JSON.stringify(existing) === JSON.stringify(field)) { + /*console.warn( + `Field "${field.name}" already exists on entity "${this.name}", but it's the same, so skipping.`, + );*/ + return; + } + + throw new Error(`Field "${field.name}" already exists on entity "${this.name}"`); + } + + this.fields.push(field); + } + + __setData(data: EntityData[]) { + this.data = data; + } + + isValidData(data: EntityData, context: TActionContext, explain?: boolean): boolean { + const fields = this.getFillableFields(context, false); + //const fields = this.fields; + //console.log("data", data); + for (const field of fields) { + if (!field.isValid(data[field.name], context)) { + console.log("Entity.isValidData:invalid", context, field.name, data[field.name]); + if (explain) { + throw new Error(`Field "${field.name}" has invalid data: "${data[field.name]}"`); + } + + return false; + } + } + + return true; + } + + toSchema(clean?: boolean): object { + const fields = Object.fromEntries(this.fields.map((field) => [field.name, field])); + const schema = Type.Object( + transformObject(fields, (field) => ({ + title: field.config.label, + $comment: field.config.description, + $field: field.type, + readOnly: !field.isFillable("update") ? true : undefined, + writeOnly: !field.isFillable("create") ? true : undefined, + ...field.toJsonSchema() + })) + ); + + return clean ? JSON.parse(JSON.stringify(schema)) : schema; + } + + toJSON() { + return { + //name: this.name, + type: this.type, + //fields: transformObject(this.fields, (field) => field.toJSON()), + fields: Object.fromEntries(this.fields.map((field) => [field.name, field.toJSON()])), + config: this.config + }; + } +} diff --git a/app/src/data/entities/EntityManager.ts b/app/src/data/entities/EntityManager.ts new file mode 100644 index 0000000..353d3a9 --- /dev/null +++ b/app/src/data/entities/EntityManager.ts @@ -0,0 +1,266 @@ +import { EventManager } from "core/events"; +import { sql } from "kysely"; +import { Connection } from "../connection/Connection"; +import { + EntityNotDefinedException, + TransformRetrieveFailedException, + UnableToConnectException +} from "../errors"; +import { MutatorEvents, RepositoryEvents } from "../events"; +import type { EntityIndex } from "../fields/indices/EntityIndex"; +import type { EntityRelation } from "../relations"; +import { RelationAccessor } from "../relations/RelationAccessor"; +import { SchemaManager } from "../schema/SchemaManager"; +import { Entity } from "./Entity"; +import { type EntityData, Mutator, Repository } from "./index"; + +export class EntityManager { + connection: Connection; + + private _entities: Entity[] = []; + private _relations: EntityRelation[] = []; + private _indices: EntityIndex[] = []; + private _schema?: SchemaManager; + readonly emgr: EventManager; + static readonly Events = { ...MutatorEvents, ...RepositoryEvents }; + + constructor( + entities: Entity[], + connection: Connection, + relations: EntityRelation[] = [], + indices: EntityIndex[] = [], + emgr?: EventManager + ) { + // add entities & relations + entities.forEach((entity) => this.addEntity(entity)); + relations.forEach((relation) => this.addRelation(relation)); + indices.forEach((index) => this.addIndex(index)); + + if (!(connection instanceof Connection)) { + throw new UnableToConnectException(""); + } + + this.connection = connection; + this.emgr = emgr ?? new EventManager(); + //console.log("registering events", EntityManager.Events); + this.emgr.registerEvents(EntityManager.Events); + } + + /** + * Forks the EntityManager without the EventManager. + * This is useful when used inside an event handler. + */ + fork(): EntityManager { + return new EntityManager(this._entities, this.connection, this._relations, this._indices); + } + + get entities(): Entity[] { + return this._entities; + } + + get relations(): RelationAccessor { + return new RelationAccessor(this._relations); + } + + get indices(): EntityIndex[] { + return this._indices; + } + + async ping(): Promise { + const res = await sql`SELECT 1`.execute(this.connection.kysely); + return res.rows.length > 0; + } + + addEntity(entity: Entity) { + const existing = this.entities.find((e) => e.name === entity.name); + // check if already exists by name + if (existing) { + // @todo: for now adding a graceful method + if (JSON.stringify(existing) === JSON.stringify(entity)) { + //console.warn(`Entity "${entity.name}" already exists, but it's the same, so skipping.`); + return; + } + + throw new Error(`Entity "${entity.name}" already exists`); + } + + this.entities.push(entity); + } + + entity(name: string): Entity { + const entity = this.entities.find((e) => e.name === name); + if (!entity) { + throw new EntityNotDefinedException(name); + } + + return entity; + } + + hasEntity(entity: string): boolean; + hasEntity(entity: Entity): boolean; + hasEntity(nameOrEntity: string | Entity): boolean { + const name = typeof nameOrEntity === "string" ? nameOrEntity : nameOrEntity.name; + return this.entities.some((e) => e.name === name); + } + + hasIndex(index: string): boolean; + hasIndex(index: EntityIndex): boolean; + hasIndex(nameOrIndex: string | EntityIndex): boolean { + const name = typeof nameOrIndex === "string" ? nameOrIndex : nameOrIndex.name; + return this.indices.some((e) => e.name === name); + } + + addRelation(relation: EntityRelation) { + // check if entities are registered + if (!this.entity(relation.source.entity.name) || !this.entity(relation.target.entity.name)) { + throw new Error("Relation source or target entity not found"); + } + + // @todo: potentially add name to relation in order to have multiple + const found = this._relations.find((r) => { + const equalSourceTarget = + r.source.entity.name === relation.source.entity.name && + r.target.entity.name === relation.target.entity.name; + const equalReferences = + r.source.reference === relation.source.reference && + r.target.reference === relation.target.reference; + + return ( + //r.type === relation.type && // ignore type for now + equalSourceTarget && equalReferences + ); + }); + + if (found) { + throw new Error( + `Relation "${relation.type}" between "${relation.source.entity.name}" ` + + `and "${relation.target.entity.name}" already exists` + ); + } + + this._relations.push(relation); + relation.initialize(this); + } + + relationsOf(entity_name: string): EntityRelation[] { + return this.relations.relationsOf(this.entity(entity_name)); + } + + relationOf(entity_name: string, reference: string): EntityRelation | undefined { + return this.relations.relationOf(this.entity(entity_name), reference); + } + + hasRelations(entity_name: string): boolean { + return this.relations.hasRelations(this.entity(entity_name)); + } + + relatedEntitiesOf(entity_name: string): Entity[] { + return this.relations.relatedEntitiesOf(this.entity(entity_name)); + } + + relationReferencesOf(entity_name: string): string[] { + return this.relations.relationReferencesOf(this.entity(entity_name)); + } + + repository(_entity: Entity | string) { + const entity = _entity instanceof Entity ? _entity : this.entity(_entity); + return new Repository(this, entity, this.emgr); + } + + repo( + _entity: E + ): Repository< + DB, + E extends Entity ? (Name extends keyof DB ? Name : never) : never + > { + return new Repository(this, _entity, this.emgr); + } + + _repo(_entity: TB): Repository { + const entity = this.entity(_entity as any); + return new Repository(this, entity, this.emgr); + } + + mutator(_entity: Entity | string) { + const entity = _entity instanceof Entity ? _entity : this.entity(_entity); + return new Mutator(this, entity, this.emgr); + } + + addIndex(index: EntityIndex, force = false) { + // check if already exists by name + if (this.indices.find((e) => e.name === index.name)) { + if (force) { + throw new Error(`Index "${index.name}" already exists`); + } + return; + } + + this._indices.push(index); + } + + getIndicesOf(_entity: Entity | string): EntityIndex[] { + const entity = _entity instanceof Entity ? _entity : this.entity(_entity); + return this.indices.filter((index) => index.entity.name === entity.name); + } + + schema() { + if (!this._schema) { + this._schema = new SchemaManager(this); + } + + return this._schema; + } + + // @todo: centralize and add tests + hydrate(entity_name: string, _data: EntityData[]) { + const entity = this.entity(entity_name); + const data: EntityData[] = []; + + for (const row of _data) { + for (let [key, value] of Object.entries(row)) { + const field = entity.getField(key); + + if (!field || field.isVirtual()) { + // if relation, use related entity to hydrate + const relation = this.relationOf(entity_name, key); + if (relation) { + if (!value) continue; + + value = relation.hydrate(key, Array.isArray(value) ? value : [value], this); + row[key] = value; + continue; + } else if (field?.isVirtual()) { + continue; + } + + throw new Error(`Field "${key}" not found on entity "${entity.name}"`); + } + + try { + if (value === null && field.hasDefault()) { + row[key] = field.getDefault(); + } + + row[key] = field.transformRetrieve(value as any); + } catch (e: any) { + throw new TransformRetrieveFailedException( + `"${field.type}" field "${key}" on entity "${entity.name}": ${e.message}` + ); + } + } + + data.push(row); + } + + return data; + } + + toJSON() { + return { + entities: Object.fromEntries(this.entities.map((e) => [e.name, e.toJSON()])), + relations: Object.fromEntries(this.relations.all.map((r) => [r.getName(), r.toJSON()])), + //relations: this.relations.all.map((r) => r.toJSON()), + indices: Object.fromEntries(this.indices.map((i) => [i.name, i.toJSON()])) + }; + } +} diff --git a/app/src/data/entities/Mutator.ts b/app/src/data/entities/Mutator.ts new file mode 100644 index 0000000..aef2bf1 --- /dev/null +++ b/app/src/data/entities/Mutator.ts @@ -0,0 +1,270 @@ +import type { PrimaryFieldType } from "core"; +import { type EmitsEvents, EventManager } from "core/events"; +import type { DeleteQueryBuilder, InsertQueryBuilder, UpdateQueryBuilder } from "kysely"; +import { type TActionContext, WhereBuilder } from ".."; +import type { Entity, EntityData, EntityManager } from "../entities"; +import { InvalidSearchParamsException } from "../errors"; +import { MutatorEvents } from "../events"; +import { RelationMutator } from "../relations"; +import type { RepoQuery } from "../server/data-query-impl"; + +type MutatorQB = + | InsertQueryBuilder + | UpdateQueryBuilder + | DeleteQueryBuilder; + +type MutatorUpdateOrDelete = + | UpdateQueryBuilder + | DeleteQueryBuilder; + +export type MutatorResponse = { + entity: Entity; + sql: string; + parameters: any[]; + result: EntityData[]; + data: T; +}; + +export class Mutator implements EmitsEvents { + em: EntityManager; + entity: Entity; + static readonly Events = MutatorEvents; + emgr: EventManager; + + // @todo: current hacky workaround to disable creation of system entities + __unstable_disable_system_entity_creation = true; + __unstable_toggleSystemEntityCreation(value: boolean) { + this.__unstable_disable_system_entity_creation = value; + } + + constructor(em: EntityManager, entity: Entity, emgr?: EventManager) { + this.em = em; + this.entity = entity; + this.emgr = emgr ?? new EventManager(MutatorEvents); + } + + private get conn() { + return this.em.connection.kysely; + } + + async getValidatedData(data: EntityData, context: TActionContext): Promise { + const entity = this.entity; + if (!context) { + throw new Error("Context must be provided for validation"); + } + + const keys = Object.keys(data); + const validatedData: EntityData = {}; + + // get relational references/keys + const relationMutator = new RelationMutator(entity, this.em); + const relational_keys = relationMutator.getRelationalKeys(); + + for (const key of keys) { + if (relational_keys.includes(key)) { + const result = await relationMutator.persistRelation(key, data[key]); + + // if relation field (include key and value in validatedData) + if (Array.isArray(result)) { + //console.log("--- (instructions)", result); + const [relation_key, relation_value] = result; + validatedData[relation_key] = relation_value; + } + continue; + } + + const field = entity.getField(key); + if (!field) { + throw new Error( + `Field "${key}" not found on entity "${entity.name}". Fields: ${entity + .getFillableFields() + .map((f) => f.name) + .join(", ")}` + ); + } + + // we should never get here, but just to be sure (why?) + if (!field.isFillable(context)) { + throw new Error(`Field "${key}" is not fillable on entity "${entity.name}"`); + } + + validatedData[key] = await field.transformPersist(data[key], this.em, context); + } + + if (Object.keys(validatedData).length === 0) { + throw new Error(`No data left to update "${entity.name}"`); + } + + return validatedData; + } + + protected async many(qb: MutatorQB): Promise { + const entity = this.entity; + const { sql, parameters } = qb.compile(); + //console.log("mutatoar:exec", sql, parameters); + const result = await qb.execute(); + + const data = this.em.hydrate(entity.name, result) as EntityData[]; + + return { + entity, + sql, + parameters: [...parameters], + result: result, + data + }; + } + + protected async single(qb: MutatorQB): Promise> { + const { data, ...response } = await this.many(qb); + return { ...response, data: data[0]! }; + } + + async insertOne(data: EntityData): Promise> { + const entity = this.entity; + if (entity.type === "system" && this.__unstable_disable_system_entity_creation) { + throw new Error(`Creation of system entity "${entity.name}" is disabled`); + } + + // @todo: establish the original order from "data" + const validatedData = { + ...entity.getDefaultObject(), + ...(await this.getValidatedData(data, "create")) + }; + + await this.emgr.emit(new Mutator.Events.MutatorInsertBefore({ entity, data: validatedData })); + + // check if required fields are present + const required = entity.getRequiredFields(); + for (const field of required) { + if ( + typeof validatedData[field.name] === "undefined" || + validatedData[field.name] === null + ) { + throw new Error(`Field "${field.name}" is required`); + } + } + + const query = this.conn + .insertInto(entity.name) + .values(validatedData) + .returning(entity.getSelect()); + + const res = await this.single(query); + + await this.emgr.emit(new Mutator.Events.MutatorInsertAfter({ entity, data: res.data })); + + return res; + } + + async updateOne(id: PrimaryFieldType, data: EntityData): Promise> { + const entity = this.entity; + if (!Number.isInteger(id)) { + throw new Error("ID must be provided for update"); + } + + const validatedData = await this.getValidatedData(data, "update"); + + await this.emgr.emit( + new Mutator.Events.MutatorUpdateBefore({ entity, entityId: id, data: validatedData }) + ); + + const query = this.conn + .updateTable(entity.name) + .set(validatedData) + .where(entity.id().name, "=", id) + .returning(entity.getSelect()); + + const res = await this.single(query); + + await this.emgr.emit( + new Mutator.Events.MutatorUpdateAfter({ entity, entityId: id, data: res.data }) + ); + + return res; + } + + async deleteOne(id: PrimaryFieldType): Promise> { + const entity = this.entity; + if (!Number.isInteger(id)) { + throw new Error("ID must be provided for deletion"); + } + + await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id })); + + const query = this.conn + .deleteFrom(entity.name) + .where(entity.id().name, "=", id) + .returning(entity.getSelect()); + + const res = await this.single(query); + + await this.emgr.emit( + new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data }) + ); + + return res; + } + + private getValidOptions(options?: Partial): Partial { + const entity = this.entity; + const validated: Partial = {}; + + if (options?.where) { + // @todo: add tests for aliased fields in where + const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => { + return typeof entity.getField(field) === "undefined"; + }); + + if (invalid.length > 0) { + throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`); + } + + validated.where = options.where; + } + + return validated; + } + + private appendWhere(qb: QB, _where?: RepoQuery["where"]): QB { + const entity = this.entity; + + const alias = entity.name; + const aliased = (field: string) => `${alias}.${field}`; + + // add where if present + if (_where) { + // @todo: add tests for aliased fields in where + const invalid = WhereBuilder.getPropertyNames(_where).filter((field) => { + return typeof entity.getField(field) === "undefined"; + }); + + if (invalid.length > 0) { + throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`); + } + + return WhereBuilder.addClause(qb, _where); + } + + return qb; + } + + // @todo: decide whether entries should be deleted all at once or one by one (for events) + async deleteMany(where?: RepoQuery["where"]): Promise> { + const entity = this.entity; + + const qb = this.appendWhere(this.conn.deleteFrom(entity.name), where).returning( + entity.getSelect() + ); + + //await this.emgr.emit(new Mutator.Events.MutatorDeleteBefore({ entity, entityId: id })); + + const res = await this.many(qb); + + /*await this.emgr.emit( + new Mutator.Events.MutatorDeleteAfter({ entity, entityId: id, data: res.data }) + );*/ + + return res; + } +} diff --git a/app/src/data/entities/index.ts b/app/src/data/entities/index.ts new file mode 100644 index 0000000..efe6446 --- /dev/null +++ b/app/src/data/entities/index.ts @@ -0,0 +1,6 @@ +export * from "./Entity"; +export * from "./EntityManager"; +export * from "./Mutator"; +export * from "./query/Repository"; +export * from "./query/WhereBuilder"; +export * from "./query/WithBuilder"; diff --git a/app/src/data/entities/query/JoinBuilder.ts b/app/src/data/entities/query/JoinBuilder.ts new file mode 100644 index 0000000..0aa1ca7 --- /dev/null +++ b/app/src/data/entities/query/JoinBuilder.ts @@ -0,0 +1,51 @@ +import { ManyToManyRelation, ManyToOneRelation } from "../../relations"; +import type { Entity } from "../Entity"; +import type { EntityManager } from "../EntityManager"; +import type { RepositoryQB } from "./Repository"; + +export class JoinBuilder { + private static buildClause( + em: EntityManager, + qb: RepositoryQB, + entity: Entity, + withString: string, + ) { + const relation = em.relationOf(entity.name, withString); + if (!relation) { + throw new Error(`Relation "${withString}" not found`); + } + + return relation.buildJoin(entity, qb, withString); + } + + // @todo: returns multiple on manytomany (edit: so?) + static getJoinedEntityNames(em: EntityManager, entity: Entity, joins: string[]): string[] { + return joins.flatMap((join) => { + const relation = em.relationOf(entity.name, join); + if (!relation) { + throw new Error(`Relation "${join}" not found`); + } + + const other = relation.other(entity); + + if (relation instanceof ManyToOneRelation) { + return [other.entity.name]; + } else if (relation instanceof ManyToManyRelation) { + return [other.entity.name, relation.connectionEntity.name]; + } + + return []; + }); + } + + static addClause(em: EntityManager, qb: RepositoryQB, entity: Entity, joins: string[]) { + if (joins.length === 0) return qb; + + let newQb = qb; + for (const entry of joins) { + newQb = JoinBuilder.buildClause(em, newQb, entity, entry); + } + + return newQb; + } +} diff --git a/app/src/data/entities/query/Repository.ts b/app/src/data/entities/query/Repository.ts new file mode 100644 index 0000000..f296adf --- /dev/null +++ b/app/src/data/entities/query/Repository.ts @@ -0,0 +1,407 @@ +import type { PrimaryFieldType } from "core"; +import { type EmitsEvents, EventManager } from "core/events"; +import { type SelectQueryBuilder, sql } from "kysely"; +import { cloneDeep } from "lodash-es"; +import { InvalidSearchParamsException } from "../../errors"; +import { MutatorEvents, RepositoryEvents, RepositoryFindManyBefore } from "../../events"; +import { type RepoQuery, defaultQuerySchema } from "../../server/data-query-impl"; +import { + type Entity, + type EntityData, + type EntityManager, + WhereBuilder, + WithBuilder +} from "../index"; +import { JoinBuilder } from "./JoinBuilder"; + +export type RepositoryQB = SelectQueryBuilder; + +export type RepositoryRawResponse = { + sql: string; + parameters: any[]; + result: EntityData[]; +}; +export type RepositoryResponse = RepositoryRawResponse & { + entity: Entity; + data: T; + meta: { + total: number; + count: number; + items: number; + time?: number; + query?: { + sql: string; + parameters: readonly any[]; + }; + }; +}; + +export type RepositoryCountResponse = RepositoryRawResponse & { + count: number; +}; +export type RepositoryExistsResponse = RepositoryRawResponse & { + exists: boolean; +}; + +export class Repository implements EmitsEvents { + em: EntityManager; + entity: Entity; + static readonly Events = RepositoryEvents; + emgr: EventManager; + + constructor(em: EntityManager, entity: Entity, emgr?: EventManager) { + this.em = em; + this.entity = entity; + this.emgr = emgr ?? new EventManager(MutatorEvents); + } + + private cloneFor(entity: Entity) { + return new Repository(this.em, entity, this.emgr); + } + + private get conn() { + return this.em.connection.kysely; + } + + private getValidOptions(options?: Partial): RepoQuery { + const entity = this.entity; + // @todo: if not cloned deep, it will keep references and error if multiple requests come in + const validated = { + ...cloneDeep(defaultQuerySchema), + sort: entity.getDefaultSort(), + select: entity.getSelect() + }; + //console.log("validated", validated); + + if (!options) return validated; + + if (options.sort) { + if (!validated.select.includes(options.sort.by)) { + throw new InvalidSearchParamsException(`Invalid sort field "${options.sort.by}"`); + } + if (!["asc", "desc"].includes(options.sort.dir)) { + throw new InvalidSearchParamsException(`Invalid sort direction "${options.sort.dir}"`); + } + + validated.sort = options.sort; + } + + if (options.select && options.select.length > 0) { + const invalid = options.select.filter((field) => !validated.select.includes(field)); + + if (invalid.length > 0) { + throw new InvalidSearchParamsException( + `Invalid select field(s): ${invalid.join(", ")}` + ); + } + + validated.select = options.select; + } + + if (options.with && options.with.length > 0) { + for (const entry of options.with) { + const related = this.em.relationOf(entity.name, entry); + if (!related) { + throw new InvalidSearchParamsException( + `WITH: "${entry}" is not a relation of "${entity.name}"` + ); + } + + validated.with.push(entry); + } + } + + if (options.join && options.join.length > 0) { + for (const entry of options.join) { + const related = this.em.relationOf(entity.name, entry); + if (!related) { + throw new InvalidSearchParamsException( + `JOIN: "${entry}" is not a relation of "${entity.name}"` + ); + } + + validated.join.push(entry); + } + } + + if (options.where) { + // @todo: auto-alias base entity when using joins! otherwise "id" is ambiguous + const aliases = [entity.name]; + if (validated.join.length > 0) { + aliases.push(...JoinBuilder.getJoinedEntityNames(this.em, entity, validated.join)); + } + + // @todo: add tests for aliased fields in where + const invalid = WhereBuilder.getPropertyNames(options.where).filter((field) => { + if (field.includes(".")) { + const [alias, prop] = field.split(".") as [string, string]; + if (!aliases.includes(alias)) { + return true; + } + + return !this.em.entity(alias).getField(prop); + } + + return typeof entity.getField(field) === "undefined"; + }); + + if (invalid.length > 0) { + throw new InvalidSearchParamsException(`Invalid where field(s): ${invalid.join(", ")}`); + } + + validated.where = options.where; + } + + // pass unfiltered + if (options.limit) validated.limit = options.limit; + if (options.offset) validated.offset = options.offset; + + return validated; + } + + protected async performQuery(qb: RepositoryQB): Promise { + const entity = this.entity; + const compiled = qb.compile(); + /*const { sql, parameters } = qb.compile(); + console.log("many", sql, parameters);*/ + + const start = performance.now(); + const selector = (as = "count") => this.conn.fn.countAll().as(as); + const countQuery = qb + .clearSelect() + .select(selector()) + .clearLimit() + .clearOffset() + .clearGroupBy() + .clearOrderBy(); + const totalQuery = this.conn.selectFrom(entity.name).select(selector("total")); + + try { + const [_count, _total, result] = await this.em.connection.batchQuery([ + countQuery, + totalQuery, + qb + ]); + //console.log("result", { _count, _total }); + + const time = Number.parseFloat((performance.now() - start).toFixed(2)); + const data = this.em.hydrate(entity.name, result); + + return { + entity, + sql: compiled.sql, + parameters: [...compiled.parameters], + result, + data, + meta: { + total: _total[0]?.total ?? 0, + count: _count[0]?.count ?? 0, // @todo: better graceful method + items: result.length, + time, + query: { sql: compiled.sql, parameters: compiled.parameters } + } + }; + } catch (e) { + console.error("many error", e, compiled); + throw e; + } + } + + protected async single( + qb: RepositoryQB, + options: RepoQuery + ): Promise> { + await this.emgr.emit( + new Repository.Events.RepositoryFindOneBefore({ entity: this.entity, options }) + ); + + const { data, ...response } = await this.performQuery(qb); + + await this.emgr.emit( + new Repository.Events.RepositoryFindOneAfter({ + entity: this.entity, + options, + data: data[0]! + }) + ); + + return { ...response, data: data[0]! }; + } + + private buildQuery( + _options?: Partial, + exclude_options: (keyof RepoQuery)[] = [] + ): { qb: RepositoryQB; options: RepoQuery } { + const entity = this.entity; + const options = this.getValidOptions(_options); + + const alias = entity.name; + const aliased = (field: string) => `${alias}.${field}`; + let qb = this.conn + .selectFrom(entity.name) + .select(entity.getAliasedSelectFrom(options.select, alias)); + + //console.log("build query options", options); + if (!exclude_options.includes("with") && options.with) { + qb = WithBuilder.addClause(this.em, qb, entity, options.with); + } + + if (!exclude_options.includes("join") && options.join) { + qb = JoinBuilder.addClause(this.em, qb, entity, options.join); + } + + // add where if present + if (!exclude_options.includes("where") && options.where) { + qb = WhereBuilder.addClause(qb, options.where); + } + + if (!exclude_options.includes("limit")) qb = qb.limit(options.limit); + if (!exclude_options.includes("offset")) qb = qb.offset(options.offset); + + // sorting + if (!exclude_options.includes("sort")) { + qb = qb.orderBy(aliased(options.sort.by), options.sort.dir); + } + + return { qb, options }; + } + + async findId( + id: PrimaryFieldType, + _options?: Partial> + ): Promise> { + const { qb, options } = this.buildQuery( + { + ..._options, + where: { [this.entity.getPrimaryField().name]: id }, + limit: 1 + }, + ["offset", "sort"] + ); + + return this.single(qb, options) as any; + } + + async findOne( + where: RepoQuery["where"], + _options?: Partial> + ): Promise> { + const { qb, options } = this.buildQuery( + { + ..._options, + where, + limit: 1 + }, + ["offset", "sort"] + ); + + return this.single(qb, options) as any; + } + + async findMany(_options?: Partial): Promise> { + const { qb, options } = this.buildQuery(_options); + //console.log("findMany:options", options); + + await this.emgr.emit( + new Repository.Events.RepositoryFindManyBefore({ entity: this.entity, options }) + ); + + const res = await this.performQuery(qb); + + await this.emgr.emit( + new Repository.Events.RepositoryFindManyAfter({ + entity: this.entity, + options, + data: res.data + }) + ); + + return res as any; + } + + // @todo: add unit tests, specially for many to many + async findManyByReference( + id: PrimaryFieldType, + reference: string, + _options?: Partial> + ): Promise> { + const entity = this.entity; + const listable_relations = this.em.relations.listableRelationsOf(entity); + const relation = listable_relations.find((r) => r.ref(reference).reference === reference); + + if (!relation) { + throw new Error( + `Relation "${reference}" not found or not listable on entity "${entity.name}"` + ); + } + + const newEntity = relation.other(entity).entity; + const refQueryOptions = relation.getReferenceQuery(newEntity, id as number, reference); + if (!("where" in refQueryOptions) || Object.keys(refQueryOptions.where as any).length === 0) { + throw new Error( + `Invalid reference query for "${reference}" on entity "${newEntity.name}"` + ); + } + + const findManyOptions = { + ..._options, + ...refQueryOptions, + where: { + ...refQueryOptions.where, + ..._options?.where + } + }; + + //console.log("findManyOptions", newEntity.name, findManyOptions); + return this.cloneFor(newEntity).findMany(findManyOptions); + } + + async count(where?: RepoQuery["where"]): Promise { + const entity = this.entity; + const options = this.getValidOptions({ where }); + + const selector = this.conn.fn.count(sql`*`).as("count"); + let qb = this.conn.selectFrom(entity.name).select(selector); + + // add where if present + if (options.where) { + qb = WhereBuilder.addClause(qb, options.where); + } + + const compiled = qb.compile(); + const result = await qb.execute(); + + return { + sql: compiled.sql, + parameters: [...compiled.parameters], + result, + count: result[0]?.count ?? 0 + }; + } + + async exists(where: Required): Promise { + const entity = this.entity; + const options = this.getValidOptions({ where }); + + const selector = this.conn.fn.count(sql`*`).as("count"); + let qb = this.conn.selectFrom(entity.name).select(selector); + + // add mandatory where + qb = WhereBuilder.addClause(qb, options.where); + + // we only need 1 + qb = qb.limit(1); + + const compiled = qb.compile(); + //console.log("exists query", compiled.sql, compiled.parameters); + const result = await qb.execute(); + //console.log("result", result); + + return { + sql: compiled.sql, + parameters: [...compiled.parameters], + result, + exists: result[0]!.count > 0 + }; + } +} diff --git a/app/src/data/entities/query/WhereBuilder.ts b/app/src/data/entities/query/WhereBuilder.ts new file mode 100644 index 0000000..455ecf4 --- /dev/null +++ b/app/src/data/entities/query/WhereBuilder.ts @@ -0,0 +1,132 @@ +import { + type BooleanLike, + type FilterQuery, + type Primitive, + type TExpression, + exp, + isBooleanLike, + isPrimitive, + makeValidator +} from "core"; +import type { + DeleteQueryBuilder, + ExpressionBuilder, + ExpressionWrapper, + SelectQueryBuilder, + UpdateQueryBuilder +} from "kysely"; +import type { RepositoryQB } from "./Repository"; + +type Builder = ExpressionBuilder; +type Wrapper = ExpressionWrapper; +type WhereQb = + | SelectQueryBuilder + | UpdateQueryBuilder + | DeleteQueryBuilder; + +function key(e: unknown): string { + if (typeof e !== "string") { + throw new Error(`Invalid key: ${e}`); + } + return e as string; +} + +const expressions: TExpression[] = [ + exp( + "$eq", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), "=", v) + ), + exp( + "$ne", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), "!=", v) + ), + exp( + "$gt", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), ">", v) + ), + exp( + "$gte", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), ">=", v) + ), + exp( + "$lt", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), "<", v) + ), + exp( + "$lte", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), "<=", v) + ), + exp( + "$isnull", + (v: BooleanLike) => isBooleanLike(v), + (v, k, eb: Builder) => eb(key(k), v ? "is" : "is not", null) + ), + exp( + "$in", + (v: any[]) => Array.isArray(v), + (v, k, eb: Builder) => eb(key(k), "in", v) + ), + exp( + "$notin", + (v: any[]) => Array.isArray(v), + (v, k, eb: Builder) => eb(key(k), "not in", v) + ), + exp( + "$between", + (v: [number, number]) => Array.isArray(v) && v.length === 2, + (v, k, eb: Builder) => eb.between(key(k), v[0], v[1]) + ), + exp( + "$like", + (v: Primitive) => isPrimitive(v), + (v, k, eb: Builder) => eb(key(k), "like", String(v).replace(/\*/g, "%")) + ) +]; + +export type WhereQuery = FilterQuery; + +const validator = makeValidator(expressions); + +export class WhereBuilder { + static addClause(qb: QB, query: WhereQuery) { + if (Object.keys(query).length === 0) { + return qb; + } + + // @ts-ignore + return qb.where((eb) => { + const fns = validator.build(query, { + value_is_kv: true, + exp_ctx: eb, + convert: true + }); + + if (fns.$or.length > 0 && fns.$and.length > 0) { + return eb.and(fns.$and).or(eb.and(fns.$or)); + } else if (fns.$or.length > 0) { + return eb.or(fns.$or); + } + + return eb.and(fns.$and); + }); + } + + static convert(query: WhereQuery): WhereQuery { + return validator.convert(query); + } + + static getPropertyNames(query: WhereQuery): string[] { + const { keys } = validator.build(query, { + value_is_kv: true, + exp_ctx: () => null, + convert: true + }); + return Array.from(keys); + } +} diff --git a/app/src/data/entities/query/WithBuilder.ts b/app/src/data/entities/query/WithBuilder.ts new file mode 100644 index 0000000..260dc86 --- /dev/null +++ b/app/src/data/entities/query/WithBuilder.ts @@ -0,0 +1,42 @@ +import type { Entity, EntityManager, RepositoryQB } from "../../entities"; + +export class WithBuilder { + private static buildClause( + em: EntityManager, + qb: RepositoryQB, + entity: Entity, + withString: string + ) { + const relation = em.relationOf(entity.name, withString); + if (!relation) { + throw new Error(`Relation "${withString}" not found`); + } + + const cardinality = relation.ref(withString).cardinality; + //console.log("with--builder", { entity: entity.name, withString, cardinality }); + + const fns = em.connection.fn; + const jsonFrom = cardinality === 1 ? fns.jsonObjectFrom : fns.jsonArrayFrom; + + if (!jsonFrom) { + throw new Error("Connection does not support jsonObjectFrom/jsonArrayFrom"); + } + + try { + return relation.buildWith(entity, qb, jsonFrom, withString); + } catch (e) { + throw new Error(`Could not build "with" relation "${withString}": ${(e as any).message}`); + } + } + + static addClause(em: EntityManager, qb: RepositoryQB, entity: Entity, withs: string[]) { + if (withs.length === 0) return qb; + + let newQb = qb; + for (const entry of withs) { + newQb = WithBuilder.buildClause(em, newQb, entity, entry); + } + + return newQb; + } +} diff --git a/app/src/data/errors.ts b/app/src/data/errors.ts new file mode 100644 index 0000000..f4f5810 --- /dev/null +++ b/app/src/data/errors.ts @@ -0,0 +1,77 @@ +import { Exception } from "core"; +import type { TypeInvalidError } from "core/utils"; +import type { Entity } from "./entities"; +import type { Field } from "./fields"; + +export class UnableToConnectException extends Exception { + override name = "UnableToConnectException"; + override code = 500; +} + +export class InvalidSearchParamsException extends Exception { + override name = "InvalidSearchParamsException"; + override code = 422; +} + +export class TransformRetrieveFailedException extends Exception { + override name = "TransformRetrieveFailedException"; + override code = 422; +} + +export class TransformPersistFailedException extends Exception { + override name = "TransformPersistFailedException"; + override code = 422; + + static invalidType(property: string, expected: string, given: any) { + const givenValue = typeof given === "object" ? JSON.stringify(given) : given; + const message = + `Property "${property}" must be of type "${expected}", ` + + `"${givenValue}" of type "${typeof given}" given.`; + return new TransformPersistFailedException(message); + } + + static required(property: string) { + return new TransformPersistFailedException(`Property "${property}" is required`); + } +} + +export class InvalidFieldConfigException extends Exception { + override name = "InvalidFieldConfigException"; + override code = 400; + + constructor( + field: Field, + public given: any, + error: TypeInvalidError + ) { + console.error("InvalidFieldConfigException", { + given, + error: error.firstToString() + }); + super(`Invalid Field config given for field "${field.name}": ${error.firstToString()}`); + } +} + +export class EntityNotDefinedException extends Exception { + override name = "EntityNotDefinedException"; + override code = 400; + + constructor(entity?: Entity | string) { + if (!entity) { + super("Cannot find an entity that is undefined"); + } else { + super(`Entity "${typeof entity !== "string" ? entity.name : entity}" not defined`); + } + } +} + +export class EntityNotFoundException extends Exception { + override name = "EntityNotFoundException"; + override code = 404; + + constructor(entity: Entity | string, id: any) { + super( + `Entity "${typeof entity !== "string" ? entity.name : entity}" with id "${id}" not found` + ); + } +} diff --git a/app/src/data/events/index.ts b/app/src/data/events/index.ts new file mode 100644 index 0000000..01311d8 --- /dev/null +++ b/app/src/data/events/index.ts @@ -0,0 +1,74 @@ +import type { PrimaryFieldType } from "core"; +import { Event } from "core/events"; +import type { Entity, EntityData } from "../entities"; +import type { RepoQuery } from "../server/data-query-impl"; + +export class MutatorInsertBefore extends Event<{ entity: Entity; data: EntityData }> { + static override slug = "mutator-insert-before"; +} +export class MutatorInsertAfter extends Event<{ entity: Entity; data: EntityData }> { + static override slug = "mutator-insert-after"; +} +export class MutatorUpdateBefore extends Event<{ + entity: Entity; + entityId: PrimaryFieldType; + data: EntityData; +}> { + static override slug = "mutator-update-before"; +} +export class MutatorUpdateAfter extends Event<{ + entity: Entity; + entityId: PrimaryFieldType; + data: EntityData; +}> { + static override slug = "mutator-update-after"; +} +export class MutatorDeleteBefore extends Event<{ entity: Entity; entityId: PrimaryFieldType }> { + static override slug = "mutator-delete-before"; +} +export class MutatorDeleteAfter extends Event<{ + entity: Entity; + entityId: PrimaryFieldType; + data: EntityData; +}> { + static override slug = "mutator-delete-after"; +} + +export const MutatorEvents = { + MutatorInsertBefore, + MutatorInsertAfter, + MutatorUpdateBefore, + MutatorUpdateAfter, + MutatorDeleteBefore, + MutatorDeleteAfter +}; + +export class RepositoryFindOneBefore extends Event<{ entity: Entity; options: RepoQuery }> { + static override slug = "repository-find-one-before"; +} +export class RepositoryFindOneAfter extends Event<{ + entity: Entity; + options: RepoQuery; + data: EntityData; +}> { + static override slug = "repository-find-one-after"; +} + +export class RepositoryFindManyBefore extends Event<{ entity: Entity; options: RepoQuery }> { + static override slug = "repository-find-many-before"; + static another = "one"; +} +export class RepositoryFindManyAfter extends Event<{ + entity: Entity; + options: RepoQuery; + data: EntityData; +}> { + static override slug = "repository-find-many-after"; +} + +export const RepositoryEvents = { + RepositoryFindOneBefore, + RepositoryFindOneAfter, + RepositoryFindManyBefore, + RepositoryFindManyAfter +}; diff --git a/app/src/data/fields/BooleanField.ts b/app/src/data/fields/BooleanField.ts new file mode 100644 index 0000000..bbae784 --- /dev/null +++ b/app/src/data/fields/BooleanField.ts @@ -0,0 +1,88 @@ +import { type Static, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const booleanFieldConfigSchema = Type.Composite([ + Type.Object({ + default_value: Type.Optional(Type.Boolean({ default: false })) + }), + baseFieldConfigSchema +]); + +export type BooleanFieldConfig = Static; + +export class BooleanField extends Field< + BooleanFieldConfig, + boolean, + Required +> { + override readonly type = "boolean"; + + protected getSchema() { + return booleanFieldConfigSchema; + } + + override getValue(value: unknown, context: TRenderContext) { + switch (context) { + case "table": + return value ? "Yes" : "No"; + default: + return value; + } + } + + schema() { + // @todo: potentially use "integer" instead + return this.useSchemaHelper("boolean"); + } + + override getHtmlConfig() { + return { + ...super.getHtmlConfig(), + element: "boolean" + }; + } + + override transformRetrieve(value: unknown): boolean | null { + //console.log("Boolean:transformRetrieve:value", value); + if (typeof value === "undefined" || value === null) { + if (this.isRequired()) return false; + if (this.hasDefault()) return this.getDefault(); + + return null; + } + + if (typeof value === "string") { + return value === "1"; + } + + // cast to boolean, as it might be stored as number + return !!value; + } + + override async transformPersist( + val: unknown, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(val, em, context); + if (this.nullish(value)) { + return this.isRequired() ? Boolean(this.config.default_value) : undefined; + } + + if (typeof value === "number") { + return value !== 0; + } + + if (typeof value !== "boolean") { + throw TransformPersistFailedException.invalidType(this.name, "boolean", value); + } + + return value as boolean; + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired(Type.Boolean({ default: this.getDefault() })); + } +} diff --git a/app/src/data/fields/DateField.ts b/app/src/data/fields/DateField.ts new file mode 100644 index 0000000..000e64e --- /dev/null +++ b/app/src/data/fields/DateField.ts @@ -0,0 +1,151 @@ +import { type Static, StringEnum, Type, dayjs } from "core/utils"; +import type { EntityManager } from "../entities"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const dateFieldConfigSchema = Type.Composite( + [ + Type.Object({ + //default_value: Type.Optional(Type.Date()), + type: StringEnum(["date", "datetime", "week"] as const, { default: "date" }), + timezone: Type.Optional(Type.String()), + min_date: Type.Optional(Type.String()), + max_date: Type.Optional(Type.String()) + }), + baseFieldConfigSchema + ], + { + additionalProperties: false + } +); + +export type DateFieldConfig = Static; + +export class DateField extends Field< + DateFieldConfig, + Date, + Required +> { + override readonly type = "date"; + + protected getSchema() { + return dateFieldConfigSchema; + } + + override schema() { + const type = this.config.type === "datetime" ? "datetime" : "date"; + return this.useSchemaHelper(type); + } + + override getHtmlConfig() { + const htmlType = this.config.type === "datetime" ? "datetime-local" : this.config.type; + + return { + ...super.getHtmlConfig(), + element: "date", + props: { + type: htmlType + } + }; + } + + private parseDateFromString(value: string): Date { + //console.log("parseDateFromString", value); + if (this.config.type === "week" && value.includes("-W")) { + const [year, week] = value.split("-W").map((n) => Number.parseInt(n, 10)) as [ + number, + number + ]; + //console.log({ year, week }); + // @ts-ignore causes errors on build? + return dayjs().year(year).week(week).toDate(); + } + + return new Date(value); + } + + override getValue(value: string, context?: TRenderContext): string | undefined { + if (value === null || !value) return; + //console.log("getValue", { value, context }); + const date = this.parseDateFromString(value); + //console.log("getValue.date", date); + + if (context === "submit") { + try { + return date.toISOString(); + } catch (e) { + //console.warn("DateField.getValue:value/submit", value, e); + return undefined; + } + } + + if (this.config.type === "week") { + try { + return `${date.getFullYear()}-W${dayjs(date).week()}`; + } catch (e) { + console.warn("error - DateField.getValue:week", value, e); + return; + } + } + + try { + const utc = new Date(); + const offset = utc.getTimezoneOffset(); + //console.log("offset", offset); + const local = new Date(date.getTime() - offset * 60000); + + return this.formatDate(local); + } catch (e) { + console.warn("DateField.getValue:value", value); + console.warn("DateField.getValue:e", e); + return; + } + } + + formatDate(_date: Date): string { + switch (this.config.type) { + case "datetime": + return _date.toISOString().split(".")[0]!.replace("T", " "); + default: + return _date.toISOString().split("T")[0]!; + /*case "week": { + const date = dayjs(_date); + return `${date.year()}-W${date.week()}`; + }*/ + } + } + + override transformRetrieve(_value: string): Date | null { + //console.log("transformRetrieve DateField", _value); + const value = super.transformRetrieve(_value); + if (value === null) return null; + + try { + return new Date(value); + } catch (e) { + return null; + } + } + + override async transformPersist( + _value: any, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(_value, em, context); + if (this.nullish(value)) return value; + + //console.log("transformPersist DateField", value); + switch (this.config.type) { + case "date": + case "week": + return new Date(value).toISOString().split("T")[0]!; + default: + return new Date(value).toISOString(); + } + } + + // @todo: check this + override toJsonSchema() { + return this.toSchemaWrapIfRequired(Type.String({ default: this.getDefault() })); + } +} diff --git a/app/src/data/fields/EnumField.ts b/app/src/data/fields/EnumField.ts new file mode 100644 index 0000000..8c44afb --- /dev/null +++ b/app/src/data/fields/EnumField.ts @@ -0,0 +1,153 @@ +import { Const, type Static, StringEnum, StringRecord, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const enumFieldConfigSchema = Type.Composite( + [ + Type.Object({ + default_value: Type.Optional(Type.String()), + options: Type.Optional( + Type.Union([ + Type.Object( + { + type: Const("strings"), + values: Type.Array(Type.String()) + }, + { title: "Strings" } + ), + Type.Object( + { + type: Const("objects"), + values: Type.Array( + Type.Object({ + label: Type.String(), + value: Type.String() + }) + ) + }, + { + title: "Objects", + additionalProperties: false + } + ) + ]) + ) + }), + baseFieldConfigSchema + ], + { + additionalProperties: false + } +); + +export type EnumFieldConfig = Static; + +export class EnumField extends Field< + EnumFieldConfig, + TypeOverride, + Required +> { + override readonly type = "enum"; + + constructor(name: string, config: Partial) { + super(name, config); + + /*if (this.config.options.values.length === 0) { + throw new Error(`Enum field "${this.name}" requires at least one option`); + }*/ + + if (this.config.default_value && !this.isValidValue(this.config.default_value)) { + throw new Error(`Default value "${this.config.default_value}" is not a valid option`); + } + } + + protected getSchema() { + return enumFieldConfigSchema; + } + + override schema() { + return this.useSchemaHelper("text"); + } + + getOptions(): { label: string; value: string }[] { + const options = this.config?.options ?? { type: "strings", values: [] }; + + /*if (options.values?.length === 0) { + throw new Error(`Enum field "${this.name}" requires at least one option`); + }*/ + + if (options.type === "strings") { + return options.values?.map((option) => ({ label: option, value: option })); + } + + return options?.values; + } + + isValidValue(value: string): boolean { + const valid_values = this.getOptions().map((option) => option.value); + return valid_values.includes(value); + } + + override getValue(value: any, context: TRenderContext) { + if (!this.isValidValue(value)) { + return this.hasDefault() ? this.getDefault() : null; + } + + switch (context) { + case "table": + return this.getOptions().find((option) => option.value === value)?.label ?? value; + } + + return value; + } + + /** + * Transform value after retrieving from database + * @param value + */ + override transformRetrieve(value: string | null): string | null { + const val = super.transformRetrieve(value); + + if (val === null && this.hasDefault()) { + return this.getDefault(); + } + + if (!this.isValidValue(val)) { + return this.hasDefault() ? this.getDefault() : null; + } + + return val; + } + + override async transformPersist( + _value: any, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(_value, em, context); + if (this.nullish(value)) return value; + + if (!this.isValidValue(value)) { + throw new TransformPersistFailedException( + `Field "${this.name}" must be one of the following values: ${this.getOptions() + .map((o) => o.value) + .join(", ")}` + ); + } + + return value; + } + + override toJsonSchema() { + const options = this.config?.options ?? { type: "strings", values: [] }; + const values = + options.values?.map((option) => (typeof option === "string" ? option : option.value)) ?? + []; + return this.toSchemaWrapIfRequired( + StringEnum(values, { + default: this.getDefault() + }) + ); + } +} diff --git a/app/src/data/fields/Field.ts b/app/src/data/fields/Field.ts new file mode 100644 index 0000000..b5e332d --- /dev/null +++ b/app/src/data/fields/Field.ts @@ -0,0 +1,244 @@ +import { + type Static, + StringEnum, + type TSchema, + Type, + TypeInvalidError, + parse, + snakeToPascalWithSpaces +} from "core/utils"; +import type { ColumnBuilderCallback, ColumnDataType, ColumnDefinitionBuilder } from "kysely"; +import type { HTMLInputTypeAttribute, InputHTMLAttributes } from "react"; +import type { EntityManager } from "../entities"; +import { InvalidFieldConfigException, TransformPersistFailedException } from "../errors"; + +export const ActionContext = ["create", "read", "update", "delete"] as const; +export type TActionContext = (typeof ActionContext)[number]; + +export const RenderContext = ["form", "table", "read", "submit"] as const; +export type TRenderContext = (typeof RenderContext)[number]; + +const TmpContext = ["create", "read", "update", "delete", "form", "table", "submit"] as const; +export type TmpActionAndRenderContext = (typeof TmpContext)[number]; + +const DEFAULT_REQUIRED = false; +const DEFAULT_FILLABLE = true; +const DEFAULT_HIDDEN = false; + +// @todo: add refine functions (e.g. if required, but not fillable, needs default value) +export const baseFieldConfigSchema = Type.Object( + { + label: Type.Optional(Type.String()), + description: Type.Optional(Type.String()), + required: Type.Optional(Type.Boolean({ default: DEFAULT_REQUIRED })), + fillable: Type.Optional( + Type.Union( + [ + Type.Boolean({ title: "Boolean", default: DEFAULT_FILLABLE }), + Type.Array(StringEnum(ActionContext), { title: "Context", uniqueItems: true }) + ], + { + default: DEFAULT_FILLABLE + } + ) + ), + hidden: Type.Optional( + Type.Union( + [ + Type.Boolean({ title: "Boolean", default: DEFAULT_HIDDEN }), + // @todo: tmp workaround + Type.Array(StringEnum(TmpContext), { title: "Context", uniqueItems: true }) + ], + { + default: DEFAULT_HIDDEN + } + ) + ), + // if field is virtual, it will not call transformPersist & transformRetrieve + virtual: Type.Optional(Type.Boolean()), + default_value: Type.Optional(Type.Any()) + }, + { + additionalProperties: false + } +); +export type BaseFieldConfig = Static; + +export type SchemaResponse = [string, ColumnDataType, ColumnBuilderCallback] | undefined; + +export abstract class Field< + Config extends BaseFieldConfig = BaseFieldConfig, + Type = any, + Required extends true | false = false +> { + _required!: Required; + _type!: Type; + + /** + * Property name that gets persisted on database + */ + readonly name: string; + readonly type: string = "field"; + readonly config: Config; + + constructor(name: string, config?: Partial) { + this.name = name; + this._type; + this._required; + + try { + this.config = parse(this.getSchema(), config || {}) as Config; + } catch (e) { + if (e instanceof TypeInvalidError) { + throw new InvalidFieldConfigException(this, config, e); + } + + throw e; + } + } + + getType() { + return this.type; + } + + protected abstract getSchema(): TSchema; + + protected useSchemaHelper( + type: ColumnDataType, + builder?: (col: ColumnDefinitionBuilder) => ColumnDefinitionBuilder + ): SchemaResponse { + return [ + this.name, + type, + (col: ColumnDefinitionBuilder) => { + if (builder) return builder(col); + return col; + } + ]; + } + + /** + * Used in SchemaManager.ts + * @param em + */ + abstract schema(em: EntityManager): SchemaResponse; + + hasDefault() { + return this.config.default_value !== undefined; + } + + getDefault() { + return this.config?.default_value; + } + + isFillable(context?: TActionContext): boolean { + if (Array.isArray(this.config.fillable)) { + return context ? this.config.fillable.includes(context) : DEFAULT_FILLABLE; + } + return !!this.config.fillable; + } + + isHidden(context?: TmpActionAndRenderContext): boolean { + if (Array.isArray(this.config.hidden)) { + return context ? this.config.hidden.includes(context as any) : DEFAULT_HIDDEN; + } + return this.config.hidden ?? false; + } + + isRequired(): boolean { + return this.config?.required ?? false; + } + + /** + * Virtual fields are not persisted or retrieved from database + * Used for MediaField, to add specifics about uploads, etc. + */ + isVirtual(): boolean { + return this.config.virtual ?? false; + } + + getLabel(): string { + return this.config.label ?? snakeToPascalWithSpaces(this.name); + } + + getDescription(): string | undefined { + return this.config.description; + } + + /** + * [GET] DB -> field.transformRetrieve -> [sent] + * table: form.getValue("table") + * form: form.getValue("form") -> modified -> form.getValue("submit") -> [sent] + * + * [PATCH] body parse json -> field.transformPersist -> [stored] + * + * @param value + * @param context + */ + getValue(value: any, context?: TRenderContext) { + return value; + } + + getHtmlConfig(): { element: HTMLInputTypeAttribute | string; props?: InputHTMLAttributes } { + return { + element: "input", + props: { type: "text" } + }; + } + + isValid(value: any, context: TActionContext): boolean { + if (value) { + return this.isFillable(context); + } else { + return !this.isRequired(); + } + } + + /** + * Transform value after retrieving from database + * @param value + */ + transformRetrieve(value: any): any { + return value; + } + + /** + * Transform value before persisting to database + * @param value + * @param em EntityManager (optional, for relation fields) + */ + async transformPersist( + value: unknown, + em: EntityManager, + context: TActionContext + ): Promise { + if (this.nullish(value)) { + if (this.isRequired() && !this.hasDefault()) { + throw TransformPersistFailedException.required(this.name); + } + return this.getDefault(); + } + + return value; + } + + protected toSchemaWrapIfRequired(schema: Schema) { + return this.isRequired() ? schema : Type.Optional(schema); + } + + protected nullish(value: any) { + return value === null || value === undefined; + } + + toJsonSchema(): TSchema { + return this.toSchemaWrapIfRequired(Type.Any()); + } + + toJSON() { + return { + //name: this.name, + type: this.type, + config: this.config + }; + } +} diff --git a/app/src/data/fields/JsonField.ts b/app/src/data/fields/JsonField.ts new file mode 100644 index 0000000..dd08ae9 --- /dev/null +++ b/app/src/data/fields/JsonField.ts @@ -0,0 +1,104 @@ +import { type Static, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const jsonFieldConfigSchema = Type.Composite([baseFieldConfigSchema, Type.Object({})]); + +export type JsonFieldConfig = Static; + +export class JsonField extends Field< + JsonFieldConfig, + TypeOverride, + Required +> { + override readonly type = "json"; + + protected getSchema() { + return jsonFieldConfigSchema; + } + + override schema() { + return this.useSchemaHelper("text"); + } + + /** + * Transform value after retrieving from database + * @param value + */ + override transformRetrieve(value: any): any { + const val = super.transformRetrieve(value); + + if (val === null && this.hasDefault()) { + return this.getDefault(); + } + + if (this.isSerialized(val)) { + return JSON.parse(val); + } + + return val; + } + + isSerializable(value: any) { + try { + const stringified = JSON.stringify(value); + if (stringified === JSON.stringify(JSON.parse(stringified))) { + return true; + } + } catch (e) {} + + return false; + } + + isSerialized(value: any) { + try { + if (typeof value === "string") { + return value === JSON.stringify(JSON.parse(value)); + } + } catch (e) {} + + return false; + } + + override getValue(value: any, context: TRenderContext): any { + switch (context) { + case "form": + if (value === null) return ""; + return JSON.stringify(value, null, 2); + case "table": + if (value === null) return null; + return JSON.stringify(value); + case "submit": + if (typeof value === "string" && value.length === 0) { + return null; + } + + return JSON.parse(value); + } + + return value; + } + + override async transformPersist( + _value: any, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(_value, em, context); + //console.log("value", value); + if (this.nullish(value)) return value; + + if (!this.isSerializable(value)) { + throw new TransformPersistFailedException( + `Field "${this.name}" must be serializable to JSON.` + ); + } + + if (this.isSerialized(value)) { + return value; + } + + return JSON.stringify(value); + } +} diff --git a/app/src/data/fields/JsonSchemaField.ts b/app/src/data/fields/JsonSchemaField.ts new file mode 100644 index 0000000..5f4e2c4 --- /dev/null +++ b/app/src/data/fields/JsonSchemaField.ts @@ -0,0 +1,132 @@ +import { type Schema as JsonSchema, Validator } from "@cfworker/json-schema"; +import { Default, FromSchema, type Static, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const jsonSchemaFieldConfigSchema = Type.Composite( + [ + Type.Object({ + schema: Type.Object({}, { default: {} }), + ui_schema: Type.Optional(Type.Object({})), + default_from_schema: Type.Optional(Type.Boolean()) + }), + baseFieldConfigSchema + ], + { + additionalProperties: false + } +); + +export type JsonSchemaFieldConfig = Static; + +export class JsonSchemaField< + Required extends true | false = false, + TypeOverride = object +> extends Field { + override readonly type = "jsonschema"; + private validator: Validator; + + constructor(name: string, config: Partial) { + super(name, config); + this.validator = new Validator(this.getJsonSchema()); + } + + protected getSchema() { + return jsonSchemaFieldConfigSchema; + } + + override schema() { + return this.useSchemaHelper("text"); + } + + getJsonSchema(): JsonSchema { + return this.config?.schema as JsonSchema; + } + + getJsonUiSchema() { + return this.config.ui_schema ?? {}; + } + + override isValid(value: any, context: TActionContext = "update"): boolean { + const parentValid = super.isValid(value, context); + //console.log("jsonSchemaField:isValid", this.getJsonSchema(), this.name, value, parentValid); + + if (parentValid) { + // already checked in parent + if (!value || typeof value !== "object") { + //console.log("jsonschema:valid: not checking", this.name, value, context); + return true; + } + + const result = this.validator.validate(value); + //console.log("jsonschema:errors", this.name, result.errors); + return result.valid; + } else { + //console.log("jsonschema:invalid", this.name, value, context); + } + + return false; + } + + override getValue(value: any, context: TRenderContext): any { + switch (context) { + case "form": + if (value === null) return ""; + return value; + case "table": + if (value === null) return null; + return value; + case "submit": + break; + } + + return value; + } + + override transformRetrieve(value: any): any { + const val = super.transformRetrieve(value); + + if (val === null) { + if (this.config.default_from_schema) { + try { + return Default(FromSchema(this.getJsonSchema()), {}); + } catch (e) { + //console.error("jsonschema:transformRetrieve", e); + return null; + } + } else if (this.hasDefault()) { + return this.getDefault(); + } + } + + return val; + } + + override async transformPersist( + _value: any, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(_value, em, context); + if (this.nullish(value)) return value; + + if (!this.isValid(value)) { + throw new TransformPersistFailedException(this.name, value); + } + + if (!value || typeof value !== "object") return this.getDefault(); + + return JSON.stringify(value); + } + + override toJsonSchema() { + const schema = this.getJsonSchema() ?? { type: "object" }; + return this.toSchemaWrapIfRequired( + FromSchema({ + default: this.getDefault(), + ...schema + }) + ); + } +} diff --git a/app/src/data/fields/NumberField.ts b/app/src/data/fields/NumberField.ts new file mode 100644 index 0000000..0e98832 --- /dev/null +++ b/app/src/data/fields/NumberField.ts @@ -0,0 +1,100 @@ +import { type Static, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, type TRenderContext, baseFieldConfigSchema } from "./Field"; + +export const numberFieldConfigSchema = Type.Composite( + [ + Type.Object({ + default_value: Type.Optional(Type.Number()), + minimum: Type.Optional(Type.Number()), + maximum: Type.Optional(Type.Number()), + exclusiveMinimum: Type.Optional(Type.Number()), + exclusiveMaximum: Type.Optional(Type.Number()), + multipleOf: Type.Optional(Type.Number()) + }), + baseFieldConfigSchema + ], + { + additionalProperties: false + } +); + +export type NumberFieldConfig = Static; + +export class NumberField extends Field< + NumberFieldConfig, + number, + Required +> { + override readonly type = "number"; + + protected getSchema() { + return numberFieldConfigSchema; + } + + override getHtmlConfig() { + return { + element: "input", + props: { + type: "number", + pattern: "d*", + inputMode: "numeric" + } as any // @todo: react expects "inputMode", but type dictates "inputmode" + }; + } + + schema() { + return this.useSchemaHelper("integer"); + } + + override getValue(value: any, context?: TRenderContext): any { + if (typeof value === "undefined" || value === null) return null; + + switch (context) { + case "submit": + return Number.parseInt(value); + } + + return value; + } + + override async transformPersist( + _value: unknown, + em: EntityManager, + context: TActionContext + ): Promise { + const value = await super.transformPersist(_value, em, context); + + if (!this.nullish(value) && typeof value !== "number") { + throw TransformPersistFailedException.invalidType(this.name, "number", value); + } + + if (this.config.maximum && (value as number) > this.config.maximum) { + throw new TransformPersistFailedException( + `Field "${this.name}" cannot be greater than ${this.config.maximum}` + ); + } + + if (this.config.minimum && (value as number) < this.config.minimum) { + throw new TransformPersistFailedException( + `Field "${this.name}" cannot be less than ${this.config.minimum}` + ); + } + + return value as number; + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired( + Type.Number({ + default: this.getDefault(), + minimum: this.config?.minimum, + maximum: this.config?.maximum, + exclusiveMinimum: this.config?.exclusiveMinimum, + exclusiveMaximum: this.config?.exclusiveMaximum, + multipleOf: this.config?.multipleOf + }) + ); + } +} diff --git a/app/src/data/fields/PrimaryField.ts b/app/src/data/fields/PrimaryField.ts new file mode 100644 index 0000000..24e6302 --- /dev/null +++ b/app/src/data/fields/PrimaryField.ts @@ -0,0 +1,46 @@ +import { config } from "core"; +import { type Static, Type } from "core/utils"; +import { Field, baseFieldConfigSchema } from "./Field"; + +export const primaryFieldConfigSchema = Type.Composite([ + Type.Omit(baseFieldConfigSchema, ["required"]), + Type.Object({ + required: Type.Optional(Type.Literal(false)) + }) +]); + +export type PrimaryFieldConfig = Static; + +export class PrimaryField extends Field< + PrimaryFieldConfig, + string, + Required +> { + override readonly type = "primary"; + + constructor(name: string = config.data.default_primary_field) { + super(name, { fillable: false, required: false }); + } + + override isRequired(): boolean { + return false; + } + + protected getSchema() { + return baseFieldConfigSchema; + } + + schema() { + return this.useSchemaHelper("integer", (col) => { + return col.primaryKey().notNull().autoIncrement(); + }); + } + + override async transformPersist(value: any): Promise { + throw new Error("This function should not be called"); + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired(Type.Number({ writeOnly: undefined })); + } +} diff --git a/app/src/data/fields/TextField.ts b/app/src/data/fields/TextField.ts new file mode 100644 index 0000000..6314618 --- /dev/null +++ b/app/src/data/fields/TextField.ts @@ -0,0 +1,120 @@ +import { type Static, Type } from "core/utils"; +import type { EntityManager } from "data"; +import { TransformPersistFailedException } from "../errors"; +import { Field, type TActionContext, baseFieldConfigSchema } from "./Field"; + +export const textFieldConfigSchema = Type.Composite( + [ + Type.Object({ + default_value: Type.Optional(Type.String()), + minLength: Type.Optional(Type.Number()), + maxLength: Type.Optional(Type.Number()), + pattern: Type.Optional(Type.String()), + html_config: Type.Optional( + Type.Object({ + element: Type.Optional(Type.String({ default: "input" })), + props: Type.Optional( + Type.Object( + {}, + { + additionalProperties: Type.Union([ + Type.String({ title: "String" }), + Type.Number({ title: "Number" }) + ]) + } + ) + ) + }) + ) + }), + baseFieldConfigSchema + ], + { + additionalProperties: false + } +); + +export type TextFieldConfig = Static; + +export class TextField extends Field< + TextFieldConfig, + string, + Required +> { + override readonly type = "text"; + + protected getSchema() { + return textFieldConfigSchema; + } + + override schema() { + return this.useSchemaHelper("text"); + } + + override getHtmlConfig() { + if (this.config.html_config) { + return this.config.html_config as any; + } + + return super.getHtmlConfig(); + } + + /** + * Transform value after retrieving from database + * @param value + */ + override transformRetrieve(value: string): string | null { + const val = super.transformRetrieve(value); + + // @todo: now sure about these two + if (this.config.maxLength) { + return val.substring(0, this.config.maxLength); + } + + if (this.isRequired()) { + return val ? val.toString() : ""; + } + + return val; + } + + override async transformPersist( + _value: any, + em: EntityManager, + context: TActionContext + ): Promise { + let value = await super.transformPersist(_value, em, context); + + if (this.nullish(value)) return value; + + // transform to string + if (value !== null && typeof value !== "string") { + value = String(value); + } + + if (this.config.maxLength && value?.length > this.config.maxLength) { + throw new TransformPersistFailedException( + `Field "${this.name}" must be at most ${this.config.maxLength} character(s)` + ); + } + + if (this.config.minLength && value?.length < this.config.minLength) { + throw new TransformPersistFailedException( + `Field "${this.name}" must be at least ${this.config.minLength} character(s)` + ); + } + + return value; + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired( + Type.String({ + default: this.getDefault(), + minLength: this.config?.minLength, + maxLength: this.config?.maxLength, + pattern: this.config?.pattern + }) + ); + } +} diff --git a/app/src/data/fields/VirtualField.ts b/app/src/data/fields/VirtualField.ts new file mode 100644 index 0000000..ebb1300 --- /dev/null +++ b/app/src/data/fields/VirtualField.ts @@ -0,0 +1,32 @@ +import { type Static, Type } from "core/utils"; +import { Field, baseFieldConfigSchema } from "./Field"; + +export const virtualFieldConfigSchema = Type.Composite([baseFieldConfigSchema, Type.Object({})]); + +export type VirtualFieldConfig = Static; + +export class VirtualField extends Field { + override readonly type = "virtual"; + + constructor(name: string, config?: Partial) { + // field must be virtual, as it doesn't store a reference to the entity + super(name, { ...config, fillable: false, virtual: true }); + } + + protected getSchema() { + return virtualFieldConfigSchema; + } + + schema() { + return undefined; + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired( + Type.Any({ + default: this.getDefault(), + readOnly: true + }) + ); + } +} diff --git a/app/src/data/fields/index.ts b/app/src/data/fields/index.ts new file mode 100644 index 0000000..ddca371 --- /dev/null +++ b/app/src/data/fields/index.ts @@ -0,0 +1,55 @@ +import { BooleanField, type BooleanFieldConfig, booleanFieldConfigSchema } from "./BooleanField"; +import { DateField, type DateFieldConfig, dateFieldConfigSchema } from "./DateField"; +import { EnumField, type EnumFieldConfig, enumFieldConfigSchema } from "./EnumField"; +import { JsonField, type JsonFieldConfig, jsonFieldConfigSchema } from "./JsonField"; +import { + JsonSchemaField, + type JsonSchemaFieldConfig, + jsonSchemaFieldConfigSchema +} from "./JsonSchemaField"; +import { NumberField, type NumberFieldConfig, numberFieldConfigSchema } from "./NumberField"; +import { PrimaryField, type PrimaryFieldConfig, primaryFieldConfigSchema } from "./PrimaryField"; +import { TextField, type TextFieldConfig, textFieldConfigSchema } from "./TextField"; + +export { + PrimaryField, + primaryFieldConfigSchema, + type PrimaryFieldConfig, + BooleanField, + booleanFieldConfigSchema, + type BooleanFieldConfig, + DateField, + dateFieldConfigSchema, + type DateFieldConfig, + EnumField, + enumFieldConfigSchema, + type EnumFieldConfig, + JsonField, + jsonFieldConfigSchema, + type JsonFieldConfig, + JsonSchemaField, + jsonSchemaFieldConfigSchema, + type JsonSchemaFieldConfig, + NumberField, + numberFieldConfigSchema, + type NumberFieldConfig, + TextField, + textFieldConfigSchema, + type TextFieldConfig +}; + +export * from "./Field"; +export * from "./PrimaryField"; +export * from "./VirtualField"; +export * from "./indices/EntityIndex"; + +export const FieldClassMap = { + primary: { schema: primaryFieldConfigSchema, field: PrimaryField }, + text: { schema: textFieldConfigSchema, field: TextField }, + number: { schema: numberFieldConfigSchema, field: NumberField }, + boolean: { schema: booleanFieldConfigSchema, field: BooleanField }, + date: { schema: dateFieldConfigSchema, field: DateField }, + enum: { schema: enumFieldConfigSchema, field: EnumField }, + json: { schema: jsonFieldConfigSchema, field: JsonField }, + jsonschema: { schema: jsonSchemaFieldConfigSchema, field: JsonSchemaField } +} as const; diff --git a/app/src/data/fields/indices/EntityIndex.ts b/app/src/data/fields/indices/EntityIndex.ts new file mode 100644 index 0000000..8346eba --- /dev/null +++ b/app/src/data/fields/indices/EntityIndex.ts @@ -0,0 +1,46 @@ +import type { Entity } from "../../entities"; +import { Field } from "../Field"; + +export class EntityIndex { + constructor( + public entity: Entity, + public fields: Field[], + public unique: boolean = false, + public name?: string + ) { + if (fields.length === 0) { + throw new Error("Indices must contain at least one field"); + } + if (fields.some((f) => !(f instanceof Field))) { + throw new Error("All fields must be instances of Field"); + } + + if (unique) { + const firstRequired = fields[0]?.isRequired(); + if (!firstRequired) { + throw new Error( + `Unique indices must have first field as required: ${fields + .map((f) => f.name) + .join(", ")}` + ); + } + } + + if (!name) { + this.name = [ + unique ? "idx_unique" : "idx", + entity.name, + ...fields.map((f) => f.name) + ].join("_"); + } + } + + toJSON() { + return { + entity: this.entity.name, + fields: this.fields.map((f) => f.name), + //name: this.name, + unique: this.unique + }; + } +} diff --git a/app/src/data/helper.ts b/app/src/data/helper.ts new file mode 100644 index 0000000..481ab0f --- /dev/null +++ b/app/src/data/helper.ts @@ -0,0 +1,48 @@ +import type { EntityData, Field } from "data"; +import { transform } from "lodash-es"; + +export function getDefaultValues(fields: Field[], data: EntityData): EntityData { + return transform( + fields, + (acc, field) => { + // form fields don't like "null" or "undefined", so return empty string + acc[field.name] = field.getValue(data?.[field.name], "form") ?? ""; + }, + {} as EntityData + ); +} + +export function getChangeSet( + action: string, + formData: EntityData, + data: EntityData, + fields: Field[] +): EntityData { + return transform( + formData, + (acc, _value, key) => { + const field = fields.find((f) => f.name === key); + // @todo: filtering virtual here, need to check (because of media) + if (!field || field.isVirtual()) return; + const value = _value === "" ? null : _value; + + const newValue = field.getValue(value, "submit"); + // @todo: add typing for "action" + if (action === "create" || newValue !== data[key]) { + acc[key] = newValue; + console.log("changed", { + key, + value, + valueType: typeof value, + prev: data[key], + newValue, + new: value, + sent: acc[key] + }); + } else { + //console.log("no change", key, value, data[key]); + } + }, + {} as typeof formData + ); +} diff --git a/app/src/data/index.ts b/app/src/data/index.ts new file mode 100644 index 0000000..674d490 --- /dev/null +++ b/app/src/data/index.ts @@ -0,0 +1,28 @@ +import { MutatorEvents, RepositoryEvents } from "./events"; + +export * from "./fields"; +export * from "./entities"; +export * from "./relations"; +export * from "./schema/SchemaManager"; + +export { + type RepoQuery, + defaultQuerySchema, + querySchema, + whereSchema +} from "./server/data-query-impl"; + +export { whereRepoSchema as deprecated__whereRepoSchema } from "./server/query"; + +export { Connection } from "./connection/Connection"; +export { LibsqlConnection, type LibSqlCredentials } from "./connection/LibsqlConnection"; +export { SqliteConnection } from "./connection/SqliteConnection"; +export { SqliteLocalConnection } from "./connection/SqliteLocalConnection"; + +export const DatabaseEvents = { + ...MutatorEvents, + ...RepositoryEvents +}; +export { MutatorEvents, RepositoryEvents }; + +export * as DataPermissions from "./permissions"; diff --git a/app/src/data/permissions/index.ts b/app/src/data/permissions/index.ts new file mode 100644 index 0000000..47a57b3 --- /dev/null +++ b/app/src/data/permissions/index.ts @@ -0,0 +1,9 @@ +import { Permission } from "core"; + +export const entityRead = new Permission("data.entity.read"); +export const entityCreate = new Permission("data.entity.create"); +export const entityUpdate = new Permission("data.entity.update"); +export const entityDelete = new Permission("data.entity.delete"); +export const databaseSync = new Permission("data.database.sync"); +export const rawQuery = new Permission("data.raw.query"); +export const rawMutate = new Permission("data.raw.mutate"); diff --git a/app/src/data/plugins/DeserializeJsonValuesPlugin.ts b/app/src/data/plugins/DeserializeJsonValuesPlugin.ts new file mode 100644 index 0000000..494b4b4 --- /dev/null +++ b/app/src/data/plugins/DeserializeJsonValuesPlugin.ts @@ -0,0 +1,36 @@ +import type { + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + QueryResult, + RootOperationNode, + UnknownRow, +} from "kysely"; + +type KeyValueObject = { [key: string]: any }; + +export class DeserializeJsonValuesPlugin implements KyselyPlugin { + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + return args.node; + } + transformResult( + args: PluginTransformResultArgs + ): Promise> { + return Promise.resolve({ + ...args.result, + rows: args.result.rows.map((row: KeyValueObject) => { + const result: KeyValueObject = {}; + for (const key in row) { + try { + // Attempt to parse the value as JSON + result[key] = JSON.parse(row[key]); + } catch (error) { + // If parsing fails, keep the original value + result[key] = row[key]; + } + } + return result; + }), + }); + } +} diff --git a/app/src/data/plugins/FilterNumericKeysPlugin.ts b/app/src/data/plugins/FilterNumericKeysPlugin.ts new file mode 100644 index 0000000..589fa1d --- /dev/null +++ b/app/src/data/plugins/FilterNumericKeysPlugin.ts @@ -0,0 +1,31 @@ +import type { + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + QueryResult, + RootOperationNode, + UnknownRow, +} from "kysely"; + +type KeyValueObject = { [key: string]: any }; + +export class FilterNumericKeysPlugin implements KyselyPlugin { + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + return args.node; + } + transformResult(args: PluginTransformResultArgs): Promise> { + return Promise.resolve({ + ...args.result, + rows: args.result.rows.map((row: KeyValueObject) => { + const filteredObj: KeyValueObject = {}; + for (const key in row) { + if (Number.isNaN(+key)) { + // Check if the key is not a number + filteredObj[key] = row[key]; + } + } + return filteredObj; + }), + }); + } +} diff --git a/app/src/data/plugins/KyselyPluginRunner.ts b/app/src/data/plugins/KyselyPluginRunner.ts new file mode 100644 index 0000000..b11cded --- /dev/null +++ b/app/src/data/plugins/KyselyPluginRunner.ts @@ -0,0 +1,23 @@ +import type { KyselyPlugin, UnknownRow } from "kysely"; + +// @todo: add test +export class KyselyPluginRunner { + protected plugins: Set; + + constructor(plugins: KyselyPlugin[] = []) { + this.plugins = new Set(plugins); + } + + async transformResultRows(rows: T[]): Promise { + let copy = rows; + for (const plugin of this.plugins) { + const res = await plugin.transformResult({ + queryId: "1" as any, + result: { rows: copy as UnknownRow[] }, + }); + copy = res.rows as T[]; + } + + return copy as T[]; + } +} diff --git a/app/src/data/prototype/index.ts b/app/src/data/prototype/index.ts new file mode 100644 index 0000000..d526c2e --- /dev/null +++ b/app/src/data/prototype/index.ts @@ -0,0 +1,295 @@ +import { + BooleanField, + type BooleanFieldConfig, + DateField, + type DateFieldConfig, + Entity, + type EntityConfig, + EnumField, + type EnumFieldConfig, + type Field, + JsonField, + type JsonFieldConfig, + JsonSchemaField, + type JsonSchemaFieldConfig, + ManyToManyRelation, + type ManyToManyRelationConfig, + ManyToOneRelation, + type ManyToOneRelationConfig, + NumberField, + type NumberFieldConfig, + OneToOneRelation, + type OneToOneRelationConfig, + PolymorphicRelation, + type PolymorphicRelationConfig, + type TEntityType, + TextField, + type TextFieldConfig +} from "data"; +import type { Generated } from "kysely"; +import { MediaField, type MediaFieldConfig, type MediaItem } from "media/MediaField"; + +type Options = { + entity: { name: string; fields: Record> }; + field_name: string; + config: Config; + is_required: boolean; +}; + +const FieldMap = { + text: (o: Options) => new TextField(o.field_name, { ...o.config, required: o.is_required }), + number: (o: Options) => new NumberField(o.field_name, { ...o.config, required: o.is_required }), + date: (o: Options) => new DateField(o.field_name, { ...o.config, required: o.is_required }), + datetime: (o: Options) => new DateField(o.field_name, { ...o.config, required: o.is_required }), + boolean: (o: Options) => + new BooleanField(o.field_name, { ...o.config, required: o.is_required }), + enumm: (o: Options) => new EnumField(o.field_name, { ...o.config, required: o.is_required }), + json: (o: Options) => new JsonField(o.field_name, { ...o.config, required: o.is_required }), + jsonSchema: (o: Options) => + new JsonSchemaField(o.field_name, { ...o.config, required: o.is_required }), + media: (o: Options) => + new MediaField(o.field_name, { ...o.config, entity: o.entity.name, required: o.is_required }), + medium: (o: Options) => + new MediaField(o.field_name, { ...o.config, entity: o.entity.name, required: o.is_required }) +} as const; +type TFieldType = keyof typeof FieldMap; + +export function text( + config?: Omit +): TextField & { required: () => TextField } { + return new FieldPrototype("text", config, false) as any; +} +export function number( + config?: Omit +): NumberField & { required: () => NumberField } { + return new FieldPrototype("number", config, false) as any; +} +export function date( + config?: Omit +): DateField & { required: () => DateField } { + return new FieldPrototype("date", { ...config, type: "date" }, false) as any; +} +export function datetime( + config?: Omit +): DateField & { required: () => DateField } { + return new FieldPrototype("date", { ...config, type: "datetime" }, false) as any; +} +export function week( + config?: Omit +): DateField & { required: () => DateField } { + return new FieldPrototype("date", { ...config, type: "week" }, false) as any; +} +export function boolean( + config?: Omit +): BooleanField & { required: () => BooleanField } { + return new FieldPrototype("boolean", config, false) as any; +} +export function enumm( + config?: Omit & { + enum: string[] | { label: string; value: string }[]; + } +): EnumField & { + required: () => EnumField; +} { + const type = typeof config?.enum?.[0] !== "string" ? "objects" : "strings"; + const actual_config = { + options: { + type, + values: config?.enum ?? [] + } + }; + return new FieldPrototype("enumm", actual_config, false) as any; +} +export function json( + config?: Omit +): JsonField & { required: () => JsonField } { + return new FieldPrototype("json", config, false) as any; +} +export function jsonSchema( + config?: Omit +): JsonField & { required: () => JsonSchemaField } { + return new FieldPrototype("jsonSchema", config, false) as any; +} +export function media(config?: Omit): MediaField { + return new FieldPrototype("media", config, false) as any; +} +export function medium( + config?: Omit +): MediaField { + return new FieldPrototype("media", { ...config, max_items: 1 }, false) as any; +} +export function make>(name: string, field: Actual): Actual { + if (field instanceof FieldPrototype) { + return field.make(name) as Actual; + } + throw new Error("Invalid field"); +} + +export class FieldPrototype { + constructor( + public type: TFieldType, + public config: any, + public is_required: boolean + ) {} + + required() { + this.is_required = true; + return this; + } + + getField(o: Options): Field { + if (!FieldMap[this.type]) { + throw new Error(`Unknown field type: ${this.type}`); + } + try { + return FieldMap[this.type](o) as unknown as Field; + } catch (e) { + throw new Error(`Faild to construct field "${this.type}": ${e}`); + } + } + + make(field_name: string): Field { + if (!FieldMap[this.type]) { + throw new Error(`Unknown field type: ${this.type}`); + } + try { + return FieldMap[this.type]({ + entity: { name: "unknown", fields: {} }, + field_name, + config: this.config, + is_required: this.is_required + }) as unknown as Field; + } catch (e) { + throw new Error(`Faild to construct field "${this.type}": ${e}`); + } + } +} + +//type Entity> = {}> = { name: string; fields: Fields }; + +export function entity< + EntityName extends string, + Fields extends Record> +>( + name: EntityName, + fields: Fields, + config?: EntityConfig, + type?: TEntityType +): Entity { + const _fields: Field[] = []; + for (const [field_name, field] of Object.entries(fields)) { + const f = field as unknown as FieldPrototype; + const o: Options = { + entity: { name, fields }, + field_name, + config: f.config, + is_required: f.is_required + }; + _fields.push(f.getField(o)); + } + return new Entity(name, _fields, config, type); +} + +export function relation(local: Local) { + return { + manyToOne: (foreign: Foreign, config?: ManyToOneRelationConfig) => { + return new ManyToOneRelation(local, foreign, config); + }, + oneToOne: (foreign: Foreign, config?: OneToOneRelationConfig) => { + return new OneToOneRelation(local, foreign, config); + }, + manyToMany: ( + foreign: Foreign, + config?: ManyToManyRelationConfig, + additionalFields?: Record> + ) => { + const add_fields: Field[] = []; + if (additionalFields) { + const fields = additionalFields!; + const _fields: Field[] = []; + const entity_name = + config?.connectionTable ?? ManyToManyRelation.defaultConnectionTable(local, foreign); + for (const [field_name, field] of Object.entries(additionalFields)) { + const f = field as unknown as FieldPrototype; + const o: Options = { + entity: { name: entity_name, fields }, + field_name, + config: f.config, + is_required: f.is_required + }; + _fields.push(f.getField(o)); + } + add_fields.push(_fields as any); + } + + return new ManyToManyRelation(local, foreign, config as any, add_fields); + }, + polyToOne: ( + foreign: Foreign, + config?: Omit + ) => { + return new PolymorphicRelation(local, foreign, { ...config, targetCardinality: 1 }); + }, + polyToMany: ( + foreign: Foreign, + config?: PolymorphicRelationConfig + ) => { + return new PolymorphicRelation(local, foreign, config); + } + }; +} + +type InferEntityFields = T extends Entity + ? { + [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required } + ? Required extends true + ? Type + : Type | undefined + : never; + } + : never; + +export type InferFields = Fields extends Record> + ? { + [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required } + ? Required extends true + ? Type + : Type | undefined + : never; + } + : never; + +type Prettify = { + [K in keyof T]: T[K]; +}; +export type Simplify = { [KeyType in keyof T]: T[KeyType] } & {}; + +// from https://github.com/type-challenges/type-challenges/issues/28200 +type Merge = { + [K in keyof T]: T[K]; +}; +type OptionalUndefined< + T, + Props extends keyof T = keyof T, + OptionsProps extends keyof T = Props extends keyof T + ? undefined extends T[Props] + ? Props + : never + : never +> = Merge< + { + [K in OptionsProps]?: T[K]; + } & { + [K in Exclude]: T[K]; + } +>; + +type InferField = Field extends { _type: infer Type; _required: infer Required } + ? Required extends true + ? Type + : Type | undefined + : never; + +export type InsertSchema = Simplify>>; +export type Schema = { id: Generated } & InsertSchema; +export type FieldSchema = Simplify>>; diff --git a/app/src/data/relations/EntityRelation.ts b/app/src/data/relations/EntityRelation.ts new file mode 100644 index 0000000..e7d680b --- /dev/null +++ b/app/src/data/relations/EntityRelation.ts @@ -0,0 +1,231 @@ +import { type Static, Type, parse } from "core/utils"; +import type { SelectQueryBuilder } from "kysely"; +import type { Entity, EntityData, EntityManager } from "../entities"; +import { + type EntityRelationAnchor, + type MutationInstructionResponse, + RelationHelper +} from "../relations"; +import type { RepoQuery } from "../server/data-query-impl"; +import type { RelationType } from "./relation-types"; + +export type KyselyJsonFrom = any; +export type KyselyQueryBuilder = SelectQueryBuilder; + +/*export type RelationConfig = { + mappedBy?: string; + inversedBy?: string; + sourceCardinality?: number; + connectionTable?: string; + connectionTableMappedName?: string; + required?: boolean; +};*/ + +export type BaseRelationConfig = Static; + +// @todo: add generic type for relation config +export abstract class EntityRelation< + Schema extends typeof EntityRelation.schema = typeof EntityRelation.schema +> { + config: Static; + + source: EntityRelationAnchor; + target: EntityRelationAnchor; + + // @todo: add unit tests + // allowed directions, used in RelationAccessor for visibility + directions: ("source" | "target")[] = ["source", "target"]; + + static schema = Type.Object({ + mappedBy: Type.Optional(Type.String()), + inversedBy: Type.Optional(Type.String()), + required: Type.Optional(Type.Boolean()) + }); + + // don't make protected, App requires it to instantiatable + constructor( + source: EntityRelationAnchor, + target: EntityRelationAnchor, + config: Partial> = {} + ) { + this.source = source; + this.target = target; + + const schema = (this.constructor as typeof EntityRelation).schema; + // @ts-ignore for now + this.config = parse(schema, config); + } + + abstract initialize(em: EntityManager): void; + + /** + * Build the "with" part of the query. + * + * @param entity requesting entity, so target needs to be added + * @param qb + * @param jsonFrom + */ + abstract buildWith( + entity: Entity, + qb: KyselyQueryBuilder, + jsonFrom: KyselyJsonFrom, + reference: string + ): KyselyQueryBuilder; + + abstract buildJoin( + entity: Entity, + qb: KyselyQueryBuilder, + reference: string + ): KyselyQueryBuilder; + + getReferenceQuery(entity: Entity, id: number, reference: string): Partial { + return {}; + } + + /** @deprecated */ + helper(entity_name: string): RelationHelper { + return new RelationHelper(this, entity_name); + } + + /** + * Get the other side of the relation quickly + * @param entity + */ + other(entity: Entity | string): EntityRelationAnchor { + const entity_name = typeof entity === "string" ? entity : entity.name; + + // special case for self referencing, check which side is not cardinality 1 + if (this.source.entity.name === this.target.entity.name) { + return this.source.cardinality === 1 ? this.target : this.source; + } + + if (this.source.entity.name === entity_name) { + return this.target; + } else if (this.target.entity.name === entity_name) { + return this.source; + } + + throw new Error( + `Entity "${entity_name}" is not part of the relation ` + + `"${this.source.entity.name} <-> ${this.target.entity.name}"` + ); + } + + ref(reference: string): EntityRelationAnchor { + return this.source.reference === reference ? this.source : this.target; + } + + otherRef(reference: string): EntityRelationAnchor { + return this.source.reference === reference ? this.target : this.source; + } + + // @todo: add unit tests + visibleFrom(from: "source" | "target"): boolean { + return this.directions.includes(from); + } + + /** + * Hydrate the relation. "entity" represents where the payload belongs to. + * E.g. if entity is "categories", then value is the result of categories + * + * IMPORTANT: This method is called from EM, high potential of recursion! + * + * @param entity + * @param value + * @param em + */ + hydrate(entity: Entity | string, value: EntityData[], em: EntityManager) { + const entity_name = typeof entity === "string" ? entity : entity.name; + const anchor = this.ref(entity_name); + const hydrated = em.hydrate(anchor.entity.name, value); + + if (anchor.cardinality === 1) { + if (Array.isArray(hydrated) && hydrated.length > 1) { + throw new Error( + `Failed to hydrate "${anchor.entity.name}" ` + + `with value: ${JSON.stringify(value)} (cardinality: 1)` + ); + } + + return hydrated[0]; + } + + if (!hydrated) { + throw new Error( + `Failed to hydrate "${anchor.entity.name}" ` + + `with value: ${JSON.stringify(value)} (cardinality: -)` + ); + } + + return hydrated; + } + + /** + * Determines if the relation is listable for the given entity + * If the given entity is the one with the local reference, then it's not listable + * Only if there are multiple, which is generally the other side (except for 1:1) + * @param entity + */ + isListableFor(entity: Entity): boolean { + //console.log("isListableFor", entity.name, this.source.entity.name, this.target.entity.name); + return this.target.entity.name === entity.name; + } + + abstract type(): RelationType; + + get required(): boolean { + return !!this.config.required; + } + + async $set( + em: EntityManager, + key: string, + value: unknown + ): Promise { + throw new Error("$set is not allowed"); + } + + async $create( + em: EntityManager, + key: string, + value: unknown + ): Promise { + throw new Error("$create is not allowed"); + } + + async $attach( + em: EntityManager, + key: string, + value: unknown + ): Promise { + throw new Error("$attach is not allowed"); + } + + async $detach( + em: EntityManager, + key: string, + value: unknown + ): Promise { + throw new Error("$detach is not allowed"); + } + + getName(): string { + const parts = [ + this.type().replace(":", ""), + this.source.entity.name, + this.target.entity.name, + this.config.mappedBy, + this.config.inversedBy + ].filter(Boolean); + return parts.join("_"); + } + + toJSON() { + return { + type: this.type(), + source: this.source.entity.name, + target: this.target.entity.name, + config: this.config + }; + } +} diff --git a/app/src/data/relations/EntityRelationAnchor.ts b/app/src/data/relations/EntityRelationAnchor.ts new file mode 100644 index 0000000..710b1bd --- /dev/null +++ b/app/src/data/relations/EntityRelationAnchor.ts @@ -0,0 +1,25 @@ +import type { Entity } from "../entities"; + +export class EntityRelationAnchor { + entity: Entity; + cardinality?: number; + + /** + * The name that the other entity will use to reference this entity + */ + reference: string; + + constructor(entity: Entity, name: string, cardinality?: number) { + this.entity = entity; + this.cardinality = cardinality; + this.reference = name; + } + + toJSON() { + return { + entity: this.entity.name, + cardinality: this.cardinality, + name: this.reference, + }; + } +} diff --git a/app/src/data/relations/ManyToManyRelation.ts b/app/src/data/relations/ManyToManyRelation.ts new file mode 100644 index 0000000..25dbdca --- /dev/null +++ b/app/src/data/relations/ManyToManyRelation.ts @@ -0,0 +1,189 @@ +import { type Static, Type } from "core/utils"; +import { Entity, type EntityManager } from "../entities"; +import { type Field, PrimaryField, VirtualField } from "../fields"; +import type { RepoQuery } from "../server/data-query-impl"; +import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation"; +import { EntityRelationAnchor } from "./EntityRelationAnchor"; +import { RelationField } from "./RelationField"; +import { type RelationType, RelationTypes } from "./relation-types"; + +export type ManyToManyRelationConfig = Static; + +export class ManyToManyRelation extends EntityRelation { + connectionEntity: Entity; + additionalFields: Field[] = []; + connectionTableMappedName: string; + private em?: EntityManager; + + static override schema = Type.Composite( + [ + EntityRelation.schema, + Type.Object({ + connectionTable: Type.Optional(Type.String()), + connectionTableMappedName: Type.Optional(Type.String()) + }) + ], + { + additionalProperties: false + } + ); + + constructor( + source: Entity, + target: Entity, + config?: ManyToManyRelationConfig, + additionalFields?: Field[] + ) { + const connectionTable = + config?.connectionTable || ManyToManyRelation.defaultConnectionTable(source, target); + + const sourceAnchor = new EntityRelationAnchor(source, source.name); + const targetAnchor = new EntityRelationAnchor(target, target.name); + super(sourceAnchor, targetAnchor, config); + + this.connectionEntity = new Entity(connectionTable, additionalFields, undefined, "generated"); + + this.connectionTableMappedName = config?.connectionTableMappedName || connectionTable; + this.additionalFields = additionalFields || []; + //this.connectionTable = connectionTable; + } + + static defaultConnectionTable(source: Entity, target: Entity) { + return `${source.name}_${target.name}`; + } + + type(): RelationType { + return RelationTypes.ManyToMany; + } + + /** + * Many to many is always listable in both directions + */ + override isListableFor(): boolean { + return true; + } + + getField(entity: Entity): RelationField { + const conn = this.connectionEntity; + const selfField = conn.fields.find( + (f) => f instanceof RelationField && f.target() === entity.name + )!; + + if (!selfField || !(selfField instanceof RelationField)) { + throw new Error( + `Connection entity "${conn.name}" does not have a relation to "${entity.name}"` + ); + } + + return selfField; + } + + private getQueryInfo(entity: Entity) { + const other = this.other(entity); + const conn = this.connectionEntity; + const entityField = this.getField(entity); + const otherField = this.getField(other.entity); + const join = [ + conn.name, + `${other.entity.name}.${other.entity.getPrimaryField().name}`, + `${conn.name}.${otherField.name}` + ] as const; + + const entityRef = `${entity.name}.${entity.getPrimaryField().name}`; + const otherRef = `${conn.name}.${entityField.name}`; + + const groupBy = `${entity.name}.${entity.getPrimaryField().name}`; + + return { + other, + join, + entityRef, + otherRef, + groupBy + }; + } + + override getReferenceQuery(entity: Entity, id: number): Partial { + const conn = this.connectionEntity; + + return { + where: { + [`${conn.name}.${entity.name}_${entity.getPrimaryField().name}`]: id + }, + join: [this.target.reference] + }; + } + + buildJoin(entity: Entity, qb: KyselyQueryBuilder) { + const { other, join, entityRef, otherRef, groupBy } = this.getQueryInfo(entity); + + return qb + .innerJoin(other.entity.name, entityRef, otherRef) + .innerJoin(...join) + .groupBy(groupBy); + } + + buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) { + if (!this.em) { + throw new Error("EntityManager not set, can't build"); + } + const jsonBuildObject = this.em.connection.fn.jsonBuildObject; + if (!jsonBuildObject) { + throw new Error("Connection does not support jsonBuildObject"); + } + + const limit = 5; + const { other, join, entityRef, otherRef } = this.getQueryInfo(entity); + const additionalFields = this.connectionEntity.fields.filter( + (f) => !(f instanceof RelationField || f instanceof PrimaryField) + ); + + return qb.select((eb) => { + const select: any[] = other.entity.getSelect(other.entity.name); + // @todo: also add to find by references + if (additionalFields.length > 0) { + const conn = this.connectionEntity.name; + select.push( + jsonBuildObject( + Object.fromEntries( + additionalFields.map((f) => [f.name, eb.ref(`${conn}.${f.name}`)]) + ) + ).as(this.connectionTableMappedName) + ); + } + + return jsonFrom( + eb + .selectFrom(other.entity.name) + .select(select) + .whereRef(entityRef, "=", otherRef) + .innerJoin(...join) + .limit(limit) + ).as(other.reference); + }); + } + + initialize(em: EntityManager) { + this.em = em; + + //this.connectionEntity.addField(new RelationField(this.source.entity)); + //this.connectionEntity.addField(new RelationField(this.target.entity)); + this.connectionEntity.addField(RelationField.create(this, this.source)); + this.connectionEntity.addField(RelationField.create(this, this.target)); + + // @todo: check this + for (const field of this.additionalFields) { + this.source.entity.addField(new VirtualField(this.connectionTableMappedName)); + this.target.entity.addField(new VirtualField(this.connectionTableMappedName)); + } + + em.addEntity(this.connectionEntity); + } + + override getName(): string { + return [ + super.getName(), + [this.connectionEntity.name, this.connectionTableMappedName].filter(Boolean) + ].join("_"); + } +} diff --git a/app/src/data/relations/ManyToOneRelation.ts b/app/src/data/relations/ManyToOneRelation.ts new file mode 100644 index 0000000..57bb993 --- /dev/null +++ b/app/src/data/relations/ManyToOneRelation.ts @@ -0,0 +1,228 @@ +import type { PrimaryFieldType } from "core"; +import { snakeToPascalWithSpaces } from "core/utils"; +import { type Static, Type } from "core/utils"; +import type { Entity, EntityManager } from "../entities"; +import type { RepoQuery } from "../server/data-query-impl"; +import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation"; +import { EntityRelationAnchor } from "./EntityRelationAnchor"; +import { RelationField, type RelationFieldBaseConfig } from "./RelationField"; +import type { MutationInstructionResponse } from "./RelationMutator"; +import { type RelationType, RelationTypes } from "./relation-types"; + +/** + * Source entity receives the mapping field + * + * Many-to-one (many) [sources] has (one) [target] + * Example: [posts] has (one) [user] + * posts gets a users_id field + */ + +export type ManyToOneRelationConfig = Static; + +export class ManyToOneRelation extends EntityRelation { + private fieldConfig?: RelationFieldBaseConfig; + static DEFAULTS = { + with_limit: 5 + }; + + static override schema = Type.Composite( + [ + EntityRelation.schema, + Type.Object({ + sourceCardinality: Type.Optional(Type.Number()), + with_limit: Type.Optional( + Type.Number({ default: ManyToOneRelation.DEFAULTS.with_limit }) + ), + fieldConfig: Type.Optional( + Type.Object({ + label: Type.String() + }) + ) + }) + ], + { + additionalProperties: false + } + ); + + constructor( + source: Entity, + target: Entity, + config: Partial> = {} + ) { + const mappedBy = config.mappedBy || target.name; + const inversedBy = config.inversedBy || source.name; + + // if source can be multiple, allow it. otherwise unlimited + const sourceCardinality = + typeof config.sourceCardinality === "number" && config.sourceCardinality > 0 + ? config.sourceCardinality + : undefined; + const sourceAnchor = new EntityRelationAnchor(source, inversedBy, sourceCardinality); + const targetAnchor = new EntityRelationAnchor(target, mappedBy, 1); + super(sourceAnchor, targetAnchor, config); + + this.fieldConfig = config.fieldConfig ?? {}; + // set relation required or not + //this.required = !!config.required; + } + + type(): RelationType { + return RelationTypes.ManyToOne; + } + + override initialize(em: EntityManager) { + const defaultLabel = snakeToPascalWithSpaces(this.target.reference); + + // add required mapping field on source + const field = RelationField.create(this, this.target, { + label: defaultLabel, + ...this.fieldConfig + }); + + if (!this.source.entity.field(field.name)) { + this.source.entity.addField( + RelationField.create(this, this.target, { + label: defaultLabel, + ...this.fieldConfig + }) + ); + } + } + + /** + * Retrieve the RelationField + */ + getField(): RelationField { + const id = this.target.entity.getPrimaryField().name; + const field = this.source.entity.getField(`${this.target.reference}_${id}`); + + if (!(field instanceof RelationField)) { + throw new Error( + `Field "${this.target.reference}_${id}" not found on entity "${this.source.entity.name}"` + ); + } + + return field; + } + + private queryInfo(entity: Entity, reference: string) { + const side = this.source.reference === reference ? "source" : "target"; + const self = this[side]; + const other = this[side === "source" ? "target" : "source"]; + let relationRef: string; + let entityRef: string; + let otherRef: string; + if (side === "source") { + relationRef = this.source.reference; + entityRef = `${relationRef}.${this.getField().name}`; + otherRef = `${entity.name}.${self.entity.getPrimaryField().name}`; + } else { + relationRef = this.target.reference; + entityRef = `${relationRef}.${entity.getPrimaryField().name}`; + otherRef = `${entity.name}.${this.getField().name}`; + } + + const groupBy = `${entity.name}.${entity.getPrimaryField().name}`; + //console.log("queryInfo", entity.name, { reference, side, relationRef, entityRef, otherRef }); + + return { + other, + self, + relationRef, + entityRef, + otherRef, + groupBy + }; + } + + override getReferenceQuery(entity: Entity, id: number, reference: string): Partial { + const side = this.source.reference === reference ? "source" : "target"; + const self = this[side]; + const other = this[side === "source" ? "target" : "source"]; + const otherRef = `${other.reference}_${other.entity.getPrimaryField().name}`; + + return { + where: { + [otherRef]: id + }, + join: other.entity.name === self.entity.name ? [] : [other.entity.name] + }; + } + + buildJoin(entity: Entity, qb: KyselyQueryBuilder, reference: string) { + const { self, entityRef, otherRef, groupBy } = this.queryInfo(entity, reference); + return qb.innerJoin(self.entity.name, entityRef, otherRef).groupBy(groupBy); + } + + buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom, reference: string) { + const { self, entityRef, otherRef, relationRef } = this.queryInfo(entity, reference); + const limit = + self.cardinality === 1 + ? 1 + : this.config.with_limit ?? ManyToOneRelation.DEFAULTS.with_limit; + //console.log("buildWith", entity.name, reference, { limit }); + + return qb.select((eb) => + jsonFrom( + eb + .selectFrom(`${self.entity.name} as ${relationRef}`) + .select(self.entity.getSelect(relationRef)) + .whereRef(entityRef, "=", otherRef) + .limit(limit) + ).as(relationRef) + ); + } + + /** + * $set is performed using the reference: + * { [reference]: { $set: { id: 1 } } } + * + * It must resolve from [reference] ("users") to field ("user_id") + * -> returns instructions + */ + override async $set( + em: EntityManager, + key: string, + value: object + ): Promise { + if (typeof value !== "object") { + throw new Error(`Invalid value for relation field "${key}" given, expected object.`); + } + + const entity = this.source.entity; + const helper = this.helper(entity.name); + const info = helper.getMutationInfo(); + if (!info.$set) { + throw new Error(`Cannot perform $set for relation "${key}"`); + } + + const local_field = info.local_field; + const field = this.getField(); + // @ts-ignore + const primaryReference = value[Object.keys(value)[0]] as PrimaryFieldType; + + if (!local_field || !(field instanceof RelationField)) { + throw new Error(`Cannot perform $set for relation "${key}"`); + } + + // if "{ $set: { id: null } }" given, and not required, allow it + if (primaryReference === null && !field.isRequired()) { + return [local_field, null] satisfies MutationInstructionResponse; + } + + const query = await em.repository(field.target()).exists({ + [field.targetField()]: primaryReference as any + }); + + if (!query.exists) { + const idProp = field.targetField(); + throw new Error( + `Cannot connect "${entity.name}.${key}" to ` + + `"${field.target()}.${idProp}" = "${primaryReference}": not found.` + ); + } + + return [local_field, primaryReference] satisfies MutationInstructionResponse; + } +} diff --git a/app/src/data/relations/OneToOneRelation.ts b/app/src/data/relations/OneToOneRelation.ts new file mode 100644 index 0000000..e9cfa85 --- /dev/null +++ b/app/src/data/relations/OneToOneRelation.ts @@ -0,0 +1,77 @@ +import type { Entity, EntityManager } from "../entities"; +import { ManyToOneRelation, type ManyToOneRelationConfig } from "./ManyToOneRelation"; +import type { MutationInstructionResponse } from "./RelationMutator"; +import { type RelationType, RelationTypes } from "./relation-types"; + +/** + * Both source and target receive a mapping field + * @todo: determine if it should be removed + */ +export type OneToOneRelationConfig = ManyToOneRelationConfig; + +/* export type OneToOneRelationConfig = { + mappedBy?: string; // author|users + inversedBy?: string; // posts + required?: boolean; +}; */ + +export class OneToOneRelation extends ManyToOneRelation { + constructor(source: Entity, target: Entity, config?: OneToOneRelationConfig) { + const { mappedBy, inversedBy, required } = config || {}; + super(source, target, { + mappedBy, + inversedBy, + sourceCardinality: 1, + required + }); + } + + override type(): RelationType { + return RelationTypes.OneToOne; + } + + /** + * One-to-one relations are not listable in either direction + */ + override isListableFor(): boolean { + return false; + } + + // need to override since it inherits manytoone + override async $set( + em: EntityManager, + key: string, + value: object + ): Promise { + throw new Error("$set is not allowed"); + } + + override async $create( + em: EntityManager, + key: string, + value: unknown + ): Promise { + if (value === null || typeof value !== "object") { + throw new Error(`Invalid value for relation field "${key}" given, expected object.`); + } + + const target = this.other(this.source.entity).entity; + const helper = this.helper(this.source.entity.name); + const info = helper.getMutationInfo(); + const primary = info.primary; + const local_field = info.local_field; + if (!info.$create || !primary || !local_field) { + throw new Error(`Cannot perform $create for relation "${key}"`); + } + + // create the relational entity + try { + const { data } = await em.mutator(target).insertOne(value); + + const retrieved_value = data[primary]; + return [local_field, retrieved_value] satisfies MutationInstructionResponse; + } catch (e) { + throw new Error(`Error performing $create on "${target.name}".`); + } + } +} diff --git a/app/src/data/relations/PolymorphicRelation.ts b/app/src/data/relations/PolymorphicRelation.ts new file mode 100644 index 0000000..f7a359d --- /dev/null +++ b/app/src/data/relations/PolymorphicRelation.ts @@ -0,0 +1,130 @@ +import { type Static, Type } from "core/utils"; +import type { Entity, EntityManager } from "../entities"; +import { NumberField, TextField } from "../fields"; +import type { RepoQuery } from "../server/data-query-impl"; +import { EntityRelation, type KyselyJsonFrom, type KyselyQueryBuilder } from "./EntityRelation"; +import { EntityRelationAnchor } from "./EntityRelationAnchor"; +import { type RelationType, RelationTypes } from "./relation-types"; + +export type PolymorphicRelationConfig = Static; + +// @todo: what about cascades? +export class PolymorphicRelation extends EntityRelation { + static override schema = Type.Composite( + [ + EntityRelation.schema, + Type.Object({ + targetCardinality: Type.Optional(Type.Number()) + }) + ], + { + additionalProperties: false + } + ); + + constructor(source: Entity, target: Entity, config: Partial = {}) { + const mappedBy = config.mappedBy || target.name; + const inversedBy = config.inversedBy || source.name; + + // if target can be multiple, allow it. otherwise unlimited + const targetCardinality = + typeof config.targetCardinality === "number" && config.targetCardinality > 0 + ? config.targetCardinality + : undefined; + const sourceAnchor = new EntityRelationAnchor(source, inversedBy, 1); + const targetAnchor = new EntityRelationAnchor(target, mappedBy, targetCardinality); + super(sourceAnchor, targetAnchor, config); + + this.directions = ["source"]; + } + + type(): RelationType { + return RelationTypes.Polymorphic; + } + + private queryInfo(entity: Entity) { + const other = this.other(entity); + const whereLhs = `${other.entity.name}.${this.getReferenceField().name}`; + const reference = `${entity.name}.${this.config.mappedBy}`; + + // this is used for "getReferenceQuery" + const reference_other = `${other.entity.name}.${this.config.mappedBy}`; + + const entityRef = `${entity.name}.${entity.getPrimaryField().name}`; + const otherRef = `${other.entity.name}.${this.getEntityIdField().name}`; + + const groupBy = `${entity.name}.${entity.getPrimaryField().name}`; + + return { + other, + whereLhs, + reference, + reference_other, + entityRef, + otherRef, + groupBy + }; + } + + buildJoin(entity: Entity, qb: KyselyQueryBuilder) { + const { other, whereLhs, reference, entityRef, otherRef, groupBy } = this.queryInfo(entity); + + return qb + .innerJoin(other.entity.name, (join) => + join.onRef(entityRef, "=", otherRef).on(whereLhs, "=", reference) + ) + .groupBy(groupBy); + } + + override getReferenceQuery(entity: Entity, id: number): Partial { + const info = this.queryInfo(entity); + + return { + where: { + [this.getReferenceField().name]: info.reference_other, + [this.getEntityIdField().name]: id + } + }; + } + + buildWith(entity: Entity, qb: KyselyQueryBuilder, jsonFrom: KyselyJsonFrom) { + const { other, whereLhs, reference, entityRef, otherRef } = this.queryInfo(entity); + const limit = other.cardinality === 1 ? 1 : 5; + + return qb.select((eb) => + jsonFrom( + eb + .selectFrom(other.entity.name) + .select(other.entity.getSelect(other.entity.name)) + .where(whereLhs, "=", reference) + .whereRef(entityRef, "=", otherRef) + .limit(limit) + ).as(other.reference) + ); + } + + override isListableFor(entity: Entity): boolean { + // @todo: only make listable if many? check cardinality + return this.source.entity.name === entity.name && this.target.cardinality !== 1; + } + + getReferenceField(): TextField { + return new TextField("reference", { hidden: true, fillable: ["create"] }); + } + + getEntityIdField(): NumberField { + return new NumberField("entity_id", { hidden: true, fillable: ["create"] }); + } + + initialize(em: EntityManager) { + const referenceField = this.getReferenceField(); + const entityIdField = this.getEntityIdField(); + + if (!this.target.entity.field(referenceField.name)) { + this.target.entity.addField(referenceField); + } + if (!this.target.entity.field(entityIdField.name)) { + this.target.entity.addField(entityIdField); + } + } +} diff --git a/app/src/data/relations/RelationAccessor.ts b/app/src/data/relations/RelationAccessor.ts new file mode 100644 index 0000000..621d874 --- /dev/null +++ b/app/src/data/relations/RelationAccessor.ts @@ -0,0 +1,74 @@ +import type { Entity } from "../entities"; +import type { EntityRelation } from "../relations"; + +export class RelationAccessor { + private readonly _relations: EntityRelation[] = []; + + constructor(relations: EntityRelation[]) { + this._relations = relations; + } + + get all(): EntityRelation[] { + return this._relations; + } + + /** + * Searches for the relations of [entity_name] + */ + relationsOf(entity: Entity): EntityRelation[] { + return this._relations.filter((relation) => { + return ( + (relation.visibleFrom("source") && relation.source.entity.name === entity.name) || + (relation.visibleFrom("target") && relation.target.entity.name === entity.name) + ); + }); + } + + sourceRelationsOf(entity: Entity): EntityRelation[] { + return this._relations.filter((relation) => { + return relation.source.entity.name === entity.name; + }); + } + + /** + * Search for relations that have [entity] as target + * - meaning it returns entities that holds a local reference field + */ + targetRelationsOf(entity: Entity): EntityRelation[] { + return this._relations.filter((relation) => { + return relation.visibleFrom("target") && relation.target.entity.name === entity.name; + }); + } + + listableRelationsOf(entity: Entity): EntityRelation[] { + return this.relationsOf(entity).filter((relation) => relation.isListableFor(entity)); + } + + /** + * Searches for the relations of [entity_name] and + * return the one that has [reference] as source or target. + */ + relationOf(entity: Entity, reference: string): EntityRelation | undefined { + return this.relationsOf(entity).find((r) => { + return r.source.reference === reference || r.target.reference === reference; + }); + } + + hasRelations(entity: Entity): boolean { + return this.relationsOf(entity).length > 0; + } + + /** + * Get a list of related entities of [entity_name] + */ + relatedEntitiesOf(entity: Entity): Entity[] { + return this.relationsOf(entity).map((r) => r.other(entity).entity); + } + + /** + * Get relation names of [entity_name] + */ + relationReferencesOf(entity): string[] { + return this.relationsOf(entity).map((r) => r.other(entity).reference); + } +} diff --git a/app/src/data/relations/RelationField.ts b/app/src/data/relations/RelationField.ts new file mode 100644 index 0000000..f3d557f --- /dev/null +++ b/app/src/data/relations/RelationField.ts @@ -0,0 +1,101 @@ +import { type Static, StringEnum, Type } from "core/utils"; +import type { EntityManager } from "../entities"; +import { Field, type SchemaResponse, baseFieldConfigSchema } from "../fields"; +import type { EntityRelation } from "./EntityRelation"; +import type { EntityRelationAnchor } from "./EntityRelationAnchor"; + +const CASCADES = ["cascade", "set null", "set default", "restrict", "no action"] as const; + +export const relationFieldConfigSchema = Type.Composite([ + baseFieldConfigSchema, + Type.Object({ + reference: Type.String(), + target: Type.String(), // @todo: potentially has to be an instance! + target_field: Type.Optional(Type.String({ default: "id" })), + on_delete: Type.Optional(StringEnum(CASCADES, { default: "set null" })) + }) +]); +/*export const relationFieldConfigSchema = baseFieldConfigSchema.extend({ + reference: z.string(), + target: z.string(), + target_field: z.string().catch("id"), +});*/ + +export type RelationFieldConfig = Static; +export type RelationFieldBaseConfig = { label?: string }; + +export class RelationField extends Field { + override readonly type = "relation"; + + protected getSchema() { + return relationFieldConfigSchema; + } + + /*constructor(name: string, config?: Partial) { + //relation_name = relation_name || target.name; + //const name = [relation_name, target.getPrimaryField().name].join("_"); + super(name, config); + + //console.log(this.config); + //this.relation.target = target; + //this.relation.name = relation_name; + }*/ + + static create( + relation: EntityRelation, + target: EntityRelationAnchor, + config?: RelationFieldBaseConfig + ) { + const name = [ + target.reference ?? target.entity.name, + target.entity.getPrimaryField().name + ].join("_"); + //console.log('name', name); + return new RelationField(name, { + ...config, + required: relation.required, + reference: target.reference, + target: target.entity.name, + target_field: target.entity.getPrimaryField().name + }); + } + + reference() { + return this.config.reference; + } + + target() { + return this.config.target; + } + + targetField(): string { + return this.config.target_field!; + } + + override schema(): SchemaResponse { + return this.useSchemaHelper("integer", (col) => { + //col.references('person.id').onDelete('cascade').notNull() + // @todo: implement cascading? + + return col + .references(`${this.config.target}.${this.config.target_field}`) + .onDelete(this.config.on_delete ?? "set null"); + }); + } + + override transformRetrieve(value: any): any { + return value; + } + + override async transformPersist(value: any, em: EntityManager): Promise { + throw new Error("This function should not be called"); + } + + override toJsonSchema() { + return this.toSchemaWrapIfRequired( + Type.Number({ + $ref: `${this.config?.target}#/properties/${this.config?.target_field}` + }) + ); + } +} diff --git a/app/src/data/relations/RelationHelper.ts b/app/src/data/relations/RelationHelper.ts new file mode 100644 index 0000000..d78a840 --- /dev/null +++ b/app/src/data/relations/RelationHelper.ts @@ -0,0 +1,86 @@ +import { + type EntityRelation, + type EntityRelationAnchor, + type ManyToOneRelation, + type OneToOneRelation, + RelationTypes, +} from "../relations"; + +export const MutationOperations = ["$set", "$create", "$attach", "$detach"] as const; +export type MutationOperation = (typeof MutationOperations)[number]; + +export class RelationHelper { + relation: EntityRelation; + access: "source" | "target"; + self: EntityRelationAnchor; + other: EntityRelationAnchor; + + constructor(relation: EntityRelation, entity_name: string) { + this.relation = relation; + + if (relation.source.entity.name === entity_name) { + this.access = "source"; + this.self = relation.source; + this.other = relation.target; + } else if (relation.target.entity.name === entity_name) { + this.access = "target"; + this.self = relation.target; + this.other = relation.source; + } else { + throw new Error( + `Entity "${entity_name}" is not part of the relation ` + + `"${relation.source.entity.name} <-> ${relation.target.entity.name}"`, + ); + } + } + + // @todo: add to respective relations + getMutationInfo() { + const ops: Record = { + $set: false, + $create: false, + $attach: false, + $detach: false, + }; + + let local_field: string | undefined; + let primary: string | undefined; + + switch (this.relation.type()) { + case RelationTypes.ManyToOne: + // only if owning side (source), target is always single (just to assure) + if (typeof this.self.cardinality === "undefined" && this.other.cardinality === 1) { + ops.$set = true; + local_field = (this.relation as ManyToOneRelation).getField()?.name; + primary = this.other.entity.getPrimaryField().name; + } + + break; + case RelationTypes.OneToOne: + // only if owning side (source) + if (this.access === "source") { + ops.$create = true; + ops.$set = true; // @todo: for now allow + local_field = (this.relation as OneToOneRelation).getField()?.name; + primary = this.other.entity.getPrimaryField().name; + } + break; + case RelationTypes.ManyToMany: + if (this.access === "source") { + ops.$attach = true; + ops.$detach = true; + primary = this.other.entity.getPrimaryField().name; + } + break; + } + + return { + reference: this.other.reference, + local_field, + ...ops, + primary, + cardinality: this.other.cardinality, + relation_type: this.relation.type(), + }; + } +} diff --git a/app/src/data/relations/RelationMutator.ts b/app/src/data/relations/RelationMutator.ts new file mode 100644 index 0000000..3aa85b0 --- /dev/null +++ b/app/src/data/relations/RelationMutator.ts @@ -0,0 +1,121 @@ +import type { PrimaryFieldType } from "core"; +import type { Entity, EntityManager } from "../entities"; +import { + type EntityRelation, + type MutationOperation, + MutationOperations, + RelationField +} from "../relations"; + +export type MutationInstructionResponse = [string, PrimaryFieldType | null]; + +export class RelationMutator { + constructor( + protected entity: Entity, + protected em: EntityManager + ) {} + + /** + * Returns all keys that are somehow relational. + * Includes local fields (users_id) and references (users|author) + * + * @param em + * @param entity_name + * + * @returns string[] + */ + getRelationalKeys(): string[] { + const references: string[] = []; + this.em.relationsOf(this.entity.name).map((r) => { + const info = r.helper(this.entity.name).getMutationInfo(); + references.push(info.reference); + info.local_field && references.push(info.local_field); + }); + return references; + } + + async persistRelationField( + field: RelationField, + key: string, + value: PrimaryFieldType + ): Promise { + // allow empty if field is not required + if (value === null && !field.isRequired()) { + return [key, value]; + } + + // make sure it's a primitive value + // @todo: this is not a good way of checking primitives. Null is also an object + if (typeof value === "object") { + console.log("value", value); + throw new Error(`Invalid value for relation field "${key}" given, expected primitive.`); + } + + const query = await this.em.repository(field.target()).exists({ + [field.targetField()]: value + }); + + if (!query.exists) { + const idProp = field.targetField(); + throw new Error( + `Cannot connect "${this.entity.name}.${key}" to ` + + `"${field.target()}.${idProp}" = "${value}": not found.` + ); + } + + return [key, value]; + } + + async persistReference( + relation: EntityRelation, + key: string, + value: unknown + ): Promise { + if (typeof value !== "object" || value === null || typeof value === "undefined") { + throw new Error( + `Invalid value for relation "${key}" given, expected object to persist reference. Like '{$set: {id: 1}}'.` + ); + } + + const operation = Object.keys(value)[0] as MutationOperation; + if (!MutationOperations.includes(operation)) { + throw new Error( + `Invalid operation "${operation}" for relation "${key}". ` + + `Allowed: ${MutationOperations.join(", ")}` + ); + } + + // @ts-ignore + const payload = value[operation]; + return await relation[operation](this.em, key, payload); + } + + async persistRelation(key: string, value: unknown): Promise { + // if field (e.g. 'user_id') + // relation types: n:1, 1:1 (mapping entity) + const field = this.entity.getField(key); + if (field instanceof RelationField) { + return this.persistRelationField(field, key, value as PrimaryFieldType); + } + + /** + * If reference given, value operations are given + * + * Could be: + * { $set: { id: 1 } } + * { $set: [{ id: 1 }, { id: 2 }] } + * { $create: { theme: "dark" } } + * { $attach: [{ id: 1 }, { id: 2 }] } + * { $detach: [{ id: 1 }, { id: 2 }] } + */ + const relation = this.em.relationOf(this.entity.name, key); + if (relation) { + return this.persistReference(relation, key, value); + } + + throw new Error( + `Relation "${key}" failed to resolve on entity "${this.entity.name}": ` + + "Unable to resolve relation origin." + ); + } +} diff --git a/app/src/data/relations/index.ts b/app/src/data/relations/index.ts new file mode 100644 index 0000000..5988689 --- /dev/null +++ b/app/src/data/relations/index.ts @@ -0,0 +1,50 @@ +import { ManyToManyRelation, type ManyToManyRelationConfig } from "./ManyToManyRelation"; +import { ManyToOneRelation, type ManyToOneRelationConfig } from "./ManyToOneRelation"; +import { OneToOneRelation, type OneToOneRelationConfig } from "./OneToOneRelation"; +import { PolymorphicRelation, type PolymorphicRelationConfig } from "./PolymorphicRelation"; +import { type RelationType, RelationTypes } from "./relation-types"; + +export * from "./EntityRelation"; +export * from "./EntityRelationAnchor"; +export * from "./RelationHelper"; +export * from "./RelationMutator"; +export * from "./RelationAccessor"; + +import { + RelationField, + type RelationFieldBaseConfig, + type RelationFieldConfig, + relationFieldConfigSchema +} from "./RelationField"; + +export { + OneToOneRelation, + type OneToOneRelationConfig, + ManyToOneRelation, + type ManyToOneRelationConfig, + ManyToManyRelation, + type ManyToManyRelationConfig, + PolymorphicRelation, + type PolymorphicRelationConfig, + RelationTypes, + type RelationType, + // field + RelationField, + relationFieldConfigSchema, + type RelationFieldBaseConfig, + type RelationFieldConfig +}; + +export const RelationClassMap = { + [RelationTypes.OneToOne]: { schema: OneToOneRelation.schema, cls: OneToOneRelation }, + [RelationTypes.ManyToOne]: { schema: ManyToOneRelation.schema, cls: ManyToOneRelation }, + [RelationTypes.ManyToMany]: { schema: ManyToManyRelation.schema, cls: ManyToManyRelation }, + [RelationTypes.Polymorphic]: { + schema: PolymorphicRelation.schema, + cls: PolymorphicRelation + } +} as const; + +export const RelationFieldClassMap = { + relation: { schema: relationFieldConfigSchema, field: RelationField } +} as const; diff --git a/app/src/data/relations/relation-types.ts b/app/src/data/relations/relation-types.ts new file mode 100644 index 0000000..ff17cfc --- /dev/null +++ b/app/src/data/relations/relation-types.ts @@ -0,0 +1,7 @@ +export const RelationTypes = { + OneToOne: "1:1", + ManyToOne: "n:1", + ManyToMany: "m:n", + Polymorphic: "poly", +} as const; +export type RelationType = (typeof RelationTypes)[keyof typeof RelationTypes]; diff --git a/app/src/data/schema/SchemaManager.ts b/app/src/data/schema/SchemaManager.ts new file mode 100644 index 0000000..ef880bd --- /dev/null +++ b/app/src/data/schema/SchemaManager.ts @@ -0,0 +1,349 @@ +import type { AlterTableColumnAlteringBuilder, CompiledQuery, TableMetadata } from "kysely"; +import type { IndexMetadata } from "../connection/Connection"; +import type { Entity, EntityManager } from "../entities"; +import { PrimaryField, type SchemaResponse } from "../fields"; + +type IntrospectedTable = TableMetadata & { + indices: IndexMetadata[]; +}; + +type SchemaTable = { + name: string; + columns: string[]; +}; + +type SchemaDiffTable = { + name: string; + isNew: boolean; + isDrop?: boolean; + columns: { + add: string[]; + drop: string[]; + change: string[]; + }; + indices: { + add: string[]; + drop: string[]; + }; +}; + +type ColumnDiff = { + name: string; + changes: { + attribute: string; + prev: any; + next: any; + }[]; +}; + +/** + * @todo: add modified fields + * @todo: add drop tables + * + * @todo: change exclude tables to startWith, then add "bknd_" tables + */ + +export class SchemaManager { + static EXCLUDE_TABLES = ["libsql_wasm_func_table", "sqlite_sequence", "_cf_KV"]; + + constructor(private readonly em: EntityManager) {} + + private getIntrospector() { + if (!this.em.connection.supportsIndices()) { + throw new Error("Indices are not supported by the current connection"); + } + + return this.em.connection.getIntrospector(); + } + + async introspect(): Promise { + const tables = await this.getIntrospector().getTables({ + withInternalKyselyTables: false + }); + + const indices = await this.getIntrospector().getIndices(); + + const cleanTables: any[] = []; + for (const table of tables) { + if (SchemaManager.EXCLUDE_TABLES.includes(table.name)) { + continue; + } + + cleanTables.push({ + ...table, + indices: indices.filter((index) => index.table === table.name) + }); + } + + return cleanTables; + } + + getIntrospectionFromEntity(entity: Entity): IntrospectedTable { + const fields = entity.getFields(false); + const indices = this.em.getIndicesOf(entity); + + // this is intentionally setting values to defaults, like "nullable" and "default" + // that is because sqlite is the main focus, but in the future, + // we might want to support full sync with extensive schema updates (e.g. postgres) + return { + name: entity.name, + isView: false, + columns: fields.map((field) => ({ + name: field.name, + dataType: "TEXT", // doesn't matter + isNullable: true, // managed by the field + isAutoIncrementing: field instanceof PrimaryField, + hasDefaultValue: false, // managed by the field + comment: undefined + })), + indices: indices.map((index) => ({ + name: index.name, + table: entity.name, + isUnique: index.unique, + columns: index.fields.map((f) => ({ + name: f.name, + order: 0 // doesn't matter + })) + })) as any + }; + } + + async getDiff(): Promise { + const introspection = await this.introspect(); + const entityStates = this.em.entities.map((e) => this.getIntrospectionFromEntity(e)); + + const diff: SchemaDiffTable[] = []; + const namesFn = (c: { name: string }) => c.name; + + // @todo: add drop tables (beware, there a system tables!) + introspection + .filter((table) => { + if (/bknd/.test(table.name) || table.isView) { + return false; + } + return !entityStates.map((e) => e.name).includes(table.name); + }) + .forEach((t) => { + diff.push({ + name: t.name, + isDrop: true, + isNew: false, + columns: { + add: [], + drop: [], + change: [] + }, + indices: { + add: [], + drop: [] + } + }); + }); + + for (const entity of entityStates) { + const table = introspection.find((t) => t.name === entity.name); + + if (!table) { + // If the table is completely new + diff.push({ + name: entity.name, + isNew: true, + columns: { + add: entity.columns.map(namesFn), + drop: [], + change: [] + }, + indices: { + add: entity.indices.map(namesFn), + drop: [] + } + }); + } else { + // If the table exists, check for new columns + const newColumns = entity.columns.filter( + (newColumn) => !table.columns.map(namesFn).includes(newColumn.name) + ); + + // check for columns to drop + const dropColumns = table.columns.filter( + (oldColumn) => !entity.columns.map(namesFn).includes(oldColumn.name) + ); + + // check for changed columns + const columnDiffs: ColumnDiff[] = []; + for (const entity_col of entity.columns) { + const db_col = table.columns.find((c) => c.name === entity_col.name); + const col_diffs: ColumnDiff["changes"] = []; + for (const [key, value] of Object.entries(entity_col)) { + if (db_col && db_col[key] !== value) { + col_diffs.push({ + attribute: key, + prev: db_col[key], + next: value + }); + } + } + if (Object.keys(col_diffs).length > 0) { + columnDiffs.push({ + name: entity_col.name, + changes: col_diffs + }); + } + } + + // new indices + const newIndices = entity.indices.filter( + (newIndex) => !table.indices.map((i) => i.name).includes(newIndex.name) + ); + + const dropIndices = table.indices.filter( + (oldIndex) => !entity.indices.map((i) => i.name).includes(oldIndex.name) + ); + + const anythingChanged = [ + newColumns, + dropColumns, + //columnDiffs, // ignored + newIndices, + dropIndices + ].some((arr) => arr.length > 0); + + if (anythingChanged) { + diff.push({ + name: entity.name, + isNew: false, + columns: { + add: newColumns.map(namesFn), + drop: dropColumns.map(namesFn), + // @todo: this is ignored for now + //change: columnDiffs.map(namesFn), + change: [] + }, + indices: { + add: newIndices.map(namesFn), + drop: dropIndices.map(namesFn) + } + }); + } + } + } + + return diff; + } + + private collectFieldSchemas(table: string, columns: string[]) { + const schemas: SchemaResponse[] = []; + if (columns.length === 0) { + return schemas; + } + + for (const column of columns) { + const field = this.em.entity(table).getField(column)!; + const fieldSchema = field.schema(this.em); + if (Array.isArray(fieldSchema) && fieldSchema.length === 3) { + schemas.push(fieldSchema); + //throw new Error(`Field "${field.name}" on entity "${table}" has no schema`); + } + } + + return schemas; + } + + async sync(config: { force?: boolean; drop?: boolean } = { force: false, drop: false }) { + const diff = await this.getDiff(); + let updates: number = 0; + const statements: { sql: string; parameters: readonly unknown[] }[] = []; + const schema = this.em.connection.kysely.schema; + + for (const table of diff) { + const qbs: { compile(): CompiledQuery; execute(): Promise }[] = []; + let local_updates: number = 0; + const addFieldSchemas = this.collectFieldSchemas(table.name, table.columns.add); + const dropFields = table.columns.drop; + const dropIndices = table.indices.drop; + + if (table.isDrop) { + updates++; + local_updates++; + if (config.drop) { + qbs.push(schema.dropTable(table.name)); + } + } else if (table.isNew) { + let createQb = schema.createTable(table.name); + // add fields + for (const fieldSchema of addFieldSchemas) { + updates++; + local_updates++; + // @ts-ignore + createQb = createQb.addColumn(...fieldSchema); + } + + qbs.push(createQb); + } else { + // if fields to add + if (addFieldSchemas.length > 0) { + // add fields + for (const fieldSchema of addFieldSchemas) { + updates++; + local_updates++; + // @ts-ignore + qbs.push(schema.alterTable(table.name).addColumn(...fieldSchema)); + } + } + + // if fields to drop + if (config.drop && dropFields.length > 0) { + // drop fields + for (const column of dropFields) { + updates++; + local_updates++; + qbs.push(schema.alterTable(table.name).dropColumn(column)); + } + } + } + + // add indices + for (const index of table.indices.add) { + const indices = this.em.getIndicesOf(table.name); + const fieldIndex = indices.find((i) => i.name === index)!; + let qb = schema + .createIndex(index) + .on(table.name) + .columns(fieldIndex.fields.map((f) => f.name)); + if (fieldIndex.unique) { + qb = qb.unique(); + } + qbs.push(qb); + local_updates++; + updates++; + } + + // drop indices + if (config.drop) { + for (const index of dropIndices) { + qbs.push(schema.dropIndex(index)); + local_updates++; + updates++; + } + } + + if (local_updates === 0) continue; + + // iterate through built qbs + for (const qb of qbs) { + const { sql, parameters } = qb.compile(); + statements.push({ sql, parameters }); + + if (config.force) { + try { + await qb.execute(); + } catch (e) { + throw new Error(`Failed to execute query: ${sql}: ${(e as any).message}`); + } + } + } + } + + return statements; + } +} diff --git a/app/src/data/server/data-query-impl.ts b/app/src/data/server/data-query-impl.ts new file mode 100644 index 0000000..329a3cc --- /dev/null +++ b/app/src/data/server/data-query-impl.ts @@ -0,0 +1,77 @@ +import { + type SchemaOptions, + type Static, + type StaticDecode, + StringEnum, + Type, + Value +} from "core/utils"; +import type { Simplify } from "type-fest"; +import { WhereBuilder } from "../entities"; + +const NumberOrString = (options: SchemaOptions = {}) => + Type.Transform(Type.Union([Type.Number(), Type.String()], options)) + .Decode((value) => Number.parseInt(String(value))) + .Encode(String); + +const limit = NumberOrString({ default: 10 }); + +const offset = NumberOrString({ default: 0 }); + +// @todo: allow "id" and "-id" +const sort = Type.Transform( + Type.Union( + [Type.String(), Type.Object({ by: Type.String(), dir: StringEnum(["asc", "desc"]) })], + { + default: { by: "id", dir: "asc" } + } + ) +) + .Decode((value) => { + if (typeof value === "string") { + return JSON.parse(value); + } + return value; + }) + .Encode(JSON.stringify); + +const stringArray = Type.Transform( + Type.Union([Type.String(), Type.Array(Type.String())], { default: [] }) +) + .Decode((value) => { + if (Array.isArray(value)) { + return value; + } else if (value.includes(",")) { + return value.split(","); + } + return [value]; + }) + .Encode((value) => (Array.isArray(value) ? value : [value])); + +export const whereSchema = Type.Transform( + Type.Union([Type.String(), Type.Object({})], { default: {} }) +) + .Decode((value) => { + const q = typeof value === "string" ? JSON.parse(value) : value; + return WhereBuilder.convert(q); + }) + .Encode(JSON.stringify); + +export const querySchema = Type.Object( + { + limit: Type.Optional(limit), + offset: Type.Optional(offset), + sort: Type.Optional(sort), + select: Type.Optional(stringArray), + with: Type.Optional(stringArray), + join: Type.Optional(stringArray), + where: Type.Optional(whereSchema) + }, + { + additionalProperties: false + } +); + +export type RepoQueryIn = Simplify>; +export type RepoQuery = Required>; +export const defaultQuerySchema = Value.Default(querySchema, {}) as RepoQuery; diff --git a/app/src/data/server/query.ts b/app/src/data/server/query.ts new file mode 100644 index 0000000..ddc03db --- /dev/null +++ b/app/src/data/server/query.ts @@ -0,0 +1,112 @@ +import { z } from "zod"; + +const date = z.union([z.date(), z.string()]); +const numeric = z.union([z.number(), date]); +const boolean = z.union([z.boolean(), z.literal(1), z.literal(0)]); +const value = z.union([z.string(), boolean, numeric]); + +const expressionCond = z.union([ + z.object({ $eq: value }).strict(), + z.object({ $ne: value }).strict(), + z.object({ $isnull: boolean }).strict(), + z.object({ $notnull: boolean }).strict(), + z.object({ $in: z.array(value) }).strict(), + z.object({ $notin: z.array(value) }).strict(), + z.object({ $gt: numeric }).strict(), + z.object({ $gte: numeric }).strict(), + z.object({ $lt: numeric }).strict(), + z.object({ $lte: numeric }).strict(), + z.object({ $between: z.array(numeric).min(2).max(2) }).strict() +] as const); + +// prettier-ignore +const nonOperandString = z + .string() + .regex(/^(?!\$).*/) + .min(1); + +// {name: 'Michael'} +const literalCond = z.record(nonOperandString, value); + +// { status: { $eq: 1 } } +const literalExpressionCond = z.record(nonOperandString, value.or(expressionCond)); + +const operandCond = z + .object({ + $and: z.array(literalCond.or(expressionCond).or(literalExpressionCond)).optional(), + $or: z.array(literalCond.or(expressionCond).or(literalExpressionCond)).optional() + }) + .strict(); + +const literalSchema = literalCond.or(literalExpressionCond); +export type LiteralSchemaIn = z.input; +export type LiteralSchema = z.output; + +export const filterSchema = literalSchema.or(operandCond); +export type FilterSchemaIn = z.input; +export type FilterSchema = z.output; + +const stringArray = z + .union([ + z.string().transform((v) => { + if (v.includes(",")) return v.split(","); + return v; + }), + z.array(z.string()) + ]) + .default([]) + .transform((v) => (Array.isArray(v) ? v : [v])); + +export const whereRepoSchema = z + .preprocess((v: unknown) => { + try { + return JSON.parse(v as string); + } catch { + return v; + } + }, filterSchema) + .default({}); + +const repoQuerySchema = z.object({ + limit: z.coerce.number().default(10), + offset: z.coerce.number().default(0), + sort: z + .preprocess( + (v: unknown) => { + try { + return JSON.parse(v as string); + } catch { + return v; + } + }, + z.union([ + z.string().transform((v) => { + if (v.includes(":")) { + let [field, dir] = v.split(":") as [string, string]; + if (!["asc", "desc"].includes(dir)) dir = "asc"; + return { by: field, dir } as { by: string; dir: "asc" | "desc" }; + } else { + return { by: v, dir: "asc" } as { by: string; dir: "asc" | "desc" }; + } + }), + z.object({ + by: z.string(), + dir: z.enum(["asc", "desc"]) + }) + ]) + ) + .default({ by: "id", dir: "asc" }), + select: stringArray, + with: stringArray, + join: stringArray, + debug: z + .preprocess((v) => { + if (["0", "false"].includes(String(v))) return false; + return Boolean(v); + }, z.boolean()) + .default(false), //z.coerce.boolean().catch(false), + where: whereRepoSchema +}); + +type RepoQueryIn = z.input; +type RepoQuery = z.output; diff --git a/app/src/data/test-types.ts b/app/src/data/test-types.ts new file mode 100644 index 0000000..1c23fb9 --- /dev/null +++ b/app/src/data/test-types.ts @@ -0,0 +1,78 @@ +type Field = { + _type: Type; + _required: Required; +}; +type TextField = Field & { + _type: string; + required: () => TextField; +}; +type NumberField = Field & { + _type: number; + required: () => NumberField; +}; + +type Entity> = {}> = { name: string; fields: Fields }; + +function entity>>( + name: string, + fields: Fields, +): Entity { + return { name, fields }; +} + +function text(): TextField { + return {} as any; +} +function number(): NumberField { + return {} as any; +} + +const field1 = text(); +const field1_req = text().required(); +const field2 = number(); +const user = entity("users", { + name: text().required(), + bio: text(), + age: number(), + some: number().required(), +}); + +type InferEntityFields = T extends Entity + ? { + [K in keyof Fields]: Fields[K] extends { _type: infer Type; _required: infer Required } + ? Required extends true + ? Type + : Type | undefined + : never; + } + : never; + +type Prettify = { + [K in keyof T]: T[K]; +}; +export type Simplify = { [KeyType in keyof T]: T[KeyType] } & {}; + +// from https://github.com/type-challenges/type-challenges/issues/28200 +type Merge = { + [K in keyof T]: T[K]; +}; +type OptionalUndefined< + T, + Props extends keyof T = keyof T, + OptionsProps extends keyof T = Props extends keyof T + ? undefined extends T[Props] + ? Props + : never + : never, +> = Merge< + { + [K in OptionsProps]?: T[K]; + } & { + [K in Exclude]: T[K]; + } +>; + +type UserFields = InferEntityFields; +type UserFields2 = Simplify>; + +const obj: UserFields2 = { name: "h", age: 1, some: 1 }; diff --git a/app/src/flows/AppFlows.ts b/app/src/flows/AppFlows.ts new file mode 100644 index 0000000..34026e8 --- /dev/null +++ b/app/src/flows/AppFlows.ts @@ -0,0 +1,81 @@ +import { type Static, transformObject } from "core/utils"; +import { Flow, HttpTrigger } from "flows"; +import { Hono } from "hono"; +import { Module } from "modules/Module"; +import { TASKS, flowsConfigSchema } from "./flows-schema"; + +export type AppFlowsSchema = Static; +export type TAppFlowSchema = AppFlowsSchema["flows"][number]; +export type TAppFlowTriggerSchema = TAppFlowSchema["trigger"]; +export type { TAppFlowTaskSchema } from "./flows-schema"; + +export class AppFlows extends Module { + private flows: Record = {}; + + override async build() { + //console.log("building flows", this.config); + const flows = transformObject(this.config.flows, (flowConfig, name) => { + return Flow.fromObject(name, flowConfig as any, TASKS); + }); + + this.flows = flows; + + const hono = new Hono(); + + hono.get("/", async (c) => { + const flowsInfo = transformObject(this.flows, (flow) => this.getFlowInfo(flow)); + return c.json(flowsInfo); + }); + + hono.get("/flow/:name", async (c) => { + const name = c.req.param("name"); + return c.json(this.flows[name]?.toJSON()); + }); + + hono.get("/flow/:name/run", async (c) => { + const name = c.req.param("name"); + const flow = this.flows[name]!; + const execution = flow.createExecution(); + + const start = performance.now(); + await execution.start(); + const time = performance.now() - start; + const errors = execution.getErrors(); + return c.json({ + success: errors.length === 0, + time, + errors, + response: execution.getResponse(), + flow: this.getFlowInfo(flow), + logs: execution.logs + }); + }); + + this.ctx.server.route(this.config.basepath, hono); + + // register flows + for (const [name, flow] of Object.entries(this.flows)) { + const trigger = flow.trigger; + + switch (true) { + case trigger instanceof HttpTrigger: + await trigger.register(flow, this.ctx.server); + break; + } + } + + this.setBuilt(); + } + + getSchema() { + return flowsConfigSchema; + } + + private getFlowInfo(flow: Flow) { + return { + ...flow.toJSON(), + tasks: flow.tasks.length, + connections: flow.connections + }; + } +} diff --git a/app/src/flows/examples/simple-fetch.ts b/app/src/flows/examples/simple-fetch.ts new file mode 100644 index 0000000..3953141 --- /dev/null +++ b/app/src/flows/examples/simple-fetch.ts @@ -0,0 +1,20 @@ +import { Flow } from "../flows/Flow"; +import { FetchTask } from "../tasks/presets/FetchTask"; +import { LogTask } from "../tasks/presets/LogTask"; + +const first = new LogTask("First", { delay: 1000 }); +const second = new LogTask("Second", { delay: 1000 }); +const third = new LogTask("Long Third", { delay: 2500 }); +const fourth = new FetchTask("Fetch Something", { + url: "https://jsonplaceholder.typicode.com/todos/1", +}); +const fifth = new LogTask("Task 4", { delay: 500 }); // without connection + +const simpleFetch = new Flow("simpleFetch", [first, second, third, fourth, fifth]); +simpleFetch.task(first).asInputFor(second); +simpleFetch.task(first).asInputFor(third); +simpleFetch.task(fourth).asOutputFor(third); + +simpleFetch.setRespondingTask(fourth); + +export { simpleFetch }; diff --git a/app/src/flows/flows-schema.ts b/app/src/flows/flows-schema.ts new file mode 100644 index 0000000..4fc2c2a --- /dev/null +++ b/app/src/flows/flows-schema.ts @@ -0,0 +1,84 @@ +import { Const, type Static, StringRecord, Type, transformObject } from "core/utils"; +import { TaskMap, TriggerMap } from "flows"; + +export const TASKS = { + ...TaskMap +} as const; + +export const TRIGGERS = TriggerMap; + +const taskSchemaObject = transformObject(TASKS, (task, name) => { + return Type.Object( + { + type: Const(name), + params: task.cls.schema + }, + { title: String(name), additionalProperties: false } + ); +}); +const taskSchema = Type.Union(Object.values(taskSchemaObject)); +export type TAppFlowTaskSchema = Static; + +const triggerSchemaObject = transformObject(TRIGGERS, (trigger, name) => { + return Type.Object( + { + type: Const(name), + config: trigger.cls.schema + }, + { title: String(name), additionalProperties: false } + ); +}); + +const connectionSchema = Type.Object({ + source: Type.String(), + target: Type.String(), + config: Type.Object( + { + condition: Type.Optional( + Type.Union([ + Type.Object( + { type: Const("success") }, + { additionalProperties: false, title: "success" } + ), + Type.Object( + { type: Const("error") }, + { additionalProperties: false, title: "error" } + ), + Type.Object( + { type: Const("matches"), path: Type.String(), value: Type.String() }, + { additionalProperties: false, title: "matches" } + ) + ]) + ), + max_retries: Type.Optional(Type.Number()) + }, + { default: {}, additionalProperties: false } + ) +}); + +// @todo: rework to have fixed ids per task and connections (and preferrably arrays) +// causes issues with canvas +export const flowSchema = Type.Object( + { + trigger: Type.Union(Object.values(triggerSchemaObject)), + tasks: Type.Optional(StringRecord(Type.Union(Object.values(taskSchemaObject)))), + connections: Type.Optional(StringRecord(connectionSchema, { default: {} })), + start_task: Type.Optional(Type.String()), + responding_task: Type.Optional(Type.String()) + }, + { + additionalProperties: false + } +); +export type TAppFlowSchema = Static; + +export const flowsConfigSchema = Type.Object( + { + basepath: Type.String({ default: "/api/flows" }), + flows: StringRecord(flowSchema, { default: {} }) + }, + { + default: {}, + additionalProperties: false + } +); diff --git a/app/src/flows/flows/Execution.ts b/app/src/flows/flows/Execution.ts new file mode 100644 index 0000000..cf5cbc7 --- /dev/null +++ b/app/src/flows/flows/Execution.ts @@ -0,0 +1,248 @@ +import { Event, EventManager, type ListenerHandler } from "core/events"; +import type { EmitsEvents } from "core/events"; +import type { Task, TaskResult } from "../tasks/Task"; +import type { Flow } from "./Flow"; + +export type TaskLog = TaskResult & { + task: Task; + end: Date; +}; + +export type InputsMap = Map; + +export class ExecutionEvent extends Event<{ + task: Task; + result?: TaskResult; + end?: Date; +}> { + static override slug = "flow-execution-event"; + + task() { + return this.params.task; + } + + getState() { + if (this.succeeded()) return "success"; + if (this.failed()) return "failed"; + if (this.isStart()) return "running"; + return "idle"; + } + + isStart() { + return this.params.end === undefined; + } + + isEnd() { + return !this.isStart(); + } + + succeeded() { + return this.isEnd() && this.params.result?.success; + } + + failed() { + return this.isEnd() && !this.params.result?.success; + } +} + +export class ExecutionState extends Event<{ execution: Execution; state: "started" | "ended" }> { + static override slug = "flow-execution-state"; +} + +type UnionFromRecord = T[keyof T]; +type ExecutionEvents = UnionFromRecord; + +export class Execution implements EmitsEvents { + flow: Flow; + + started_at?: Date; + finished_at?: Date; + + logs: TaskLog[] = []; + inputs: InputsMap = new Map(); + + // next tasks to execute + protected queue: Task[] = []; + + emgr: EventManager; + + static Events = { ExecutionEvent, ExecutionState }; + + constructor(flow: Flow) { + this.flow = flow; + this.logs = []; + this.queue = [this.flow.startTask]; + this.emgr = new EventManager(Execution.Events); + } + + subscribe(handler: ListenerHandler) { + this.emgr.onAny(handler as any); + } + + async onDone(task: Task, result: TaskResult) { + //console.log("Execution: resolved", task.name, result.success); + + const end = new Date(); + this.logs.push({ ...result, task, end }); + this.inputs.set(task.name, result); + + // if responding task completed + if (this.flow.respondingTask === task) { + this.queue = []; + return; + } + + // clear task from queue + this.queue = this.queue.filter((t) => t !== task); + + // check outgoing tasks and add to queue if all in-tasks are finished + /*console.log( + "Out tasks that matches", + this.flow + .task(task) + .getOutConnections(result) + .map((c) => [c.target.name, c.max_retries]) + );*/ + const nextTasks = this.flow + .task(task) + .getOutConnections(result) + .filter((c) => { + const t = c.target; + // @todo: potentially filter on "end" instead of "success" + // @todo: behaves weird + const target_runs = this.logs.filter((log) => log.task === t && log.success).length; + + // max retry is set to the IN connection + const max_retries = + this.flow + .task(t) + .getInConnections() + .find((c) => c.source === task)?.max_retries ?? 0; + + /*console.log(`tried ${task.name}->${t.name}`, { + target_runs, + max_retries + });*/ + + if (target_runs > max_retries) { + //console.log("*** Task reached max retries", t.name); + throw new Error( + `Task "${t.name}" reached max retries (${target_runs}/${max_retries})` + ); + } + + /*console.log( + "tasks?", + this.flow + .task(t) + .getInTasks(true) + .map((t) => t.name) + );*/ + + return this.flow + .task(t) + .getInTasks(true) // only lower + .every((t) => this.logs.some((log) => log.task === t && log.end !== undefined)); + }) + .map((c) => c.target); + + /*console.log( + "--- next tasks", + nextTasks.map((t) => t.name) + );*/ + + //console.log("------"); + this.queue.push(...nextTasks); + //await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + __getLastTaskLog(task: Task) { + for (let i = this.logs.length - 1; i >= 0; i--) { + if (this.logs[i]?.task === task) { + return this.logs[i]; + } + } + + return null; + } + + private async run() { + const tasks = this.queue; + if (tasks.length === 0) { + return; + } + + //const promises = tasks.map((t) => t.run()); + const promises = tasks.map(async (t) => { + await this.emgr.emit(new ExecutionEvent({ task: t })); + const result = await t.run(this.inputs); + await this.emgr.emit(new ExecutionEvent({ task: t, result, end: new Date() })); + await this.onDone(t, result); + return result; + }); + + try { + await Promise.all(promises); + return this.run(); + } catch (e) { + console.log("RuntimeExecutor: error", e); + + // for now just throw + // biome-ignore lint/complexity/noUselessCatch: @todo: add error task on flow + throw e; + } + } + + async start(input?: any) { + await this.emgr.emit(new ExecutionState({ execution: this, state: "started" })); + + // set initial input + this.inputs.set("flow", { + start: new Date(), + output: input, // @todo: remove + error: undefined, + success: true, + params: input + }); + + //graceful && (await new Promise((resolve) => setTimeout(resolve, 100))); + this.started_at = new Date(); + await this.run(); + this.finished_at = new Date(); + await this.emgr.emit(new ExecutionState({ execution: this, state: "ended" })); + } + + finished(): boolean { + return this.finished_at !== undefined; + } + + errorCount(): number { + return this.logs.filter((log) => !log.success).length; + } + + hasErrors(): boolean { + return this.errorCount() > 0; + } + + getErrorLogs(): TaskLog[] { + return this.logs.filter((log) => !log.success); + } + + getErrors(): any[] { + return this.getErrorLogs().map((log) => log.error); + } + + getResponse() { + let respondingTask = this.flow.respondingTask; + if (!respondingTask) { + respondingTask = this.flow.tasks[this.flow.tasks.length - 1]; + } + + const lastLog = this.__getLastTaskLog(respondingTask!); + if (!lastLog) { + return; + } + + return lastLog.output; + } +} diff --git a/app/src/flows/flows/Flow.ts b/app/src/flows/flows/Flow.ts new file mode 100644 index 0000000..2890b7f --- /dev/null +++ b/app/src/flows/flows/Flow.ts @@ -0,0 +1,214 @@ +import { objectTransform, transformObject } from "core/utils"; +import { type TaskMapType, TriggerMap } from "../index"; +import type { Task } from "../tasks/Task"; +import { Condition, TaskConnection } from "../tasks/TaskConnection"; +import { Execution } from "./Execution"; +import { FlowTaskConnector } from "./FlowTaskConnector"; +import { Trigger } from "./triggers/Trigger"; + +type Jsoned object }> = ReturnType; + +export class Flow { + name: string; + + trigger: Trigger; + + /** + * The tasks that are part of the flow + */ + tasks: Task[] = []; + + /** + * The connections between tasks + */ + connections: TaskConnection[] = []; + + /** + * The task that should mark the flow response. + * If none given, then the flow has no response. + */ + respondingTask?: Task; + + startTask: Task; + + // sequence of tasks + sequence: Task[][]; + + constructor(name: string, tasks: Task[], connections?: TaskConnection[], trigger?: Trigger) { + this.name = name; + this.trigger = trigger ?? new Trigger(); + + tasks.map((t) => this.addTask(t)); + this.connections = connections || []; + + // defaulting to the first given + this.startTask = tasks[0]!; + this.sequence = this.getSequence(); + } + + setStartTask(task: Task) { + this.startTask = task; + this.sequence = this.getSequence(); + return this; + } + + getSequence(sequence: Task[][] = []): Task[][] { + //console.log("queue", queue.map((step) => step.map((t) => t.name))); + + // start task + if (sequence.length === 0) { + sequence.push([this.startTask]); + return this.getSequence(sequence); + } + + const tasks = sequence[sequence.length - 1]; + const nextStep: Task[] = []; + tasks?.forEach((task) => { + const outTasks = this.task(task).getOutTasks(); + outTasks.forEach((outTask) => { + // check if task already in one of queue steps + // this is when we have a circle back + if (sequence.some((step) => step.includes(outTask))) { + //console.log("Task already in queue", outTask.name); + return; + } + nextStep.push(outTask); + }); + }); + + // if no next steps, break out + if (nextStep.length === 0) { + return sequence; + } + + sequence.push(nextStep); + + return this.getSequence(sequence); + } + + addTask(task: Task) { + // check if task exists + if (this.tasks.includes(task)) { + throw new Error("Task already defined"); + } + if (this.tasks.some((t) => t.name === task.name)) { + throw new Error(`Task with name "${task.name}" already defined. Use a unique name.`); + } + + this.tasks.push(task); + + return this; + } + + setRespondingTask(task: Task) { + // check if task exists + if (!this.tasks.includes(task)) { + throw new Error(`Cannot set task "${task.name}" as responding, not registered.`); + } + + this.respondingTask = task; + return this; + } + + /*getResponse() { + if (!this.respondingTask) { + return; + } + + return this.respondingTask.log.output; + }*/ + + // @todo: check for existence + addConnection(connection: TaskConnection) { + // check if connection already exists + const exists = this.connections.some((c) => { + return ( + c.source === connection.source && + c.target === connection.target && + // @todo: should it check for condition at all? + c.condition[0] === connection.condition[0] && + c.condition[1] === connection.condition[1] + ); + }); + if (exists) { + throw new Error("Connection already defined"); + } + + this.connections.push(connection); + + return this; + } + + task(source: Task) { + return new FlowTaskConnector(this, source); + } + + createExecution() { + this.sequence = this.getSequence(); + return new Execution(this); + } + + /** + * Shorthand for creating and starting an execution + */ + async start(input: any = undefined) { + const execution = this.createExecution(); + await execution.start(input); + return execution; + } + + toJSON() { + return { + trigger: this.trigger.toJSON(), + tasks: Object.fromEntries(this.tasks.map((t) => [t.name, t.toJSON()])), + connections: Object.fromEntries(this.connections.map((c) => [c.id, c.toJSON()])), + start_task: this.startTask.name, + responding_task: this.respondingTask ? this.respondingTask.name : null + }; + } + + static fromObject(name: string, obj: Jsoned, taskMap: TaskMapType) { + const tasks = transformObject(obj.tasks ?? {}, (obj, name) => { + const taskClass = taskMap[obj.type]; + if (!taskClass) { + throw new Error(`Task ${name} not found in taskMap`); + } + + try { + const cls = taskClass.cls; + // @ts-ignore + return new cls(name, obj.params); + } catch (e: any) { + console.log("Error creating task", name, obj.type, obj, taskClass); + throw new Error(`Error creating task ${obj.type}: ${e.message}`); + } + }); + + const connections = transformObject(obj.connections ?? {}, (obj, id) => { + const condition = obj.config.condition + ? Condition.fromObject(obj.config.condition) + : undefined; + return new TaskConnection( + tasks[obj.source], + tasks[obj.target], + { ...obj.config, condition }, + id as string + ); + }); + + let trigger: Trigger | undefined; + if (obj.trigger) { + const cls = TriggerMap[obj.trigger.type as any]?.cls; + if (cls) { + trigger = new cls(obj.trigger.config); + } + } + + const flow = new Flow(name, Object.values(tasks), Object.values(connections), trigger); + flow.startTask = obj.start_task ? tasks[obj.start_task] : null; + if (obj.responding_task) { + flow.respondingTask = tasks[obj.responding_task]; + } + return flow; + } +} diff --git a/app/src/flows/flows/FlowTaskConnector.ts b/app/src/flows/flows/FlowTaskConnector.ts new file mode 100644 index 0000000..006ef2d --- /dev/null +++ b/app/src/flows/flows/FlowTaskConnector.ts @@ -0,0 +1,118 @@ +import type { Task, TaskResult } from "../tasks/Task"; +import { type Condition, TaskConnection } from "../tasks/TaskConnection"; +import type { Flow } from "./Flow"; + +// @todo: make singleton +export class FlowTaskConnector { + flow: Flow; + source: Task; + + constructor(flow: Flow, source: Task) { + this.flow = flow; + this.source = source; + } + + // helper function to use itself + private task(task: Task) { + return new FlowTaskConnector(this.flow, task); + } + + asInputFor(target: Task, condition?: Condition, max_retries?: number) { + const ownDepth = this.getDepth(); + const outConnections = this.getOutConnections(); + const definedOutConditions = outConnections.map((c) => c.condition); + const hasOutGoingBack = outConnections.some( + (c) => this.task(c.target).getDepth() <= ownDepth + ); + + if (definedOutConditions.length > 0 && hasOutGoingBack) { + if (this.getOutConnections().some((c) => c.condition.sameAs(condition))) { + throw new Error("Task cannot be connected to a deeper task with the same condition"); + } + } + + /*const targetDepth = this.task(target).getDepth(); + console.log("depth", ownDepth, targetDepth); + + // if target has a lower depth + if (targetDepth > 0 && ownDepth >= targetDepth) { + // check for unique out conditions + console.log( + "out conditions", + this.source.name, + this.getOutConnections().map((c) => [c.target.name, c.condition]) + ); + if ( + this.getOutConnections().some( + (c) => + c.condition[0] === condition[0] && + c.condition[1] === condition[1] + ) + ) { + throw new Error( + "Task cannot be connected to a deeper task with the same condition" + ); + } + }*/ + + this.flow.addConnection(new TaskConnection(this.source, target, { condition, max_retries })); + } + + asOutputFor(target: Task, condition?: Condition) { + this.task(target).asInputFor(this.source, condition); + //new FlowTaskConnector(this.flow, target).asInputFor(this.source); + //this.flow.addConnection(new TaskConnection(target, this.source)); + } + + getNext() { + return this.flow.connections.filter((c) => c.source === this.source).map((c) => c.target); + } + + getDepth(): number { + return this.flow.getSequence().findIndex((s) => s.includes(this.source)); + } + + getInConnections(lower_only: boolean = false): TaskConnection[] { + if (lower_only) { + const depth = this.getDepth(); + return this.getInConnections().filter( + (c) => c.target === this.source && this.task(c.source).getDepth() < depth + ); + } + + return this.flow.connections.filter((c) => c.target === this.source); + } + + getInTasks(lower_only: boolean = false): Task[] { + if (lower_only) { + const depth = this.getDepth(); + return this.getInConnections() + .map((c) => c.source) + .filter((t) => this.task(t).getDepth() < depth); + } + + return this.getInConnections().map((c) => c.source); + } + + getOutConnections(result?: TaskResult): TaskConnection[] { + if (result) { + return this.flow.connections.filter( + (c) => c.source === this.source && c.condition.isMet(result) + ); + } + + return this.flow.connections.filter((c) => c.source === this.source); + } + + getOutTasks(result?: TaskResult): Task[] { + return this.getOutConnections(result).map((c) => c.target); + } + + /*getNextRunnableConnections() { + return this.getOutConnections().filter((c) => c.source.log.success); + } + + getNextRunnableTasks() { + return this.getNextRunnableConnections().map((c) => c.target); + }*/ +} diff --git a/app/src/flows/flows/executors/RuntimeExecutor.ts b/app/src/flows/flows/executors/RuntimeExecutor.ts new file mode 100644 index 0000000..f1204ea --- /dev/null +++ b/app/src/flows/flows/executors/RuntimeExecutor.ts @@ -0,0 +1,28 @@ +import type { Task } from "../../tasks/Task"; + +export class RuntimeExecutor { + async run( + nextTasks: () => Task[], + onDone?: (task: Task, result: Awaited>) => void + ) { + const tasks = nextTasks(); + if (tasks.length === 0) { + return; + } + + //const promises = tasks.map((t) => t.run()); + const promises = tasks.map(async (t) => { + const result = await t.run(); + onDone?.(t, result); + return result; + }); + + try { + await Promise.all(promises); + } catch (e) { + console.log("RuntimeExecutor: error", e); + } + + return this.run(nextTasks, onDone); + } +} diff --git a/app/src/flows/flows/triggers/EventTrigger.ts b/app/src/flows/flows/triggers/EventTrigger.ts new file mode 100644 index 0000000..2fa194c --- /dev/null +++ b/app/src/flows/flows/triggers/EventTrigger.ts @@ -0,0 +1,41 @@ +import type { EventManager } from "core/events"; +import { Type } from "core/utils"; +import type { Flow } from "../Flow"; +import { Trigger } from "./Trigger"; + +export class EventTrigger extends Trigger { + override type = "event"; + + static override schema = Type.Composite([ + Trigger.schema, + Type.Object({ + event: Type.String() + // add match + }) + ]); + + override async register(flow: Flow, emgr: EventManager) { + if (!emgr.eventExists(this.config.event)) { + throw new Error(`Event ${this.config.event} is not registered.`); + } + + emgr.on( + this.config.event, + async (event) => { + console.log("event", event); + /*if (!this.match(event)) { + return; + }*/ + const execution = flow.createExecution(); + this.executions.push(execution); + + try { + await execution.start(event.params); + } catch (e) { + console.error(e); + } + }, + this.config.mode + ); + } +} diff --git a/app/src/flows/flows/triggers/HttpTrigger.ts b/app/src/flows/flows/triggers/HttpTrigger.ts new file mode 100644 index 0000000..9c91a9f --- /dev/null +++ b/app/src/flows/flows/triggers/HttpTrigger.ts @@ -0,0 +1,49 @@ +import { StringEnum, Type } from "core/utils"; +import type { Context, Hono } from "hono"; +import type { Flow } from "../Flow"; +import { Trigger } from "./Trigger"; + +const httpMethods = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const; + +export class HttpTrigger extends Trigger { + override type = "http"; + + static override schema = Type.Composite([ + Trigger.schema, + Type.Object( + { + path: Type.String({ pattern: "^/.*$" }), + method: StringEnum(httpMethods, { default: "GET" }), + response_type: StringEnum(["json", "text", "html"], { default: "json" }) + } + //{ additionalProperties: false } + ) + ]); + + override async register(flow: Flow, hono: Hono) { + const method = this.config.method.toLowerCase() as any; + + hono[method](this.config.path, async (c: Context) => { + const params = c.req.raw; + const respond = c[this.config.response_type] as any; + const execution = flow.createExecution(); + this.executions.push(execution); + + if (this.config.mode === "sync") { + await execution.start(params); + const response = execution.getResponse(); + const errors = execution.getErrors(); + if (errors.length > 0) { + return c.json({ success: false, errors }); + } + + return respond(response); + } + + execution.start(params); + return c.json({ success: true }); + }); + + //console.log("--registered flow", flow.name, "on", method, this.config.path); + } +} diff --git a/app/src/flows/flows/triggers/Trigger.ts b/app/src/flows/flows/triggers/Trigger.ts new file mode 100644 index 0000000..9b8afcc --- /dev/null +++ b/app/src/flows/flows/triggers/Trigger.ts @@ -0,0 +1,35 @@ +import { type Static, StringEnum, Type, parse } from "core/utils"; +import type { Execution } from "../Execution"; +import type { Flow } from "../Flow"; + +export class Trigger { + // @todo: remove this + executions: Execution[] = []; + type = "manual"; + config: Static; + + static schema = Type.Object( + { + mode: StringEnum(["sync", "async"], { default: "async" }) + } + //{ additionalProperties: false } + ); + + constructor(config?: Partial>) { + const schema = (this.constructor as typeof Trigger).schema; + // @ts-ignore for now + this.config = parse(schema, config ?? {}); + } + + async register(flow: Flow, ...args: any[]): Promise { + // @todo: remove this + this.executions.push(await flow.start()); + } + + toJSON() { + return { + type: this.type, + config: this.config + }; + } +} diff --git a/app/src/flows/flows/triggers/index.ts b/app/src/flows/flows/triggers/index.ts new file mode 100644 index 0000000..fa2529f --- /dev/null +++ b/app/src/flows/flows/triggers/index.ts @@ -0,0 +1,13 @@ +import { EventTrigger } from "./EventTrigger"; +import { HttpTrigger } from "./HttpTrigger"; +import { Trigger } from "./Trigger"; + +export { Trigger, EventTrigger, HttpTrigger }; + +//export type TriggerMapType = { [key: string]: { cls: typeof Trigger } }; +export const TriggerMap = { + manual: { cls: Trigger }, + event: { cls: EventTrigger }, + http: { cls: HttpTrigger } +} as const; +export type TriggerMapType = typeof TriggerMap; diff --git a/app/src/flows/index.ts b/app/src/flows/index.ts new file mode 100644 index 0000000..4a08dd9 --- /dev/null +++ b/app/src/flows/index.ts @@ -0,0 +1,41 @@ +import { FetchTask } from "./tasks/presets/FetchTask"; +import { LogTask } from "./tasks/presets/LogTask"; +import { RenderTask } from "./tasks/presets/RenderTask"; +import { SubFlowTask } from "./tasks/presets/SubFlowTask"; + +export { Flow } from "./flows/Flow"; +export { + Execution, + type TaskLog, + type InputsMap, + ExecutionState, + ExecutionEvent +} from "./flows/Execution"; +export { RuntimeExecutor } from "./flows/executors/RuntimeExecutor"; +export { FlowTaskConnector } from "./flows/FlowTaskConnector"; + +export { + Trigger, + EventTrigger, + HttpTrigger, + TriggerMap, + type TriggerMapType +} from "./flows/triggers"; + +import { Task } from "./tasks/Task"; +export { type TaskResult, type TaskRenderProps } from "./tasks/Task"; +export { TaskConnection, Condition } from "./tasks/TaskConnection"; + +// test +//export { simpleFetch } from "./examples/simple-fetch"; + +//export type TaskMapType = { [key: string]: { cls: typeof Task } }; +export const TaskMap = { + fetch: { cls: FetchTask }, + log: { cls: LogTask }, + render: { cls: RenderTask }, + subflow: { cls: SubFlowTask } +} as const; +export type TaskMapType = typeof TaskMap; + +export { Task, FetchTask, LogTask, RenderTask, SubFlowTask }; diff --git a/app/src/flows/tasks/Task.tsx b/app/src/flows/tasks/Task.tsx new file mode 100644 index 0000000..1ce4a5c --- /dev/null +++ b/app/src/flows/tasks/Task.tsx @@ -0,0 +1,235 @@ +import type { StaticDecode, TSchema } from "@sinclair/typebox"; +import type { NodeProps } from "@xyflow/react"; +import { BkndError, SimpleRenderer } from "core"; +import { type Static, type TObject, Type, Value, parse, ucFirst } from "core/utils"; +import type { ExecutionEvent, InputsMap } from "../flows/Execution"; +//type InstanceOf = T extends new (...args: any) => infer R ? R : never; + +export type TaskResult = { + start: Date; + output?: Output; + error?: any; + success: boolean; + params: any; +}; + +/*export type TaskRenderProps = NodeProps<{ + task: T; + state: { i: number; isStartTask: boolean; isRespondingTask; event: ExecutionEvent | undefined }; +}>;*/ +export type TaskRenderProps = any; + +export function dynamic( + type: Type, + parse?: (val: any | string) => Static +) { + const guessDecode = (val: unknown): Static => { + if (typeof val === "string") { + switch (type.type) { + case "object": + case "array": + return JSON.parse(val); + case "number": + return Number.parseInt(val); + case "boolean": + return val === "true" || val === "1"; + } + } + + return val as Static; + }; + + const decode = (val: unknown): Static => { + if (typeof val === "string") { + return parse ? parse(val) : guessDecode(val); + } + + return val as Static; + }; + const title = type.title ?? type.type ? ucFirst(type.type) : "Raw"; + + return ( + Type.Transform(Type.Union([{ title, ...type }, Type.String({ title: "Template" })])) + .Decode(decode) + // @ts-ignore + .Encode((val) => val) + ); +} + +export abstract class Task { + abstract type: string; + name: string; + + /** + * The schema of the task's parameters. + */ + static schema = Type.Object({}); + + /** + * The task's parameters. + */ + _params: Static; + + constructor(name: string, params?: Static) { + if (typeof name !== "string") { + throw new Error(`Task name must be a string, got ${typeof name}`); + } + + // @todo: should name be easier for object access? + + this.name = name; + + const schema = (this.constructor as typeof Task).schema; + + if ( + schema === Task.schema && + typeof params !== "undefined" && + Object.keys(params).length > 0 + ) { + throw new Error( + `Task "${name}" has no schema defined but params passed: ${JSON.stringify(params)}` + ); + } + + // @todo: string enums fail to validate + this._params = parse(schema, params || {}); + + /*const validator = new Validator(schema as any); + const _params = Default(schema, params || {}); + const result = validator.validate(_params); + if (!result.valid) { + //console.log("---errors", result, { params, _params }); + const error = result.errors[0]!; + throw new Error( + `Invalid params for task "${name}.${error.keyword}": "${ + error.error + }". Params given: ${JSON.stringify(params)}` + ); + } + + this._params = _params as Static;*/ + } + + get params() { + return this._params as StaticDecode; + } + + protected clone(name: string, params: Static): Task { + return new (this.constructor as any)(name, params); + } + + static async resolveParams( + schema: S, + params: any, + inputs: object = {} + ): Promise> { + const newParams: any = {}; + const renderer = new SimpleRenderer(inputs, { strictVariables: true, renderKeys: true }); + + //console.log("--resolveParams", params); + + for (const [key, value] of Object.entries(params)) { + if (value && SimpleRenderer.hasMarkup(value)) { + //console.log("--- has markup", value); + try { + newParams[key] = await renderer.render(value as string); + } catch (e: any) { + // wrap in bknd error for better error display + if (!(e instanceof BkndError)) { + throw new BkndError( + "Failed to resolve param", + { + key, + value, + error: e.message + }, + "resolve-params" + ); + } + + throw e; + } + continue; + } else { + //console.log("-- no markup", key, value); + } + + newParams[key] = value; + } + + //console.log("--beforeDecode", newParams); + const v = Value.Decode(schema, newParams); + //console.log("--afterDecode", v); + //process.exit(); + return v; + } + + private async cloneWithResolvedParams(_inputs: Map) { + const inputs = Object.fromEntries(_inputs.entries()); + //console.log("--clone:inputs", inputs, this.params); + const newParams = await Task.resolveParams( + (this.constructor as any).schema, + this._params, + inputs + ); + //console.log("--clone:newParams", this.name, newParams); + + return this.clone(this.name, newParams as any); + } + + /** + * The internal execution of the flow. + * Wraps the execute() function to gather log results. + */ + async run(inputs: InputsMap = new Map()) { + const start = new Date(); + let output: Output | undefined; + let error: any; + let success: boolean; + let params: any; + let time: number; + + const starttime = performance.now(); + + try { + // create a copy with resolved params + const newTask = await this.cloneWithResolvedParams(inputs); + params = newTask.params; + + output = (await newTask.execute(inputs)) as any; + success = true; + } catch (e: any) { + success = false; + //status.output = undefined; + + if (e instanceof BkndError) { + error = e.toJSON(); + } else { + error = { + type: "unknown", + message: (e as any).message + }; + } + } + + return { start, output, error, success, params, time: performance.now() - starttime }; + } + + protected error(message: string, details?: Record) { + return new BkndError(message, details, "runtime"); + } + + abstract execute(inputs: Map): Promise; + + // that's for react flow's default node + get label() { + return this.name; + } + + toJSON() { + return { + type: this.type, + params: this.params + }; + } +} diff --git a/app/src/flows/tasks/TaskConnection.ts b/app/src/flows/tasks/TaskConnection.ts new file mode 100644 index 0000000..1a4e579 --- /dev/null +++ b/app/src/flows/tasks/TaskConnection.ts @@ -0,0 +1,106 @@ +import { uuid } from "core/utils"; +import { get } from "lodash-es"; +import type { Task, TaskResult } from "./Task"; + +type TaskConnectionConfig = { + condition?: Condition; + max_retries?: number; +}; + +export class TaskConnection { + source: Task; + target: Task; + config: TaskConnectionConfig; + public id: string; + + constructor(source: Task, target: Task, config?: TaskConnectionConfig, id?: string) { + this.source = source; + this.target = target; + this.config = config ?? {}; + + if (!(this.config.condition instanceof Condition)) { + this.config.condition = Condition.default(); + } + + this.id = id ?? uuid(); + } + + get condition(): Condition { + return this.config.condition as any; + } + + get max_retries(): number { + return this.config.max_retries ?? 0; + } + + toJSON() { + return { + source: this.source.name, + target: this.target.name, + config: { + ...this.config, + condition: this.config.condition?.toJSON() + } + }; + } +} + +export class Condition { + private constructor( + public type: "success" | "error" | "matches", + public path: string = "", + public value: any = undefined + ) {} + + static default() { + return Condition.success(); + } + + static success() { + return new Condition("success"); + } + + static error() { + return new Condition("error"); + } + + static matches(path: string, value: any) { + if (typeof path !== "string" || path.length === 0) { + throw new Error("Invalid path"); + } + + return new Condition("matches", path, value); + } + + isMet(result: TaskResult) { + switch (this.type) { + case "success": + return result.success; + case "error": + return result.success === false; + case "matches": + return get(result.output, this.path) === this.value; + //return this.value === output[this.path]; + } + } + + sameAs(condition: Condition = Condition.default()) { + return ( + this.type === condition.type && + this.path === condition.path && + this.value === condition.value + ); + } + + toJSON() { + return { + type: this.type, + path: this.path.length === 0 ? undefined : this.path, + value: this.value + }; + } + + static fromObject(obj: ReturnType) { + return new Condition(obj.type, obj.path, obj.value); + } +} diff --git a/app/src/flows/tasks/presets/FetchTask.ts b/app/src/flows/tasks/presets/FetchTask.ts new file mode 100644 index 0000000..55671b4 --- /dev/null +++ b/app/src/flows/tasks/presets/FetchTask.ts @@ -0,0 +1,81 @@ +import { StringEnum, Type } from "core/utils"; +import type { InputsMap } from "../../flows/Execution"; +import { Task, dynamic } from "../Task"; + +const FetchMethods = ["GET", "POST", "PUT", "PATCH", "DELETE"]; + +export class FetchTask> extends Task< + typeof FetchTask.schema, + Output +> { + type = "fetch"; + + static override schema = Type.Object({ + url: Type.String({ + pattern: "^(http|https)://" + }), + //method: Type.Optional(Type.Enum(FetchMethodsEnum)), + //method: Type.Optional(dynamic(Type.String({ enum: FetchMethods, default: "GET" }))), + method: Type.Optional(dynamic(StringEnum(FetchMethods, { default: "GET" }))), + headers: Type.Optional( + dynamic( + Type.Array( + Type.Object({ + key: Type.String(), + value: Type.String() + }) + ), + JSON.parse + ) + ), + body: Type.Optional(dynamic(Type.String())), + normal: Type.Optional(dynamic(Type.Number(), Number.parseInt)) + }); + + protected getBody(): string | undefined { + const body = this.params.body; + if (!body) return; + if (typeof body === "string") return body; + if (typeof body === "object") return JSON.stringify(body); + + throw new Error(`Invalid body type: ${typeof body}`); + } + + async execute() { + //console.log(`method: (${this.params.method})`); + if (!FetchMethods.includes(this.params.method ?? "GET")) { + throw this.error("Invalid method", { + given: this.params.method, + valid: FetchMethods + }); + } + + const body = this.getBody(); + const headers = new Headers(this.params.headers?.map((h) => [h.key, h.value])); + + /*console.log("[FETCH]", { + url: this.params.url, + method: this.params.method ?? "GET", + headers, + body + });*/ + const result = await fetch(this.params.url, { + method: this.params.method ?? "GET", + headers, + body + }); + + //console.log("fetch:response", result); + if (!result.ok) { + throw this.error("Failed to fetch", { + status: result.status, + statusText: result.statusText + }); + } + + const data = (await result.json()) as Output; + //console.log("fetch:response:data", data); + + return data; + } +} diff --git a/app/src/flows/tasks/presets/LogTask.ts b/app/src/flows/tasks/presets/LogTask.ts new file mode 100644 index 0000000..e307ed6 --- /dev/null +++ b/app/src/flows/tasks/presets/LogTask.ts @@ -0,0 +1,16 @@ +import { Type } from "core/utils"; +import { Task } from "../Task"; + +export class LogTask extends Task { + type = "log"; + + static override schema = Type.Object({ + delay: Type.Number({ default: 10 }) + }); + + async execute() { + await new Promise((resolve) => setTimeout(resolve, this.params.delay)); + console.log(`[DONE] LogTask: ${this.name}`); + return true; + } +} diff --git a/app/src/flows/tasks/presets/RenderTask.ts b/app/src/flows/tasks/presets/RenderTask.ts new file mode 100644 index 0000000..b4a750f --- /dev/null +++ b/app/src/flows/tasks/presets/RenderTask.ts @@ -0,0 +1,17 @@ +import { Type } from "core/utils"; +import { Task } from "../Task"; + +export class RenderTask> extends Task< + typeof RenderTask.schema, + Output +> { + type = "render"; + + static override schema = Type.Object({ + render: Type.String() + }); + + async execute() { + return this.params.render as unknown as Output; + } +} diff --git a/app/src/flows/tasks/presets/SubFlowTask.ts b/app/src/flows/tasks/presets/SubFlowTask.ts new file mode 100644 index 0000000..8a0fc4c --- /dev/null +++ b/app/src/flows/tasks/presets/SubFlowTask.ts @@ -0,0 +1,40 @@ +import { Type } from "core/utils"; +import { Flow } from "../../flows/Flow"; +import { Task, dynamic } from "../Task"; + +export class SubFlowTask> extends Task< + typeof SubFlowTask.schema, + Output +> { + type = "subflow"; + + static override schema = Type.Object({ + flow: Type.Any(), + input: Type.Optional(dynamic(Type.Any(), JSON.parse)), + loop: Type.Optional(Type.Boolean()) + }); + + async execute() { + const flow = this.params.flow; + if (!(flow instanceof Flow)) { + throw new Error("Invalid flow provided"); + } + + if (this.params.loop) { + const _input = Array.isArray(this.params.input) ? this.params.input : [this.params.input]; + + const results: any[] = []; + for (const input of _input) { + const execution = flow.createExecution(); + await execution.start(input); + results.push(await execution.getResponse()); + } + + return results; + } + + const execution = flow.createExecution(); + await execution.start(this.params.input); + return execution.getResponse(); + } +} diff --git a/app/src/index.ts b/app/src/index.ts new file mode 100644 index 0000000..338bc7c --- /dev/null +++ b/app/src/index.ts @@ -0,0 +1,12 @@ +export { App, type AppConfig, type CreateAppConfig } from "./App"; + +export { MediaField } from "media/MediaField"; +export { + getDefaultConfig, + getDefaultSchema, + type ModuleConfigs, + type ModuleSchemas +} from "modules/ModuleManager"; + +export * from "./adapter"; +export { Api, type ApiOptions } from "./Api"; diff --git a/app/src/media/AppMedia.ts b/app/src/media/AppMedia.ts new file mode 100644 index 0000000..1e8e692 --- /dev/null +++ b/app/src/media/AppMedia.ts @@ -0,0 +1,171 @@ +import { EntityIndex, type EntityManager } from "data"; +import { type FileUploadedEventData, Storage, type StorageAdapter } from "media"; +import { Module } from "modules/Module"; +import { + type FieldSchema, + type InferFields, + type Schema, + boolean, + datetime, + entity, + json, + number, + text +} from "../data/prototype"; +import { MediaController } from "./api/MediaController"; +import { ADAPTERS, buildMediaSchema, type mediaConfigSchema, registry } from "./media-schema"; + +export type MediaFieldSchema = FieldSchema; +declare global { + interface DB { + media: MediaFieldSchema; + } +} + +export class AppMedia extends Module { + private _storage?: Storage; + + override async build() { + if (!this.config.enabled) { + this.setBuilt(); + return; + } + + if (!this.config.adapter) { + console.info("No storage adapter provided, skip building media."); + return; + } + + // build adapter + let adapter: StorageAdapter; + try { + const { type, config } = this.config.adapter; + adapter = new (registry.get(type as any).cls)(config as any); + + this._storage = new Storage(adapter, this.config.storage, this.ctx.emgr); + this.setBuilt(); + this.setupListeners(); + this.ctx.server.route(this.basepath, new MediaController(this).getController()); + + // @todo: add check for media entity + const mediaEntity = this.getMediaEntity(); + if (!this.ctx.em.hasEntity(mediaEntity)) { + this.ctx.em.addEntity(mediaEntity); + } + + const pathIndex = new EntityIndex(mediaEntity, [mediaEntity.field("path")!], true); + if (!this.ctx.em.hasIndex(pathIndex)) { + this.ctx.em.addIndex(pathIndex); + } + + // @todo: check indices + } catch (e) { + console.error(e); + throw new Error( + `Could not build adapter with config ${JSON.stringify(this.config.adapter)}` + ); + } + } + + getSchema() { + return buildMediaSchema(); + } + + get basepath() { + return this.config.basepath; + } + + get storage(): Storage { + this.throwIfNotBuilt(); + return this._storage!; + } + + uploadedEventDataToMediaPayload(info: FileUploadedEventData) { + return { + path: info.name, + mime_type: info.meta.type, + size: info.meta.size, + etag: info.etag, + modified_at: new Date() + }; + } + + static mediaFields = { + path: text().required(), + folder: boolean({ default_value: false, hidden: true, fillable: ["create"] }), + mime_type: text(), + size: number(), + scope: text({ hidden: true, fillable: ["create"] }), + etag: text(), + modified_at: datetime(), + reference: text(), + entity_id: number(), + metadata: json() + }; + + getMediaEntity() { + const entity_name = this.config.entity_name; + if (!this.em.hasEntity(entity_name)) { + return entity(entity_name, AppMedia.mediaFields, undefined, "system"); + } + + return this.em.entity(entity_name); + } + + get em(): EntityManager { + return this.ctx.em; + } + + private setupListeners() { + //const media = this._entity; + const { emgr, em } = this.ctx; + const media = this.getMediaEntity(); + + // when file is uploaded, sync with media entity + // @todo: need a way for singleton events! + emgr.onEvent( + Storage.Events.FileUploadedEvent, + async (e) => { + const mutator = em.mutator(media); + mutator.__unstable_toggleSystemEntityCreation(false); + await mutator.insertOne(this.uploadedEventDataToMediaPayload(e.params)); + mutator.__unstable_toggleSystemEntityCreation(true); + console.log("App:storage:file uploaded", e); + }, + "sync" + ); + + // when file is deleted, sync with media entity + emgr.onEvent( + Storage.Events.FileDeletedEvent, + async (e) => { + // simple file deletion sync + const item = await em.repo(media).findOne({ path: e.params.name }); + if (item.data) { + console.log("item.data", item.data); + await em.mutator(media).deleteOne(item.data.id); + } + + console.log("App:storage:file deleted", e); + }, + "sync" + ); + } + + override getOverwritePaths() { + // if using 'set' or mocked 'set' (patch), then "." is prepended + return [/^\.?adapter$/]; + } + + // @todo: add unit tests for toJSON! + override toJSON(secrets?: boolean) { + if (!this.isBuilt() || !this.config.enabled) { + return this.configDefault; + } + + return { + ...this.config, + adapter: this.storage.getAdapter().toJSON(secrets) + }; + } +} diff --git a/app/src/media/MediaField.ts b/app/src/media/MediaField.ts new file mode 100644 index 0000000..6dbb34f --- /dev/null +++ b/app/src/media/MediaField.ts @@ -0,0 +1,71 @@ +import { type Static, Type } from "core/utils"; +import { Field, baseFieldConfigSchema } from "data"; + +export const mediaFieldConfigSchema = Type.Composite([ + Type.Object({ + entity: Type.String(), // @todo: is this really required? + min_items: Type.Optional(Type.Number()), + max_items: Type.Optional(Type.Number()), + mime_types: Type.Optional(Type.Array(Type.String())) + }), + baseFieldConfigSchema +]); + +export type MediaFieldConfig = Static; + +export type MediaItem = { + id: number; + path: string; + mime_type: string; + size: number; + scope: number; + etag: string; + modified_at: Date; + folder: boolean; +}; + +export class MediaField< + Required extends true | false = false, + TypeOverride = MediaItem[] +> extends Field { + override readonly type = "media"; + + constructor(name: string, config: Partial) { + // field must be virtual, as it doesn't store a reference to the entity + super(name, { ...config, fillable: ["update"], virtual: true }); + } + + protected getSchema() { + return mediaFieldConfigSchema; + } + + getMaxItems(): number | undefined { + return this.config.max_items; + } + + getMinItems(): number | undefined { + return this.config.min_items; + } + + schema() { + return undefined; + } + + override toJsonSchema() { + // @todo: should be a variable, since media could be a diff entity + const $ref = "../schema.json#/properties/media"; + const minItems = this.config?.min_items; + const maxItems = this.config?.max_items; + + if (maxItems === 1) { + return { $ref } as any; + } else { + return { + type: "array", + items: { $ref }, + minItems, + maxItems + }; + } + } +} diff --git a/app/src/media/api/MediaApi.ts b/app/src/media/api/MediaApi.ts new file mode 100644 index 0000000..c69c952 --- /dev/null +++ b/app/src/media/api/MediaApi.ts @@ -0,0 +1,44 @@ +import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules/ModuleApi"; +import type { FileWithPath } from "ui/modules/media/components/dropzone/file-selector"; + +export type MediaApiOptions = BaseModuleApiOptions & {}; + +export class MediaApi extends ModuleApi { + protected override getDefaultOptions(): Partial { + return { + basepath: "/api/media" + }; + } + + async getFiles() { + return this.get(["files"]); + } + + async getFile(filename: string) { + return this.get(["file", filename]); + } + + getFileUploadUrl(file: FileWithPath): string { + return this.getUrl(`/upload/${file.path}`); + } + + getEntityUploadUrl(entity: string, id: PrimaryFieldType, field: string) { + return this.getUrl(`/entity/${entity}/${id}/${field}`); + } + + getUploadHeaders(): Headers { + return new Headers({ + Authorization: `Bearer ${this.options.token}` + }); + } + + async uploadFile(file: File) { + const formData = new FormData(); + formData.append("file", file); + return this.post(["upload"], formData); + } + + async deleteFile(filename: string) { + return this.delete(["file", filename]); + } +} diff --git a/app/src/media/api/MediaController.ts b/app/src/media/api/MediaController.ts new file mode 100644 index 0000000..59665fa --- /dev/null +++ b/app/src/media/api/MediaController.ts @@ -0,0 +1,193 @@ +import { type ClassController, tbValidator as tb } from "core"; +import { Type } from "core/utils"; +import { Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; +import type { StorageAdapter } from "media"; +import { StorageEvents } from "media"; +import { getRandomizedFilename } from "media"; +import type { AppMedia } from "../AppMedia"; +import { MediaField } from "../MediaField"; + +const booleanLike = Type.Transform(Type.String()) + .Decode((v) => v === "1") + .Encode((v) => (v ? "1" : "0")); + +export class MediaController implements ClassController { + constructor(private readonly media: AppMedia) {} + + private getStorageAdapter(): StorageAdapter { + return this.getStorage().getAdapter(); + } + + private getStorage() { + return this.media.storage; + } + + getController(): Hono { + // @todo: multiple providers? + // @todo: implement range requests + + const hono = new Hono(); + + // get files list (temporary) + hono.get("/files", async (c) => { + const files = await this.getStorageAdapter().listObjects(); + return c.json(files); + }); + + // get file by name + hono.get("/file/:filename", async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + //console.log("getting file", filename, headersToObject(c.req.raw.headers)); + + await this.getStorage().emgr.emit(new StorageEvents.FileAccessEvent({ name: filename })); + return await this.getStorageAdapter().getObject(filename, c.req.raw.headers); + }); + + // delete a file by name + hono.delete("/file/:filename", async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + await this.getStorage().deleteFile(filename); + + return c.json({ message: "File deleted" }); + }); + + const uploadSizeMiddleware = bodyLimit({ + maxSize: this.getStorage().getConfig().body_max_size, + onError: (c: any) => { + return c.text(`Payload exceeds ${this.getStorage().getConfig().body_max_size}`, 413); + } + }); + + // upload file + // @todo: add required type for "upload endpoints" + hono.post("/upload/:filename", uploadSizeMiddleware, async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + + const file = await this.getStorage().getFileFromRequest(c); + console.log("----file", file); + return c.json(await this.getStorage().uploadFile(file, filename)); + }); + + // add upload file to entity + // @todo: add required type for "upload endpoints" + hono.post( + "/entity/:entity/:id/:field", + tb( + "query", + Type.Object({ + overwrite: Type.Optional(booleanLike) + }) + ), + uploadSizeMiddleware, + async (c) => { + const entity_name = c.req.param("entity"); + const field_name = c.req.param("field"); + const entity_id = Number.parseInt(c.req.param("id")); + console.log("params", { entity_name, field_name, entity_id }); + + // check if entity exists + const entity = this.media.em.entity(entity_name); + if (!entity) { + return c.json({ error: `Entity "${entity_name}" not found` }, 404); + } + + // check if field exists and is of type MediaField + const field = entity.field(field_name); + if (!field || !(field instanceof MediaField)) { + return c.json({ error: `Invalid field "${field_name}"` }, 400); + } + + const mediaEntity = this.media.getMediaEntity(); + const reference = `${entity_name}.${field_name}`; + const mediaRef = { + scope: field_name, + reference, + entity_id: entity_id + }; + + // check max items + const max_items = field.getMaxItems(); + const ids_to_delete: number[] = []; + const id_field = mediaEntity.getPrimaryField().name; + if (max_items) { + const { overwrite } = c.req.valid("query"); + const { count } = await this.media.em.repository(mediaEntity).count(mediaRef); + + // if there are more than or equal to max items + if (count >= max_items) { + // if overwrite not set, abort early + if (!overwrite) { + return c.json({ error: `Max items (${max_items}) reached` }, 400); + } + + // if already more in database than allowed, abort early + // because we don't know if we can delete multiple items + if (count > max_items) { + return c.json( + { error: `Max items (${max_items}) exceeded already with ${count} items.` }, + 400 + ); + } + + // collect items to delete + const deleteRes = await this.media.em.repo(mediaEntity).findMany({ + select: [id_field], + where: mediaRef, + sort: { + by: id_field, + dir: "asc" + }, + limit: count - max_items + 1 + }); + + if (deleteRes.data && deleteRes.data.length > 0) { + deleteRes.data.map((item) => ids_to_delete.push(item[id_field])); + } + } + } + + // check if entity exists in database + const { exists } = await this.media.em.repository(entity).exists({ id: entity_id }); + if (!exists) { + return c.json( + { error: `Entity "${entity_name}" with ID "${entity_id}" doesn't exist found` }, + 404 + ); + } + + const file = await this.getStorage().getFileFromRequest(c); + const file_name = getRandomizedFilename(file as File); + const info = await this.getStorage().uploadFile(file, file_name, true); + + const mutator = this.media.em.mutator(mediaEntity); + mutator.__unstable_toggleSystemEntityCreation(false); + const result = await mutator.insertOne({ + ...this.media.uploadedEventDataToMediaPayload(info), + ...mediaRef + }); + mutator.__unstable_toggleSystemEntityCreation(true); + + // delete items if needed + if (ids_to_delete.length > 0) { + await this.media.em + .mutator(mediaEntity) + .deleteMany({ [id_field]: { $in: ids_to_delete } }); + } + + return c.json({ ok: true, result: result.data, ...info }); + } + ); + + return hono; + } +} diff --git a/app/src/media/index.ts b/app/src/media/index.ts new file mode 100644 index 0000000..bdc94c1 --- /dev/null +++ b/app/src/media/index.ts @@ -0,0 +1,54 @@ +import type { TObject, TString } from "@sinclair/typebox"; +import { type Constructor, Registry } from "core"; + +export { MIME_TYPES } from "./storage/mime-types"; +export { + Storage, + type StorageAdapter, + type FileMeta, + type FileListObject, + type StorageConfig +} from "./storage/Storage"; +import type { StorageAdapter } from "./storage/Storage"; +import { + type CloudinaryConfig, + StorageCloudinaryAdapter +} from "./storage/adapters/StorageCloudinaryAdapter"; +import { type S3AdapterConfig, StorageS3Adapter } from "./storage/adapters/StorageS3Adapter"; + +export { StorageS3Adapter, type S3AdapterConfig, StorageCloudinaryAdapter, type CloudinaryConfig }; +/*export { + StorageLocalAdapter, + type LocalAdapterConfig +} from "./storage/adapters/StorageLocalAdapter";*/ + +export * as StorageEvents from "./storage/events"; +export { type FileUploadedEventData } from "./storage/events"; +export * from "./utils"; + +type ClassThatImplements = Constructor & { prototype: T }; + +export const MediaAdapterRegistry = new Registry<{ + cls: ClassThatImplements; + schema: TObject; +}>().set({ + s3: { + cls: StorageS3Adapter, + schema: StorageS3Adapter.prototype.getSchema() + }, + cloudinary: { + cls: StorageCloudinaryAdapter, + schema: StorageCloudinaryAdapter.prototype.getSchema() + } +}); + +export const Adapters = { + s3: { + cls: StorageS3Adapter, + schema: StorageS3Adapter.prototype.getSchema() + }, + cloudinary: { + cls: StorageCloudinaryAdapter, + schema: StorageCloudinaryAdapter.prototype.getSchema() + } +} as const; diff --git a/app/src/media/media-schema.ts b/app/src/media/media-schema.ts new file mode 100644 index 0000000..045a0ca --- /dev/null +++ b/app/src/media/media-schema.ts @@ -0,0 +1,49 @@ +import { Const, Type, objectTransform } from "core/utils"; +import { Adapters } from "media"; +import { registries } from "modules/registries"; + +export const ADAPTERS = { + ...Adapters +} as const; + +export const registry = registries.media; + +export function buildMediaSchema() { + const adapterSchemaObject = objectTransform(registry.all(), (adapter, name) => { + return Type.Object( + { + type: Const(name), + config: adapter.schema + }, + { + title: name, + additionalProperties: false + } + ); + }); + const adapterSchema = Type.Union(Object.values(adapterSchemaObject)); + + return Type.Object( + { + enabled: Type.Boolean({ default: false }), + basepath: Type.String({ default: "/api/media" }), + entity_name: Type.String({ default: "media" }), + storage: Type.Object( + { + body_max_size: Type.Optional( + Type.Number({ + description: "Max size of the body in bytes. Leave blank for unlimited." + }) + ) + }, + { default: {} } + ), + adapter: Type.Optional(adapterSchema) + }, + { + additionalProperties: false + } + ); +} + +export const mediaConfigSchema = buildMediaSchema(); diff --git a/app/src/media/storage/Storage.ts b/app/src/media/storage/Storage.ts new file mode 100644 index 0000000..701319a --- /dev/null +++ b/app/src/media/storage/Storage.ts @@ -0,0 +1,228 @@ +import { type EmitsEvents, EventManager } from "core/events"; +import type { TSchema } from "core/utils"; +import { type Context, Hono } from "hono"; +import { bodyLimit } from "hono/body-limit"; +import * as StorageEvents from "./events"; +import type { FileUploadedEventData } from "./events"; + +export type FileListObject = { + key: string; + last_modified: Date; + size: number; +}; + +export type FileMeta = { type: string; size: number }; +export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File; +export type FileUploadPayload = { + name: string; + meta: FileMeta; + etag: string; +}; + +export interface StorageAdapter { + /** + * The unique name of the storage adapter + */ + getName(): string; + + // @todo: method requires limit/offset parameters + listObjects(prefix?: string): Promise; + putObject(key: string, body: FileBody): Promise; + deleteObject(key: string): Promise; + objectExists(key: string): Promise; + getObject(key: string, headers: Headers): Promise; + getObjectUrl(key: string): string; + getObjectMeta(key: string): Promise; + getSchema(): TSchema | undefined; + toJSON(secrets?: boolean): any; +} + +export type StorageConfig = { + body_max_size: number; +}; + +export class Storage implements EmitsEvents { + readonly #adapter: StorageAdapter; + static readonly Events = StorageEvents; + readonly emgr: EventManager; + readonly config: StorageConfig; + + constructor( + adapter: StorageAdapter, + config: Partial = {}, + emgr?: EventManager + ) { + this.#adapter = adapter; + this.config = { + ...config, + body_max_size: config.body_max_size ?? 20 * 1024 * 1024 + }; + + this.emgr = emgr ?? new EventManager(); + this.emgr.registerEvents(Storage.Events); + } + + getAdapter(): StorageAdapter { + return this.#adapter; + } + + async objectMetadata(key: string): Promise { + return await this.#adapter.getObjectMeta(key); + } + + //randomizeFilename(filename: string): string {} + + getConfig(): StorageConfig { + return this.config; + } + + async uploadFile( + file: FileBody, + name: string, + noEmit?: boolean + ): Promise { + const result = await this.#adapter.putObject(name, file); + console.log("result", result); + + let info: FileUploadPayload; + + switch (typeof result) { + case "undefined": + throw new Error("Failed to upload file"); + case "string": { + // get object meta + const meta = await this.#adapter.getObjectMeta(name); + if (!meta) { + throw new Error("Failed to get object meta"); + } + + info = { name, meta, etag: result }; + break; + } + case "object": + info = result; + break; + } + + const eventData = { + file, + ...info, + state: { + name: info.name, + path: info.name + } + }; + if (!noEmit) { + await this.emgr.emit(new StorageEvents.FileUploadedEvent(eventData)); + } + + return eventData; + } + + async deleteFile(name: string): Promise { + await this.#adapter.deleteObject(name); + await this.emgr.emit(new StorageEvents.FileDeletedEvent({ name })); + } + + async fileExists(name: string) { + return await this.#adapter.objectExists(name); + } + + getController(): any { + // @todo: multiple providers? + // @todo: implement range requests + + const hono = new Hono(); + + // get files list (temporary) + hono.get("/files", async (c) => { + const files = await this.#adapter.listObjects(); + return c.json(files); + }); + + // get file by name + hono.get("/file/:filename", async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + //console.log("getting file", filename, headersToObject(c.req.raw.headers)); + + await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename })); + return await this.#adapter.getObject(filename, c.req.raw.headers); + }); + + // delete a file by name + hono.delete("/file/:filename", async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + await this.deleteFile(filename); + + return c.json({ message: "File deleted" }); + }); + + // upload file + hono.post( + "/upload/:filename", + bodyLimit({ + maxSize: this.config.body_max_size, + onError: (c: any) => { + return c.text(`Payload exceeds ${this.config.body_max_size}`, 413); + } + }), + async (c) => { + const { filename } = c.req.param(); + if (!filename) { + throw new Error("No file name provided"); + } + + const file = await this.getFileFromRequest(c); + return c.json(await this.uploadFile(file, filename)); + } + ); + + return hono; + } + + /** + * If uploaded through HttpPie -> ReadableStream + * If uploaded in tests -> file == ReadableStream + * If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn + * file File { + * size: 223052, + * type: 'image/png', + * name: 'noise_white.png', + * lastModified: 1731743671176 + * } + * @param c + */ + async getFileFromRequest(c: Context): Promise { + const content_type = c.req.header("Content-Type") ?? "application/octet-stream"; + console.log("content_type:body", content_type); + const body = c.req.raw.body; + if (!body) { + throw new Error("No body"); + } + + let file: FileBody | undefined; + if (content_type?.startsWith("multipart/form-data")) { + file = (await c.req.formData()).get("file") as File; + // @todo: check nextjs, it's not *that* [File] type (but it's uploadable) + if (typeof file === "undefined") { + throw new Error("No file given at form data 'file'"); + } + /*console.log("file", file); + if (!(file instanceof File)) { + throw new Error("No file given at form data 'file'"); + }*/ + } else if (content_type?.startsWith("application/octet-stream")) { + file = body; + } else { + throw new Error(`Unsupported content type: ${content_type}`); + } + + return file; + } +} diff --git a/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts b/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts new file mode 100644 index 0000000..771b389 --- /dev/null +++ b/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts @@ -0,0 +1,256 @@ +import { pickHeaders } from "core/utils"; +import { type Static, Type, parse } from "core/utils"; +import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../Storage"; + +export const cloudinaryAdapterConfig = Type.Object( + { + cloud_name: Type.String(), + api_key: Type.String(), + api_secret: Type.String(), + upload_preset: Type.Optional(Type.String()) + }, + { title: "Cloudinary" } +); + +export type CloudinaryConfig = Static; +/*export type CloudinaryConfig = { + cloud_name: string; + api_key: string; + api_secret: string; + upload_preset?: string; +};*/ + +type CloudinaryObject = { + asset_id: string; + public_id: string; + version: number; + version_id: string; + signature: string; + width?: number; + height?: number; + format: string; + resource_type: string; + created_at: string; // date format + tags: string[]; + bytes: number; + type: string; // "upload" ? + etag: string; + placeholder: boolean; + url: string; + secure_url: string; + folder: string; + existing: boolean; + original_filename: string; +}; + +type CloudinaryPutObjectResponse = CloudinaryObject; +type CloudinaryListObjectsResponse = { + total_count: number; + time: number; + next_cursor: string; + resources: (CloudinaryObject & { + uploaded_at: string; // date format + backup_bytes: number; + aspect_ratio?: number; + pixels?: number; + status: string; + access_mode: string; + })[]; +}; + +// @todo: add signed uploads +export class StorageCloudinaryAdapter implements StorageAdapter { + private config: CloudinaryConfig; + + constructor(config: CloudinaryConfig) { + this.config = parse(cloudinaryAdapterConfig, config); + } + + getSchema() { + return cloudinaryAdapterConfig; + } + + private getMimeType(object: CloudinaryObject): string { + switch (true) { + case object.format === "jpeg" || object.format === "jpg": + return "image/jpeg"; + } + + return `${object.resource_type}/${object.format}`; + } + + getName(): string { + return "cloudinary"; + } + + private getAuthorizationHeader() { + const credentials = btoa(`${this.config.api_key}:${this.config.api_secret}`); + return { + Authorization: `Basic ${credentials}` + }; + } + + async putObject(_key: string, body: FileBody) { + //console.log("_key", _key); + // remove extension, as it is added by cloudinary + const key = _key.replace(/\.[a-z0-9]{2,5}$/, ""); + //console.log("key", key); + + const formData = new FormData(); + formData.append("file", body as any); + formData.append("public_id", key); + formData.append("api_key", this.config.api_key); + + if (this.config.upload_preset) { + formData.append("upload_preset", this.config.upload_preset); + } + + const result = await fetch( + `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/auto/upload`, + { + method: "POST", + headers: { + Accept: "application/json" + // content type must be undefined to use correct boundaries + //"Content-Type": "multipart/form-data", + }, + body: formData + } + ); + //console.log("putObject:cloudinary", formData); + + if (!result.ok) { + /*console.log( + "failed to upload using cloudinary", + Object.fromEntries(formData.entries()), + result + );*/ + return undefined; + } + + //console.log("putObject:result", result); + + const data = (await result.json()) as CloudinaryPutObjectResponse; + //console.log("putObject:result:json", data); + + return { + name: data.public_id + "." + data.format, + etag: data.etag, + meta: { + type: this.getMimeType(data), + size: data.bytes + } + }; + } + + async listObjects(prefix?: string): Promise { + const result = await fetch( + `https://api.cloudinary.com/v1_1/${this.config.cloud_name}/resources/search`, + { + method: "GET", + headers: { + Accept: "application/json", + ...this.getAuthorizationHeader() + } + } + ); + //console.log("result", result); + + if (!result.ok) { + throw new Error("Failed to list objects"); + } + + const data = (await result.json()) as CloudinaryListObjectsResponse; + return data.resources.map((item) => ({ + key: item.public_id, + last_modified: new Date(item.uploaded_at), + size: item.bytes + })); + } + + private async headObject(key: string) { + const url = this.getObjectUrl(key); + return await fetch(url, { + method: "GET", + headers: { + Range: "bytes=0-1" + } + }); + } + + async objectExists(key: string): Promise { + //console.log("--object exists?", key); + const result = await this.headObject(key); + //console.log("object exists", result); + + return result.ok; + } + + async getObjectMeta(key: string): Promise { + const result = await this.headObject(key); + if (result.ok) { + const type = result.headers.get("content-type"); + const size = Number(result.headers.get("content-range")?.split("/")[1]); + return { + type: type as string, + size: size + }; + } + + throw new Error("Cannot get object meta"); + } + + private guessType(key: string): string | undefined { + const extensions = { + image: ["jpg", "jpeg", "png", "gif", "webp", "svg"], + video: ["mp4", "webm", "ogg"] + }; + + const ext = key.split(".").pop(); + return Object.keys(extensions).find((type) => extensions[type].includes(ext)); + } + + getObjectUrl(key: string): string { + const type = this.guessType(key) ?? "image"; + + const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`; + //console.log("objectUrl", objectUrl); + return objectUrl; + } + + async getObject(key: string, headers: Headers): Promise { + //console.log("url", this.getObjectUrl(key)); + const res = await fetch(this.getObjectUrl(key), { + method: "GET", + headers: pickHeaders(headers, ["range"]) + }); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: res.headers + }); + } + + async deleteObject(key: string): Promise { + const type = this.guessType(key) ?? "image"; + const formData = new FormData(); + formData.append("public_ids[]", key); + + const result = await fetch( + `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, + { + method: "DELETE", + body: formData + } + ); + //console.log("deleteObject:result", result); + } + + toJSON(secrets?: boolean) { + return { + type: "cloudinary", + config: secrets ? this.config : { cloud_name: this.config.cloud_name } + }; + } +} diff --git a/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts b/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts new file mode 100644 index 0000000..f6c1bb1 --- /dev/null +++ b/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts @@ -0,0 +1,125 @@ +import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises"; +import { type Static, Type, parse } from "core/utils"; +import type { + FileBody, + FileListObject, + FileMeta, + FileUploadPayload, + StorageAdapter +} from "../../Storage"; +import { guessMimeType } from "../../mime-types"; + +export const localAdapterConfig = Type.Object( + { + path: Type.String() + }, + { title: "Local" } +); +export type LocalAdapterConfig = Static; + +export class StorageLocalAdapter implements StorageAdapter { + private config: LocalAdapterConfig; + + constructor(config: any) { + this.config = parse(localAdapterConfig, config); + } + + getSchema() { + return localAdapterConfig; + } + + getName(): string { + return "local"; + } + + async listObjects(prefix?: string): Promise { + const files = await readdir(this.config.path); + const fileStats = await Promise.all( + files + .filter((file) => !prefix || file.startsWith(prefix)) + .map(async (file) => { + const stats = await stat(`${this.config.path}/${file}`); + return { + key: file, + last_modified: stats.mtime, + size: stats.size + }; + }) + ); + return fileStats; + } + + private async computeEtag(content: BufferSource): Promise { + const hashBuffer = await crypto.subtle.digest("SHA-256", content); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join(""); + + // Wrap the hex string in quotes for ETag format + return `"${hashHex}"`; + } + + async putObject(key: string, body: FileBody): Promise { + if (body === null) { + throw new Error("Body is empty"); + } + + // @todo: this is too hacky + const file = body as File; + + const filePath = `${this.config.path}/${key}`; + await writeFile(filePath, file.stream()); + return await this.computeEtag(await file.arrayBuffer()); + } + + async deleteObject(key: string): Promise { + try { + await unlink(`${this.config.path}/${key}`); + } catch (e) {} + } + + async objectExists(key: string): Promise { + try { + const stats = await stat(`${this.config.path}/${key}`); + return stats.isFile(); + } catch (error) { + return false; + } + } + + async getObject(key: string, headers: Headers): Promise { + try { + const content = await readFile(`${this.config.path}/${key}`); + const mimeType = guessMimeType(key); + + return new Response(content, { + status: 200, + headers: { + "Content-Type": mimeType || "application/octet-stream", + "Content-Length": content.length.toString() + } + }); + } catch (error) { + // Handle file reading errors + return new Response("", { status: 404 }); + } + } + + getObjectUrl(key: string): string { + throw new Error("Method not implemented."); + } + + async getObjectMeta(key: string): Promise { + const stats = await stat(`${this.config.path}/${key}`); + return { + type: guessMimeType(key) || "application/octet-stream", + size: stats.size + }; + } + + toJSON(secrets?: boolean) { + return { + type: this.getName(), + config: this.config + }; + } +} diff --git a/app/src/media/storage/adapters/StorageLocalAdapter/index.ts b/app/src/media/storage/adapters/StorageLocalAdapter/index.ts new file mode 100644 index 0000000..6fc7e70 --- /dev/null +++ b/app/src/media/storage/adapters/StorageLocalAdapter/index.ts @@ -0,0 +1,5 @@ +export { + StorageLocalAdapter, + type LocalAdapterConfig, + localAdapterConfig +} from "./StorageLocalAdapter"; diff --git a/app/src/media/storage/adapters/StorageR2Adapter.ts b/app/src/media/storage/adapters/StorageR2Adapter.ts new file mode 100644 index 0000000..911fe37 --- /dev/null +++ b/app/src/media/storage/adapters/StorageR2Adapter.ts @@ -0,0 +1,137 @@ +import { isDebug } from "core"; +import type { FileBody, StorageAdapter } from "../Storage"; +import { guessMimeType } from "../mime-types"; + +/** + * Adapter for R2 storage + * @todo: add tests (bun tests won't work, need node native tests) + */ +export class StorageR2Adapter implements StorageAdapter { + constructor(private readonly bucket: R2Bucket) {} + + getName(): string { + return "r2"; + } + + getSchema() { + return undefined; + } + + async putObject(key: string, body: FileBody) { + try { + const res = await this.bucket.put(key, body); + return res?.etag; + } catch (e) { + return undefined; + } + } + async listObjects( + prefix?: string + ): Promise<{ key: string; last_modified: Date; size: number }[]> { + const list = await this.bucket.list({ limit: 50 }); + return list.objects.map((item) => ({ + key: item.key, + size: item.size, + last_modified: item.uploaded + })); + } + + private async headObject(key: string): Promise { + return await this.bucket.head(key); + } + + async objectExists(key: string): Promise { + return (await this.headObject(key)) !== null; + } + + async getObject(key: string, headers: Headers): Promise { + let object: R2ObjectBody | null; + const responseHeaders = new Headers({ + "Accept-Ranges": "bytes" + }); + + //console.log("getObject:headers", headersToObject(headers)); + if (headers.has("range")) { + const options = isDebug() + ? {} // miniflare doesn't support range requests + : { + range: headers, + onlyIf: headers + }; + object = (await this.bucket.get(key, options)) as R2ObjectBody; + + if (!object) { + return new Response(null, { status: 404 }); + } + + if (object.range) { + const offset = "offset" in object.range ? object.range.offset : 0; + const end = "end" in object.range ? object.range.end : object.size - 1; + responseHeaders.set("Content-Range", `bytes ${offset}-${end}/${object.size}`); + responseHeaders.set("Connection", "keep-alive"); + responseHeaders.set("Vary", "Accept-Encoding"); + } + } else { + object = (await this.bucket.get(key)) as R2ObjectBody; + + if (object === null) { + return new Response(null, { status: 404 }); + } + } + + //console.log("response headers:before", headersToObject(responseHeaders)); + this.writeHttpMetadata(responseHeaders, object); + responseHeaders.set("etag", object.httpEtag); + responseHeaders.set("Content-Length", String(object.size)); + responseHeaders.set("Last-Modified", object.uploaded.toUTCString()); + //console.log("response headers:after", headersToObject(responseHeaders)); + + return new Response(object.body, { + status: object.range ? 206 : 200, + headers: responseHeaders + }); + } + + private writeHttpMetadata(headers: Headers, object: R2Object | R2ObjectBody): void { + let metadata = object.httpMetadata; + if (!metadata || Object.keys(metadata).length === 0) { + // guessing is especially required for dev environment (miniflare) + metadata = { + contentType: guessMimeType(object.key) + }; + } + //console.log("writeHttpMetadata", object.httpMetadata, metadata); + + for (const [key, value] of Object.entries(metadata)) { + const camelToDash = key.replace(/([A-Z])/g, "-$1").toLowerCase(); + headers.set(camelToDash, value); + } + } + + async getObjectMeta(key: string): Promise<{ type: string; size: number }> { + const head = await this.headObject(key); + if (!head) { + throw new Error("Object not found"); + } + + return { + type: String(head.httpMetadata?.contentType ?? "application/octet-stream"), + size: head.size + }; + } + + async deleteObject(key: string): Promise { + await this.bucket.delete(key); + } + + getObjectUrl(key: string): string { + throw new Error("Method getObjectUrl not implemented."); + } + + toJSON(secrets?: boolean) { + return { + type: this.getName(), + config: {} + }; + } +} diff --git a/app/src/media/storage/adapters/StorageS3Adapter.ts b/app/src/media/storage/adapters/StorageS3Adapter.ts new file mode 100644 index 0000000..90c3cb2 --- /dev/null +++ b/app/src/media/storage/adapters/StorageS3Adapter.ts @@ -0,0 +1,213 @@ +import type { + DeleteObjectRequest, + GetObjectRequest, + HeadObjectRequest, + ListObjectsV2Output, + ListObjectsV2Request, + PutObjectRequest +} from "@aws-sdk/client-s3"; +import { AwsClient, isDebug } from "core"; +import { type Static, Type, parse, pickHeaders } from "core/utils"; +import { transform } from "lodash-es"; +import type { FileBody, FileListObject, StorageAdapter } from "../Storage"; + +export const s3AdapterConfig = Type.Object( + { + access_key: Type.String(), + secret_access_key: Type.String(), + url: Type.String({ + pattern: "^https?://(?:.*)?[^/.]+$", + description: "URL to S3 compatible endpoint without trailing slash", + examples: [ + "https://{account_id}.r2.cloudflarestorage.com/{bucket}", + "https://{bucket}.s3.{region}.amazonaws.com" + ] + }) + }, + { + title: "S3" + } +); + +export type S3AdapterConfig = Static; + +export class StorageS3Adapter extends AwsClient implements StorageAdapter { + readonly #config: S3AdapterConfig; + + constructor(config: S3AdapterConfig) { + super( + { + accessKeyId: config.access_key, + secretAccessKey: config.secret_access_key, + retries: isDebug() ? 0 : 10 + }, + { + convertParams: "pascalToKebab", + responseType: "xml" + } + ); + this.#config = parse(s3AdapterConfig, config); + } + + getName(): string { + return "s3"; + } + + getSchema() { + return s3AdapterConfig; + } + + override getUrl(path: string = "", searchParamsObj: Record = {}): string { + let url = this.getObjectUrl("").slice(0, -1); + if (path.length > 0) url += `/${path}`; + return super.getUrl(url, searchParamsObj); + } + + /** + * Returns the URL of an object + * @param key the key of the object + */ + getObjectUrl(key: string): string { + return `${this.#config.url}/${key}`; + } + + /** + * https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + */ + async listObjects(key: string = ""): Promise { + const params: Omit & { ListType: number } = { + ListType: 2, + Prefix: key + }; + + const url = this.getUrl("", params); + //console.log("url", url); + const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, { + method: "GET" + }); + //console.log("res", res); + + // absolutely weird, but if only one object is there, it's an object, not an array + const { Contents } = res.ListBucketResult; + const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents]; + + //console.log(JSON.stringify(res.ListBucketResult, null, 2), objects); + const transformed = transform( + objects, + (acc, obj) => { + // s3 contains folders, but Size is 0, which is filtered here + if (obj.Key && obj.LastModified && obj.Size) { + acc.push({ + key: obj.Key, + last_modified: obj.LastModified, + size: obj.Size + }); + } + }, + [] as FileListObject[] + ); + //console.log(transformed); + + return transformed; + } + + async putObject( + key: string, + body: FileBody | null, + // @todo: params must be added as headers, skipping for now + params: Omit = {} + ) { + const url = this.getUrl(key, {}); + //console.log("url", url); + const res = await this.fetch(url, { + method: "PUT", + body + }); + /*console.log("putObject:raw:res", { + ok: res.ok, + status: res.status, + statusText: res.statusText, + });*/ + + if (res.ok) { + // "df20fcb574dba1446cf5ec997940492b" + return String(res.headers.get("etag")); + } + + return undefined; + } + + private async headObject( + key: string, + params: Pick = {} + ) { + const url = this.getUrl(key, {}); + return await this.fetch(url, { + method: "HEAD", + headers: { + Range: "bytes=0-1" + } + }); + } + + async getObjectMeta(key: string) { + const res = await this.headObject(key); + const type = String(res.headers.get("content-type")); + const size = Number(String(res.headers.get("content-range")?.split("/")[1])); + + return { + type, + size + }; + } + + /** + * Check if an object exists by fetching the first byte of the object + * @param key + * @param params + */ + async objectExists( + key: string, + params: Pick = {} + ) { + return (await this.headObject(key)).ok; + } + + /** + * Simply returns the Response of the object to download body as needed + */ + async getObject(key: string, headers: Headers): Promise { + const url = this.getUrl(key); + const res = await this.fetch(url, { + method: "GET", + headers: pickHeaders(headers, ["range"]) + }); + + // Response has to be copied, because of middlewares that might set headers + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: res.headers + }); + } + + /** + * Deletes a single object. Method is void, because it doesn't return anything + */ + async deleteObject( + key: string, + params: Omit = {} + ): Promise { + const url = this.getUrl(key, params); + const res = await this.fetch(url, { + method: "DELETE" + }); + } + + toJSON(secrets?: boolean) { + return { + type: this.getName(), + config: secrets ? this.#config : undefined + }; + } +} diff --git a/app/src/media/storage/events/index.ts b/app/src/media/storage/events/index.ts new file mode 100644 index 0000000..c285600 --- /dev/null +++ b/app/src/media/storage/events/index.ts @@ -0,0 +1,17 @@ +import { Event } from "core/events"; +import type { FileBody, FileUploadPayload } from "../Storage"; + +export type FileUploadedEventData = FileUploadPayload & { + file: FileBody; +}; +export class FileUploadedEvent extends Event { + static override slug = "file-uploaded"; +} + +export class FileDeletedEvent extends Event<{ name: string }> { + static override slug = "file-deleted"; +} + +export class FileAccessEvent extends Event<{ name: string }> { + static override slug = "file-access"; +} diff --git a/app/src/media/storage/mime-types.ts b/app/src/media/storage/mime-types.ts new file mode 100644 index 0000000..a5e4647 --- /dev/null +++ b/app/src/media/storage/mime-types.ts @@ -0,0 +1,1214 @@ +/** + * File costs 10kb, can be optimized. + * @todo: optimize for lower file size + */ +export const MIME_TYPES = new Map([ + // https://github.com/guzzle/psr7/blob/2d9260799e713f1c475d3c5fdc3d6561ff7441b2/src/MimeType.php + ["1km", "application/vnd.1000minds.decision-model+xml"], + ["3dml", "text/vnd.in3d.3dml"], + ["3ds", "image/x-3ds"], + ["3g2", "video/3gpp2"], + ["3gp", "video/3gp"], + ["3gpp", "video/3gpp"], + ["3mf", "model/3mf"], + ["7z", "application/x-7z-compressed"], + ["7zip", "application/x-7z-compressed"], + ["123", "application/vnd.lotus-1-2-3"], + ["aab", "application/x-authorware-bin"], + ["aac", "audio/x-acc"], + ["aam", "application/x-authorware-map"], + ["aas", "application/x-authorware-seg"], + ["abw", "application/x-abiword"], + ["ac", "application/vnd.nokia.n-gage.ac+xml"], + ["ac3", "audio/ac3"], + ["acc", "application/vnd.americandynamics.acc"], + ["ace", "application/x-ace-compressed"], + ["acu", "application/vnd.acucobol"], + ["acutc", "application/vnd.acucorp"], + ["adp", "audio/adpcm"], + ["aep", "application/vnd.audiograph"], + ["afm", "application/x-font-type1"], + ["afp", "application/vnd.ibm.modcap"], + ["ahead", "application/vnd.ahead.space"], + ["ai", "application/pdf"], + ["aif", "audio/x-aiff"], + ["aifc", "audio/x-aiff"], + ["aiff", "audio/x-aiff"], + ["air", "application/vnd.adobe.air-application-installer-package+zip"], + ["ait", "application/vnd.dvb.ait"], + ["ami", "application/vnd.amiga.ami"], + ["amr", "audio/amr"], + ["apk", "application/vnd.android.package-archive"], + ["apng", "image/apng"], + ["appcache", "text/cache-manifest"], + ["application", "application/x-ms-application"], + ["apr", "application/vnd.lotus-approach"], + ["arc", "application/x-freearc"], + ["arj", "application/x-arj"], + ["asc", "application/pgp-signature"], + ["asf", "video/x-ms-asf"], + ["asm", "text/x-asm"], + ["aso", "application/vnd.accpac.simply.aso"], + ["asx", "video/x-ms-asf"], + ["atc", "application/vnd.acucorp"], + ["atom", "application/atom+xml"], + ["atomcat", "application/atomcat+xml"], + ["atomdeleted", "application/atomdeleted+xml"], + ["atomsvc", "application/atomsvc+xml"], + ["atx", "application/vnd.antix.game-component"], + ["au", "audio/x-au"], + ["avi", "video/x-msvideo"], + ["avif", "image/avif"], + ["aw", "application/applixware"], + ["azf", "application/vnd.airzip.filesecure.azf"], + ["azs", "application/vnd.airzip.filesecure.azs"], + ["azv", "image/vnd.airzip.accelerator.azv"], + ["azw", "application/vnd.amazon.ebook"], + ["b16", "image/vnd.pco.b16"], + ["bat", "application/x-msdownload"], + ["bcpio", "application/x-bcpio"], + ["bdf", "application/x-font-bdf"], + ["bdm", "application/vnd.syncml.dm+wbxml"], + ["bdoc", "application/x-bdoc"], + ["bed", "application/vnd.realvnc.bed"], + ["bh2", "application/vnd.fujitsu.oasysprs"], + ["bin", "application/octet-stream"], + ["blb", "application/x-blorb"], + ["blorb", "application/x-blorb"], + ["bmi", "application/vnd.bmi"], + ["bmml", "application/vnd.balsamiq.bmml+xml"], + ["bmp", "image/bmp"], + ["book", "application/vnd.framemaker"], + ["box", "application/vnd.previewsystems.box"], + ["boz", "application/x-bzip2"], + ["bpk", "application/octet-stream"], + ["bpmn", "application/octet-stream"], + ["bsp", "model/vnd.valve.source.compiled-map"], + ["btif", "image/prs.btif"], + ["buffer", "application/octet-stream"], + ["bz", "application/x-bzip"], + ["bz2", "application/x-bzip2"], + ["c", "text/x-c"], + ["c4d", "application/vnd.clonk.c4group"], + ["c4f", "application/vnd.clonk.c4group"], + ["c4g", "application/vnd.clonk.c4group"], + ["c4p", "application/vnd.clonk.c4group"], + ["c4u", "application/vnd.clonk.c4group"], + ["c11amc", "application/vnd.cluetrust.cartomobile-config"], + ["c11amz", "application/vnd.cluetrust.cartomobile-config-pkg"], + ["cab", "application/vnd.ms-cab-compressed"], + ["caf", "audio/x-caf"], + ["cap", "application/vnd.tcpdump.pcap"], + ["car", "application/vnd.curl.car"], + ["cat", "application/vnd.ms-pki.seccat"], + ["cb7", "application/x-cbr"], + ["cba", "application/x-cbr"], + ["cbr", "application/x-cbr"], + ["cbt", "application/x-cbr"], + ["cbz", "application/x-cbr"], + ["cc", "text/x-c"], + ["cco", "application/x-cocoa"], + ["cct", "application/x-director"], + ["ccxml", "application/ccxml+xml"], + ["cdbcmsg", "application/vnd.contact.cmsg"], + ["cda", "application/x-cdf"], + ["cdf", "application/x-netcdf"], + ["cdfx", "application/cdfx+xml"], + ["cdkey", "application/vnd.mediastation.cdkey"], + ["cdmia", "application/cdmi-capability"], + ["cdmic", "application/cdmi-container"], + ["cdmid", "application/cdmi-domain"], + ["cdmio", "application/cdmi-object"], + ["cdmiq", "application/cdmi-queue"], + ["cdr", "application/cdr"], + ["cdx", "chemical/x-cdx"], + ["cdxml", "application/vnd.chemdraw+xml"], + ["cdy", "application/vnd.cinderella"], + ["cer", "application/pkix-cert"], + ["cfs", "application/x-cfs-compressed"], + ["cgm", "image/cgm"], + ["chat", "application/x-chat"], + ["chm", "application/vnd.ms-htmlhelp"], + ["chrt", "application/vnd.kde.kchart"], + ["cif", "chemical/x-cif"], + ["cii", "application/vnd.anser-web-certificate-issue-initiation"], + ["cil", "application/vnd.ms-artgalry"], + ["cjs", "application/node"], + ["cla", "application/vnd.claymore"], + ["class", "application/octet-stream"], + ["clkk", "application/vnd.crick.clicker.keyboard"], + ["clkp", "application/vnd.crick.clicker.palette"], + ["clkt", "application/vnd.crick.clicker.template"], + ["clkw", "application/vnd.crick.clicker.wordbank"], + ["clkx", "application/vnd.crick.clicker"], + ["clp", "application/x-msclip"], + ["cmc", "application/vnd.cosmocaller"], + ["cmdf", "chemical/x-cmdf"], + ["cml", "chemical/x-cml"], + ["cmp", "application/vnd.yellowriver-custom-menu"], + ["cmx", "image/x-cmx"], + ["cod", "application/vnd.rim.cod"], + ["coffee", "text/coffeescript"], + ["com", "application/x-msdownload"], + ["conf", "text/plain"], + ["cpio", "application/x-cpio"], + ["cpp", "text/x-c"], + ["cpt", "application/mac-compactpro"], + ["crd", "application/x-mscardfile"], + ["crl", "application/pkix-crl"], + ["crt", "application/x-x509-ca-cert"], + ["crx", "application/x-chrome-extension"], + ["cryptonote", "application/vnd.rig.cryptonote"], + ["csh", "application/x-csh"], + ["csl", "application/vnd.citationstyles.style+xml"], + ["csml", "chemical/x-csml"], + ["csp", "application/vnd.commonspace"], + ["csr", "application/octet-stream"], + ["css", "text/css"], + ["cst", "application/x-director"], + ["csv", "text/csv"], + ["cu", "application/cu-seeme"], + ["curl", "text/vnd.curl"], + ["cww", "application/prs.cww"], + ["cxt", "application/x-director"], + ["cxx", "text/x-c"], + ["dae", "model/vnd.collada+xml"], + ["daf", "application/vnd.mobius.daf"], + ["dart", "application/vnd.dart"], + ["dataless", "application/vnd.fdsn.seed"], + ["davmount", "application/davmount+xml"], + ["dbf", "application/vnd.dbf"], + ["dbk", "application/docbook+xml"], + ["dcr", "application/x-director"], + ["dcurl", "text/vnd.curl.dcurl"], + ["dd2", "application/vnd.oma.dd2+xml"], + ["ddd", "application/vnd.fujixerox.ddd"], + ["ddf", "application/vnd.syncml.dmddf+xml"], + ["dds", "image/vnd.ms-dds"], + ["deb", "application/x-debian-package"], + ["def", "text/plain"], + ["deploy", "application/octet-stream"], + ["der", "application/x-x509-ca-cert"], + ["dfac", "application/vnd.dreamfactory"], + ["dgc", "application/x-dgc-compressed"], + ["dic", "text/x-c"], + ["dir", "application/x-director"], + ["dis", "application/vnd.mobius.dis"], + ["disposition-notification", "message/disposition-notification"], + ["dist", "application/octet-stream"], + ["distz", "application/octet-stream"], + ["djv", "image/vnd.djvu"], + ["djvu", "image/vnd.djvu"], + ["dll", "application/octet-stream"], + ["dmg", "application/x-apple-diskimage"], + ["dmn", "application/octet-stream"], + ["dmp", "application/vnd.tcpdump.pcap"], + ["dms", "application/octet-stream"], + ["dna", "application/vnd.dna"], + ["doc", "application/msword"], + ["docm", "application/vnd.ms-word.template.macroEnabled.12"], + ["docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"], + ["dot", "application/msword"], + ["dotm", "application/vnd.ms-word.template.macroEnabled.12"], + ["dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template"], + ["dp", "application/vnd.osgi.dp"], + ["dpg", "application/vnd.dpgraph"], + ["dra", "audio/vnd.dra"], + ["drle", "image/dicom-rle"], + ["dsc", "text/prs.lines.tag"], + ["dssc", "application/dssc+der"], + ["dtb", "application/x-dtbook+xml"], + ["dtd", "application/xml-dtd"], + ["dts", "audio/vnd.dts"], + ["dtshd", "audio/vnd.dts.hd"], + ["dump", "application/octet-stream"], + ["dvb", "video/vnd.dvb.file"], + ["dvi", "application/x-dvi"], + ["dwd", "application/atsc-dwd+xml"], + ["dwf", "model/vnd.dwf"], + ["dwg", "image/vnd.dwg"], + ["dxf", "image/vnd.dxf"], + ["dxp", "application/vnd.spotfire.dxp"], + ["dxr", "application/x-director"], + ["ear", "application/java-archive"], + ["ecelp4800", "audio/vnd.nuera.ecelp4800"], + ["ecelp7470", "audio/vnd.nuera.ecelp7470"], + ["ecelp9600", "audio/vnd.nuera.ecelp9600"], + ["ecma", "application/ecmascript"], + ["edm", "application/vnd.novadigm.edm"], + ["edx", "application/vnd.novadigm.edx"], + ["efif", "application/vnd.picsel"], + ["ei6", "application/vnd.pg.osasli"], + ["elc", "application/octet-stream"], + ["emf", "image/emf"], + ["eml", "message/rfc822"], + ["emma", "application/emma+xml"], + ["emotionml", "application/emotionml+xml"], + ["emz", "application/x-msmetafile"], + ["eol", "audio/vnd.digital-winds"], + ["eot", "application/vnd.ms-fontobject"], + ["eps", "application/postscript"], + ["epub", "application/epub+zip"], + ["es", "application/ecmascript"], + ["es3", "application/vnd.eszigno3+xml"], + ["esa", "application/vnd.osgi.subsystem"], + ["esf", "application/vnd.epson.esf"], + ["et3", "application/vnd.eszigno3+xml"], + ["etx", "text/x-setext"], + ["eva", "application/x-eva"], + ["evy", "application/x-envoy"], + ["exe", "application/octet-stream"], + ["exi", "application/exi"], + ["exp", "application/express"], + ["exr", "image/aces"], + ["ext", "application/vnd.novadigm.ext"], + ["ez", "application/andrew-inset"], + ["ez2", "application/vnd.ezpix-album"], + ["ez3", "application/vnd.ezpix-package"], + ["f", "text/x-fortran"], + ["f4v", "video/mp4"], + ["f77", "text/x-fortran"], + ["f90", "text/x-fortran"], + ["fbs", "image/vnd.fastbidsheet"], + ["fcdt", "application/vnd.adobe.formscentral.fcdt"], + ["fcs", "application/vnd.isac.fcs"], + ["fdf", "application/vnd.fdf"], + ["fdt", "application/fdt+xml"], + ["fe_launch", "application/vnd.denovo.fcselayout-link"], + ["fg5", "application/vnd.fujitsu.oasysgp"], + ["fgd", "application/x-director"], + ["fh", "image/x-freehand"], + ["fh4", "image/x-freehand"], + ["fh5", "image/x-freehand"], + ["fh7", "image/x-freehand"], + ["fhc", "image/x-freehand"], + ["fig", "application/x-xfig"], + ["fits", "image/fits"], + ["flac", "audio/x-flac"], + ["fli", "video/x-fli"], + ["flo", "application/vnd.micrografx.flo"], + ["flv", "video/x-flv"], + ["flw", "application/vnd.kde.kivio"], + ["flx", "text/vnd.fmi.flexstor"], + ["fly", "text/vnd.fly"], + ["fm", "application/vnd.framemaker"], + ["fnc", "application/vnd.frogans.fnc"], + ["fo", "application/vnd.software602.filler.form+xml"], + ["for", "text/x-fortran"], + ["fpx", "image/vnd.fpx"], + ["frame", "application/vnd.framemaker"], + ["fsc", "application/vnd.fsc.weblaunch"], + ["fst", "image/vnd.fst"], + ["ftc", "application/vnd.fluxtime.clip"], + ["fti", "application/vnd.anser-web-funds-transfer-initiation"], + ["fvt", "video/vnd.fvt"], + ["fxp", "application/vnd.adobe.fxp"], + ["fxpl", "application/vnd.adobe.fxp"], + ["fzs", "application/vnd.fuzzysheet"], + ["g2w", "application/vnd.geoplan"], + ["g3", "image/g3fax"], + ["g3w", "application/vnd.geospace"], + ["gac", "application/vnd.groove-account"], + ["gam", "application/x-tads"], + ["gbr", "application/rpki-ghostbusters"], + ["gca", "application/x-gca-compressed"], + ["gdl", "model/vnd.gdl"], + ["gdoc", "application/vnd.google-apps.document"], + ["geo", "application/vnd.dynageo"], + ["geojson", "application/geo+json"], + ["gex", "application/vnd.geometry-explorer"], + ["ggb", "application/vnd.geogebra.file"], + ["ggt", "application/vnd.geogebra.tool"], + ["ghf", "application/vnd.groove-help"], + ["gif", "image/gif"], + ["gim", "application/vnd.groove-identity-message"], + ["glb", "model/gltf-binary"], + ["gltf", "model/gltf+json"], + ["gml", "application/gml+xml"], + ["gmx", "application/vnd.gmx"], + ["gnumeric", "application/x-gnumeric"], + ["gpg", "application/gpg-keys"], + ["gph", "application/vnd.flographit"], + ["gpx", "application/gpx+xml"], + ["gqf", "application/vnd.grafeq"], + ["gqs", "application/vnd.grafeq"], + ["gram", "application/srgs"], + ["gramps", "application/x-gramps-xml"], + ["gre", "application/vnd.geometry-explorer"], + ["grv", "application/vnd.groove-injector"], + ["grxml", "application/srgs+xml"], + ["gsf", "application/x-font-ghostscript"], + ["gsheet", "application/vnd.google-apps.spreadsheet"], + ["gslides", "application/vnd.google-apps.presentation"], + ["gtar", "application/x-gtar"], + ["gtm", "application/vnd.groove-tool-message"], + ["gtw", "model/vnd.gtw"], + ["gv", "text/vnd.graphviz"], + ["gxf", "application/gxf"], + ["gxt", "application/vnd.geonext"], + ["gz", "application/gzip"], + ["gzip", "application/gzip"], + ["h", "text/x-c"], + ["h261", "video/h261"], + ["h263", "video/h263"], + ["h264", "video/h264"], + ["hal", "application/vnd.hal+xml"], + ["hbci", "application/vnd.hbci"], + ["hbs", "text/x-handlebars-template"], + ["hdd", "application/x-virtualbox-hdd"], + ["hdf", "application/x-hdf"], + ["heic", "image/heic"], + ["heics", "image/heic-sequence"], + ["heif", "image/heif"], + ["heifs", "image/heif-sequence"], + ["hej2", "image/hej2k"], + ["held", "application/atsc-held+xml"], + ["hh", "text/x-c"], + ["hjson", "application/hjson"], + ["hlp", "application/winhlp"], + ["hpgl", "application/vnd.hp-hpgl"], + ["hpid", "application/vnd.hp-hpid"], + ["hps", "application/vnd.hp-hps"], + ["hqx", "application/mac-binhex40"], + ["hsj2", "image/hsj2"], + ["htc", "text/x-component"], + ["htke", "application/vnd.kenameaapp"], + ["htm", "text/html"], + ["html", "text/html"], + ["hvd", "application/vnd.yamaha.hv-dic"], + ["hvp", "application/vnd.yamaha.hv-voice"], + ["hvs", "application/vnd.yamaha.hv-script"], + ["i2g", "application/vnd.intergeo"], + ["icc", "application/vnd.iccprofile"], + ["ice", "x-conference/x-cooltalk"], + ["icm", "application/vnd.iccprofile"], + ["ico", "image/x-icon"], + ["ics", "text/calendar"], + ["ief", "image/ief"], + ["ifb", "text/calendar"], + ["ifm", "application/vnd.shana.informed.formdata"], + ["iges", "model/iges"], + ["igl", "application/vnd.igloader"], + ["igm", "application/vnd.insors.igm"], + ["igs", "model/iges"], + ["igx", "application/vnd.micrografx.igx"], + ["iif", "application/vnd.shana.informed.interchange"], + ["img", "application/octet-stream"], + ["imp", "application/vnd.accpac.simply.imp"], + ["ims", "application/vnd.ms-ims"], + ["in", "text/plain"], + ["ini", "text/plain"], + ["ink", "application/inkml+xml"], + ["inkml", "application/inkml+xml"], + ["install", "application/x-install-instructions"], + ["iota", "application/vnd.astraea-software.iota"], + ["ipfix", "application/ipfix"], + ["ipk", "application/vnd.shana.informed.package"], + ["irm", "application/vnd.ibm.rights-management"], + ["irp", "application/vnd.irepository.package+xml"], + ["iso", "application/x-iso9660-image"], + ["itp", "application/vnd.shana.informed.formtemplate"], + ["its", "application/its+xml"], + ["ivp", "application/vnd.immervision-ivp"], + ["ivu", "application/vnd.immervision-ivu"], + ["jad", "text/vnd.sun.j2me.app-descriptor"], + ["jade", "text/jade"], + ["jam", "application/vnd.jam"], + ["jar", "application/java-archive"], + ["jardiff", "application/x-java-archive-diff"], + ["java", "text/x-java-source"], + ["jhc", "image/jphc"], + ["jisp", "application/vnd.jisp"], + ["jls", "image/jls"], + ["jlt", "application/vnd.hp-jlyt"], + ["jng", "image/x-jng"], + ["jnlp", "application/x-java-jnlp-file"], + ["joda", "application/vnd.joost.joda-archive"], + ["jp2", "image/jp2"], + ["jpe", "image/jpeg"], + ["jpeg", "image/jpeg"], + ["jpf", "image/jpx"], + ["jpg", "image/jpeg"], + ["jpg2", "image/jp2"], + ["jpgm", "video/jpm"], + ["jpgv", "video/jpeg"], + ["jph", "image/jph"], + ["jpm", "video/jpm"], + ["jpx", "image/jpx"], + ["js", "application/javascript"], + ["json", "application/json"], + ["json5", "application/json5"], + ["jsonld", "application/ld+json"], + ["jsonml", "application/jsonml+json"], + ["jsx", "text/jsx"], + ["jxr", "image/jxr"], + ["jxra", "image/jxra"], + ["jxrs", "image/jxrs"], + ["jxs", "image/jxs"], + ["jxsc", "image/jxsc"], + ["jxsi", "image/jxsi"], + ["jxss", "image/jxss"], + ["kar", "audio/midi"], + ["karbon", "application/vnd.kde.karbon"], + ["kdb", "application/octet-stream"], + ["kdbx", "application/x-keepass2"], + ["key", "application/x-iwork-keynote-sffkey"], + ["kfo", "application/vnd.kde.kformula"], + ["kia", "application/vnd.kidspiration"], + ["kml", "application/vnd.google-earth.kml+xml"], + ["kmz", "application/vnd.google-earth.kmz"], + ["kne", "application/vnd.kinar"], + ["knp", "application/vnd.kinar"], + ["kon", "application/vnd.kde.kontour"], + ["kpr", "application/vnd.kde.kpresenter"], + ["kpt", "application/vnd.kde.kpresenter"], + ["kpxx", "application/vnd.ds-keypoint"], + ["ksp", "application/vnd.kde.kspread"], + ["ktr", "application/vnd.kahootz"], + ["ktx", "image/ktx"], + ["ktx2", "image/ktx2"], + ["ktz", "application/vnd.kahootz"], + ["kwd", "application/vnd.kde.kword"], + ["kwt", "application/vnd.kde.kword"], + ["lasxml", "application/vnd.las.las+xml"], + ["latex", "application/x-latex"], + ["lbd", "application/vnd.llamagraphics.life-balance.desktop"], + ["lbe", "application/vnd.llamagraphics.life-balance.exchange+xml"], + ["les", "application/vnd.hhe.lesson-player"], + ["less", "text/less"], + ["lgr", "application/lgr+xml"], + ["lha", "application/octet-stream"], + ["link66", "application/vnd.route66.link66+xml"], + ["list", "text/plain"], + ["list3820", "application/vnd.ibm.modcap"], + ["listafp", "application/vnd.ibm.modcap"], + ["litcoffee", "text/coffeescript"], + ["lnk", "application/x-ms-shortcut"], + ["log", "text/plain"], + ["lostxml", "application/lost+xml"], + ["lrf", "application/octet-stream"], + ["lrm", "application/vnd.ms-lrm"], + ["ltf", "application/vnd.frogans.ltf"], + ["lua", "text/x-lua"], + ["luac", "application/x-lua-bytecode"], + ["lvp", "audio/vnd.lucent.voice"], + ["lwp", "application/vnd.lotus-wordpro"], + ["lzh", "application/octet-stream"], + ["m1v", "video/mpeg"], + ["m2a", "audio/mpeg"], + ["m2v", "video/mpeg"], + ["m3a", "audio/mpeg"], + ["m3u", "text/plain"], + ["m3u8", "application/vnd.apple.mpegurl"], + ["m4a", "audio/x-m4a"], + ["m4p", "application/mp4"], + ["m4s", "video/iso.segment"], + ["m4u", "application/vnd.mpegurl"], + ["m4v", "video/x-m4v"], + ["m13", "application/x-msmediaview"], + ["m14", "application/x-msmediaview"], + ["m21", "application/mp21"], + ["ma", "application/mathematica"], + ["mads", "application/mads+xml"], + ["maei", "application/mmt-aei+xml"], + ["mag", "application/vnd.ecowin.chart"], + ["maker", "application/vnd.framemaker"], + ["man", "text/troff"], + ["manifest", "text/cache-manifest"], + ["map", "application/json"], + ["mar", "application/octet-stream"], + ["markdown", "text/markdown"], + ["mathml", "application/mathml+xml"], + ["mb", "application/mathematica"], + ["mbk", "application/vnd.mobius.mbk"], + ["mbox", "application/mbox"], + ["mc1", "application/vnd.medcalcdata"], + ["mcd", "application/vnd.mcd"], + ["mcurl", "text/vnd.curl.mcurl"], + ["md", "text/markdown"], + ["mdb", "application/x-msaccess"], + ["mdi", "image/vnd.ms-modi"], + ["mdx", "text/mdx"], + ["me", "text/troff"], + ["mesh", "model/mesh"], + ["meta4", "application/metalink4+xml"], + ["metalink", "application/metalink+xml"], + ["mets", "application/mets+xml"], + ["mfm", "application/vnd.mfmp"], + ["mft", "application/rpki-manifest"], + ["mgp", "application/vnd.osgeo.mapguide.package"], + ["mgz", "application/vnd.proteus.magazine"], + ["mid", "audio/midi"], + ["midi", "audio/midi"], + ["mie", "application/x-mie"], + ["mif", "application/vnd.mif"], + ["mime", "message/rfc822"], + ["mj2", "video/mj2"], + ["mjp2", "video/mj2"], + ["mjs", "application/javascript"], + ["mk3d", "video/x-matroska"], + ["mka", "audio/x-matroska"], + ["mkd", "text/x-markdown"], + ["mks", "video/x-matroska"], + ["mkv", "video/x-matroska"], + ["mlp", "application/vnd.dolby.mlp"], + ["mmd", "application/vnd.chipnuts.karaoke-mmd"], + ["mmf", "application/vnd.smaf"], + ["mml", "text/mathml"], + ["mmr", "image/vnd.fujixerox.edmics-mmr"], + ["mng", "video/x-mng"], + ["mny", "application/x-msmoney"], + ["mobi", "application/x-mobipocket-ebook"], + ["mods", "application/mods+xml"], + ["mov", "video/quicktime"], + ["movie", "video/x-sgi-movie"], + ["mp2", "audio/mpeg"], + ["mp2a", "audio/mpeg"], + ["mp3", "audio/mpeg"], + ["mp4", "video/mp4"], + ["mp4a", "audio/mp4"], + ["mp4s", "application/mp4"], + ["mp4v", "video/mp4"], + ["mp21", "application/mp21"], + ["mpc", "application/vnd.mophun.certificate"], + ["mpd", "application/dash+xml"], + ["mpe", "video/mpeg"], + ["mpeg", "video/mpeg"], + ["mpg", "video/mpeg"], + ["mpg4", "video/mp4"], + ["mpga", "audio/mpeg"], + ["mpkg", "application/vnd.apple.installer+xml"], + ["mpm", "application/vnd.blueice.multipass"], + ["mpn", "application/vnd.mophun.application"], + ["mpp", "application/vnd.ms-project"], + ["mpt", "application/vnd.ms-project"], + ["mpy", "application/vnd.ibm.minipay"], + ["mqy", "application/vnd.mobius.mqy"], + ["mrc", "application/marc"], + ["mrcx", "application/marcxml+xml"], + ["ms", "text/troff"], + ["mscml", "application/mediaservercontrol+xml"], + ["mseed", "application/vnd.fdsn.mseed"], + ["mseq", "application/vnd.mseq"], + ["msf", "application/vnd.epson.msf"], + ["msg", "application/vnd.ms-outlook"], + ["msh", "model/mesh"], + ["msi", "application/x-msdownload"], + ["msl", "application/vnd.mobius.msl"], + ["msm", "application/octet-stream"], + ["msp", "application/octet-stream"], + ["msty", "application/vnd.muvee.style"], + ["mtl", "model/mtl"], + ["mts", "model/vnd.mts"], + ["mus", "application/vnd.musician"], + ["musd", "application/mmt-usd+xml"], + ["musicxml", "application/vnd.recordare.musicxml+xml"], + ["mvb", "application/x-msmediaview"], + ["mvt", "application/vnd.mapbox-vector-tile"], + ["mwf", "application/vnd.mfer"], + ["mxf", "application/mxf"], + ["mxl", "application/vnd.recordare.musicxml"], + ["mxmf", "audio/mobile-xmf"], + ["mxml", "application/xv+xml"], + ["mxs", "application/vnd.triscape.mxs"], + ["mxu", "video/vnd.mpegurl"], + ["n-gage", "application/vnd.nokia.n-gage.symbian.install"], + ["n3", "text/n3"], + ["nb", "application/mathematica"], + ["nbp", "application/vnd.wolfram.player"], + ["nc", "application/x-netcdf"], + ["ncx", "application/x-dtbncx+xml"], + ["nfo", "text/x-nfo"], + ["ngdat", "application/vnd.nokia.n-gage.data"], + ["nitf", "application/vnd.nitf"], + ["nlu", "application/vnd.neurolanguage.nlu"], + ["nml", "application/vnd.enliven"], + ["nnd", "application/vnd.noblenet-directory"], + ["nns", "application/vnd.noblenet-sealer"], + ["nnw", "application/vnd.noblenet-web"], + ["npx", "image/vnd.net-fpx"], + ["nq", "application/n-quads"], + ["nsc", "application/x-conference"], + ["nsf", "application/vnd.lotus-notes"], + ["nt", "application/n-triples"], + ["ntf", "application/vnd.nitf"], + ["numbers", "application/x-iwork-numbers-sffnumbers"], + ["nzb", "application/x-nzb"], + ["oa2", "application/vnd.fujitsu.oasys2"], + ["oa3", "application/vnd.fujitsu.oasys3"], + ["oas", "application/vnd.fujitsu.oasys"], + ["obd", "application/x-msbinder"], + ["obgx", "application/vnd.openblox.game+xml"], + ["obj", "model/obj"], + ["oda", "application/oda"], + ["odb", "application/vnd.oasis.opendocument.database"], + ["odc", "application/vnd.oasis.opendocument.chart"], + ["odf", "application/vnd.oasis.opendocument.formula"], + ["odft", "application/vnd.oasis.opendocument.formula-template"], + ["odg", "application/vnd.oasis.opendocument.graphics"], + ["odi", "application/vnd.oasis.opendocument.image"], + ["odm", "application/vnd.oasis.opendocument.text-master"], + ["odp", "application/vnd.oasis.opendocument.presentation"], + ["ods", "application/vnd.oasis.opendocument.spreadsheet"], + ["odt", "application/vnd.oasis.opendocument.text"], + ["oga", "audio/ogg"], + ["ogex", "model/vnd.opengex"], + ["ogg", "audio/ogg"], + ["ogv", "video/ogg"], + ["ogx", "application/ogg"], + ["omdoc", "application/omdoc+xml"], + ["onepkg", "application/onenote"], + ["onetmp", "application/onenote"], + ["onetoc", "application/onenote"], + ["onetoc2", "application/onenote"], + ["opf", "application/oebps-package+xml"], + ["opml", "text/x-opml"], + ["oprc", "application/vnd.palm"], + ["opus", "audio/ogg"], + ["org", "text/x-org"], + ["osf", "application/vnd.yamaha.openscoreformat"], + ["osfpvg", "application/vnd.yamaha.openscoreformat.osfpvg+xml"], + ["osm", "application/vnd.openstreetmap.data+xml"], + ["otc", "application/vnd.oasis.opendocument.chart-template"], + ["otf", "font/otf"], + ["otg", "application/vnd.oasis.opendocument.graphics-template"], + ["oth", "application/vnd.oasis.opendocument.text-web"], + ["oti", "application/vnd.oasis.opendocument.image-template"], + ["otp", "application/vnd.oasis.opendocument.presentation-template"], + ["ots", "application/vnd.oasis.opendocument.spreadsheet-template"], + ["ott", "application/vnd.oasis.opendocument.text-template"], + ["ova", "application/x-virtualbox-ova"], + ["ovf", "application/x-virtualbox-ovf"], + ["owl", "application/rdf+xml"], + ["oxps", "application/oxps"], + ["oxt", "application/vnd.openofficeorg.extension"], + ["p", "text/x-pascal"], + ["p7a", "application/x-pkcs7-signature"], + ["p7b", "application/x-pkcs7-certificates"], + ["p7c", "application/pkcs7-mime"], + ["p7m", "application/pkcs7-mime"], + ["p7r", "application/x-pkcs7-certreqresp"], + ["p7s", "application/pkcs7-signature"], + ["p8", "application/pkcs8"], + ["p10", "application/x-pkcs10"], + ["p12", "application/x-pkcs12"], + ["pac", "application/x-ns-proxy-autoconfig"], + ["pages", "application/x-iwork-pages-sffpages"], + ["pas", "text/x-pascal"], + ["paw", "application/vnd.pawaafile"], + ["pbd", "application/vnd.powerbuilder6"], + ["pbm", "image/x-portable-bitmap"], + ["pcap", "application/vnd.tcpdump.pcap"], + ["pcf", "application/x-font-pcf"], + ["pcl", "application/vnd.hp-pcl"], + ["pclxl", "application/vnd.hp-pclxl"], + ["pct", "image/x-pict"], + ["pcurl", "application/vnd.curl.pcurl"], + ["pcx", "image/x-pcx"], + ["pdb", "application/x-pilot"], + ["pde", "text/x-processing"], + ["pdf", "application/pdf"], + ["pem", "application/x-x509-user-cert"], + ["pfa", "application/x-font-type1"], + ["pfb", "application/x-font-type1"], + ["pfm", "application/x-font-type1"], + ["pfr", "application/font-tdpfr"], + ["pfx", "application/x-pkcs12"], + ["pgm", "image/x-portable-graymap"], + ["pgn", "application/x-chess-pgn"], + ["pgp", "application/pgp"], + ["php", "application/x-httpd-php"], + ["php3", "application/x-httpd-php"], + ["php4", "application/x-httpd-php"], + ["phps", "application/x-httpd-php-source"], + ["phtml", "application/x-httpd-php"], + ["pic", "image/x-pict"], + ["pkg", "application/octet-stream"], + ["pki", "application/pkixcmp"], + ["pkipath", "application/pkix-pkipath"], + ["pkpass", "application/vnd.apple.pkpass"], + ["pl", "application/x-perl"], + ["plb", "application/vnd.3gpp.pic-bw-large"], + ["plc", "application/vnd.mobius.plc"], + ["plf", "application/vnd.pocketlearn"], + ["pls", "application/pls+xml"], + ["pm", "application/x-perl"], + ["pml", "application/vnd.ctc-posml"], + ["png", "image/png"], + ["pnm", "image/x-portable-anymap"], + ["portpkg", "application/vnd.macports.portpkg"], + ["pot", "application/vnd.ms-powerpoint"], + ["potm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12"], + ["potx", "application/vnd.openxmlformats-officedocument.presentationml.template"], + ["ppa", "application/vnd.ms-powerpoint"], + ["ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12"], + ["ppd", "application/vnd.cups-ppd"], + ["ppm", "image/x-portable-pixmap"], + ["pps", "application/vnd.ms-powerpoint"], + ["ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12"], + ["ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow"], + ["ppt", "application/powerpoint"], + ["pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12"], + ["pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"], + ["pqa", "application/vnd.palm"], + ["prc", "application/x-pilot"], + ["pre", "application/vnd.lotus-freelance"], + ["prf", "application/pics-rules"], + ["provx", "application/provenance+xml"], + ["ps", "application/postscript"], + ["psb", "application/vnd.3gpp.pic-bw-small"], + ["psd", "application/x-photoshop"], + ["psf", "application/x-font-linux-psf"], + ["pskcxml", "application/pskc+xml"], + ["pti", "image/prs.pti"], + ["ptid", "application/vnd.pvi.ptid1"], + ["pub", "application/x-mspublisher"], + ["pvb", "application/vnd.3gpp.pic-bw-var"], + ["pwn", "application/vnd.3m.post-it-notes"], + ["pya", "audio/vnd.ms-playready.media.pya"], + ["pyv", "video/vnd.ms-playready.media.pyv"], + ["qam", "application/vnd.epson.quickanime"], + ["qbo", "application/vnd.intu.qbo"], + ["qfx", "application/vnd.intu.qfx"], + ["qps", "application/vnd.publishare-delta-tree"], + ["qt", "video/quicktime"], + ["qwd", "application/vnd.quark.quarkxpress"], + ["qwt", "application/vnd.quark.quarkxpress"], + ["qxb", "application/vnd.quark.quarkxpress"], + ["qxd", "application/vnd.quark.quarkxpress"], + ["qxl", "application/vnd.quark.quarkxpress"], + ["qxt", "application/vnd.quark.quarkxpress"], + ["ra", "audio/x-realaudio"], + ["ram", "audio/x-pn-realaudio"], + ["raml", "application/raml+yaml"], + ["rapd", "application/route-apd+xml"], + ["rar", "application/x-rar"], + ["ras", "image/x-cmu-raster"], + ["rcprofile", "application/vnd.ipunplugged.rcprofile"], + ["rdf", "application/rdf+xml"], + ["rdz", "application/vnd.data-vision.rdz"], + ["relo", "application/p2p-overlay+xml"], + ["rep", "application/vnd.businessobjects"], + ["res", "application/x-dtbresource+xml"], + ["rgb", "image/x-rgb"], + ["rif", "application/reginfo+xml"], + ["rip", "audio/vnd.rip"], + ["ris", "application/x-research-info-systems"], + ["rl", "application/resource-lists+xml"], + ["rlc", "image/vnd.fujixerox.edmics-rlc"], + ["rld", "application/resource-lists-diff+xml"], + ["rm", "audio/x-pn-realaudio"], + ["rmi", "audio/midi"], + ["rmp", "audio/x-pn-realaudio-plugin"], + ["rms", "application/vnd.jcp.javame.midlet-rms"], + ["rmvb", "application/vnd.rn-realmedia-vbr"], + ["rnc", "application/relax-ng-compact-syntax"], + ["rng", "application/xml"], + ["roa", "application/rpki-roa"], + ["roff", "text/troff"], + ["rp9", "application/vnd.cloanto.rp9"], + ["rpm", "audio/x-pn-realaudio-plugin"], + ["rpss", "application/vnd.nokia.radio-presets"], + ["rpst", "application/vnd.nokia.radio-preset"], + ["rq", "application/sparql-query"], + ["rs", "application/rls-services+xml"], + ["rsa", "application/x-pkcs7"], + ["rsat", "application/atsc-rsat+xml"], + ["rsd", "application/rsd+xml"], + ["rsheet", "application/urc-ressheet+xml"], + ["rss", "application/rss+xml"], + ["rtf", "text/rtf"], + ["rtx", "text/richtext"], + ["run", "application/x-makeself"], + ["rusd", "application/route-usd+xml"], + ["rv", "video/vnd.rn-realvideo"], + ["s", "text/x-asm"], + ["s3m", "audio/s3m"], + ["saf", "application/vnd.yamaha.smaf-audio"], + ["sass", "text/x-sass"], + ["sbml", "application/sbml+xml"], + ["sc", "application/vnd.ibm.secure-container"], + ["scd", "application/x-msschedule"], + ["scm", "application/vnd.lotus-screencam"], + ["scq", "application/scvp-cv-request"], + ["scs", "application/scvp-cv-response"], + ["scss", "text/x-scss"], + ["scurl", "text/vnd.curl.scurl"], + ["sda", "application/vnd.stardivision.draw"], + ["sdc", "application/vnd.stardivision.calc"], + ["sdd", "application/vnd.stardivision.impress"], + ["sdkd", "application/vnd.solent.sdkm+xml"], + ["sdkm", "application/vnd.solent.sdkm+xml"], + ["sdp", "application/sdp"], + ["sdw", "application/vnd.stardivision.writer"], + ["sea", "application/octet-stream"], + ["see", "application/vnd.seemail"], + ["seed", "application/vnd.fdsn.seed"], + ["sema", "application/vnd.sema"], + ["semd", "application/vnd.semd"], + ["semf", "application/vnd.semf"], + ["senmlx", "application/senml+xml"], + ["sensmlx", "application/sensml+xml"], + ["ser", "application/java-serialized-object"], + ["setpay", "application/set-payment-initiation"], + ["setreg", "application/set-registration-initiation"], + ["sfd-hdstx", "application/vnd.hydrostatix.sof-data"], + ["sfs", "application/vnd.spotfire.sfs"], + ["sfv", "text/x-sfv"], + ["sgi", "image/sgi"], + ["sgl", "application/vnd.stardivision.writer-global"], + ["sgm", "text/sgml"], + ["sgml", "text/sgml"], + ["sh", "application/x-sh"], + ["shar", "application/x-shar"], + ["shex", "text/shex"], + ["shf", "application/shf+xml"], + ["shtml", "text/html"], + ["sid", "image/x-mrsid-image"], + ["sieve", "application/sieve"], + ["sig", "application/pgp-signature"], + ["sil", "audio/silk"], + ["silo", "model/mesh"], + ["sis", "application/vnd.symbian.install"], + ["sisx", "application/vnd.symbian.install"], + ["sit", "application/x-stuffit"], + ["sitx", "application/x-stuffitx"], + ["siv", "application/sieve"], + ["skd", "application/vnd.koan"], + ["skm", "application/vnd.koan"], + ["skp", "application/vnd.koan"], + ["skt", "application/vnd.koan"], + ["sldm", "application/vnd.ms-powerpoint.slide.macroenabled.12"], + ["sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide"], + ["slim", "text/slim"], + ["slm", "text/slim"], + ["sls", "application/route-s-tsid+xml"], + ["slt", "application/vnd.epson.salt"], + ["sm", "application/vnd.stepmania.stepchart"], + ["smf", "application/vnd.stardivision.math"], + ["smi", "application/smil"], + ["smil", "application/smil"], + ["smv", "video/x-smv"], + ["smzip", "application/vnd.stepmania.package"], + ["snd", "audio/basic"], + ["snf", "application/x-font-snf"], + ["so", "application/octet-stream"], + ["spc", "application/x-pkcs7-certificates"], + ["spdx", "text/spdx"], + ["spf", "application/vnd.yamaha.smaf-phrase"], + ["spl", "application/x-futuresplash"], + ["spot", "text/vnd.in3d.spot"], + ["spp", "application/scvp-vp-response"], + ["spq", "application/scvp-vp-request"], + ["spx", "audio/ogg"], + ["sql", "application/x-sql"], + ["src", "application/x-wais-source"], + ["srt", "application/x-subrip"], + ["sru", "application/sru+xml"], + ["srx", "application/sparql-results+xml"], + ["ssdl", "application/ssdl+xml"], + ["sse", "application/vnd.kodak-descriptor"], + ["ssf", "application/vnd.epson.ssf"], + ["ssml", "application/ssml+xml"], + ["sst", "application/octet-stream"], + ["st", "application/vnd.sailingtracker.track"], + ["stc", "application/vnd.sun.xml.calc.template"], + ["std", "application/vnd.sun.xml.draw.template"], + ["stf", "application/vnd.wt.stf"], + ["sti", "application/vnd.sun.xml.impress.template"], + ["stk", "application/hyperstudio"], + ["stl", "model/stl"], + ["stpx", "model/step+xml"], + ["stpxz", "model/step-xml+zip"], + ["stpz", "model/step+zip"], + ["str", "application/vnd.pg.format"], + ["stw", "application/vnd.sun.xml.writer.template"], + ["styl", "text/stylus"], + ["stylus", "text/stylus"], + ["sub", "text/vnd.dvb.subtitle"], + ["sus", "application/vnd.sus-calendar"], + ["susp", "application/vnd.sus-calendar"], + ["sv4cpio", "application/x-sv4cpio"], + ["sv4crc", "application/x-sv4crc"], + ["svc", "application/vnd.dvb.service"], + ["svd", "application/vnd.svd"], + ["svg", "image/svg+xml"], + ["svgz", "image/svg+xml"], + ["swa", "application/x-director"], + ["swf", "application/x-shockwave-flash"], + ["swi", "application/vnd.aristanetworks.swi"], + ["swidtag", "application/swid+xml"], + ["sxc", "application/vnd.sun.xml.calc"], + ["sxd", "application/vnd.sun.xml.draw"], + ["sxg", "application/vnd.sun.xml.writer.global"], + ["sxi", "application/vnd.sun.xml.impress"], + ["sxm", "application/vnd.sun.xml.math"], + ["sxw", "application/vnd.sun.xml.writer"], + ["t", "text/troff"], + ["t3", "application/x-t3vm-image"], + ["t38", "image/t38"], + ["taglet", "application/vnd.mynfc"], + ["tao", "application/vnd.tao.intent-module-archive"], + ["tap", "image/vnd.tencent.tap"], + ["tar", "application/x-tar"], + ["tcap", "application/vnd.3gpp2.tcap"], + ["tcl", "application/x-tcl"], + ["td", "application/urc-targetdesc+xml"], + ["teacher", "application/vnd.smart.teacher"], + ["tei", "application/tei+xml"], + ["teicorpus", "application/tei+xml"], + ["tex", "application/x-tex"], + ["texi", "application/x-texinfo"], + ["texinfo", "application/x-texinfo"], + ["text", "text/plain"], + ["tfi", "application/thraud+xml"], + ["tfm", "application/x-tex-tfm"], + ["tfx", "image/tiff-fx"], + ["tga", "image/x-tga"], + ["tgz", "application/x-tar"], + ["thmx", "application/vnd.ms-officetheme"], + ["tif", "image/tiff"], + ["tiff", "image/tiff"], + ["tk", "application/x-tcl"], + ["tmo", "application/vnd.tmobile-livetv"], + ["toml", "application/toml"], + ["torrent", "application/x-bittorrent"], + ["tpl", "application/vnd.groove-tool-template"], + ["tpt", "application/vnd.trid.tpt"], + ["tr", "text/troff"], + ["tra", "application/vnd.trueapp"], + ["trig", "application/trig"], + ["trm", "application/x-msterminal"], + ["ts", "video/mp2t"], + ["tsd", "application/timestamped-data"], + ["tsv", "text/tab-separated-values"], + ["ttc", "font/collection"], + ["ttf", "font/ttf"], + ["ttl", "text/turtle"], + ["ttml", "application/ttml+xml"], + ["twd", "application/vnd.simtech-mindmapper"], + ["twds", "application/vnd.simtech-mindmapper"], + ["txd", "application/vnd.genomatix.tuxedo"], + ["txf", "application/vnd.mobius.txf"], + ["txt", "text/plain"], + ["u8dsn", "message/global-delivery-status"], + ["u8hdr", "message/global-headers"], + ["u8mdn", "message/global-disposition-notification"], + ["u8msg", "message/global"], + ["u32", "application/x-authorware-bin"], + ["ubj", "application/ubjson"], + ["udeb", "application/x-debian-package"], + ["ufd", "application/vnd.ufdl"], + ["ufdl", "application/vnd.ufdl"], + ["ulx", "application/x-glulx"], + ["umj", "application/vnd.umajin"], + ["unityweb", "application/vnd.unity"], + ["uoml", "application/vnd.uoml+xml"], + ["uri", "text/uri-list"], + ["uris", "text/uri-list"], + ["urls", "text/uri-list"], + ["usdz", "model/vnd.usdz+zip"], + ["ustar", "application/x-ustar"], + ["utz", "application/vnd.uiq.theme"], + ["uu", "text/x-uuencode"], + ["uva", "audio/vnd.dece.audio"], + ["uvd", "application/vnd.dece.data"], + ["uvf", "application/vnd.dece.data"], + ["uvg", "image/vnd.dece.graphic"], + ["uvh", "video/vnd.dece.hd"], + ["uvi", "image/vnd.dece.graphic"], + ["uvm", "video/vnd.dece.mobile"], + ["uvp", "video/vnd.dece.pd"], + ["uvs", "video/vnd.dece.sd"], + ["uvt", "application/vnd.dece.ttml+xml"], + ["uvu", "video/vnd.uvvu.mp4"], + ["uvv", "video/vnd.dece.video"], + ["uvva", "audio/vnd.dece.audio"], + ["uvvd", "application/vnd.dece.data"], + ["uvvf", "application/vnd.dece.data"], + ["uvvg", "image/vnd.dece.graphic"], + ["uvvh", "video/vnd.dece.hd"], + ["uvvi", "image/vnd.dece.graphic"], + ["uvvm", "video/vnd.dece.mobile"], + ["uvvp", "video/vnd.dece.pd"], + ["uvvs", "video/vnd.dece.sd"], + ["uvvt", "application/vnd.dece.ttml+xml"], + ["uvvu", "video/vnd.uvvu.mp4"], + ["uvvv", "video/vnd.dece.video"], + ["uvvx", "application/vnd.dece.unspecified"], + ["uvvz", "application/vnd.dece.zip"], + ["uvx", "application/vnd.dece.unspecified"], + ["uvz", "application/vnd.dece.zip"], + ["vbox", "application/x-virtualbox-vbox"], + ["vbox-extpack", "application/x-virtualbox-vbox-extpack"], + ["vcard", "text/vcard"], + ["vcd", "application/x-cdlink"], + ["vcf", "text/x-vcard"], + ["vcg", "application/vnd.groove-vcard"], + ["vcs", "text/x-vcalendar"], + ["vcx", "application/vnd.vcx"], + ["vdi", "application/x-virtualbox-vdi"], + ["vds", "model/vnd.sap.vds"], + ["vhd", "application/x-virtualbox-vhd"], + ["vis", "application/vnd.visionary"], + ["viv", "video/vnd.vivo"], + ["vlc", "application/videolan"], + ["vmdk", "application/x-virtualbox-vmdk"], + ["vob", "video/x-ms-vob"], + ["vor", "application/vnd.stardivision.writer"], + ["vox", "application/x-authorware-bin"], + ["vrml", "model/vrml"], + ["vsd", "application/vnd.visio"], + ["vsf", "application/vnd.vsf"], + ["vss", "application/vnd.visio"], + ["vst", "application/vnd.visio"], + ["vsw", "application/vnd.visio"], + ["vtf", "image/vnd.valve.source.texture"], + ["vtt", "text/vtt"], + ["vtu", "model/vnd.vtu"], + ["vxml", "application/voicexml+xml"], + ["w3d", "application/x-director"], + ["wad", "application/x-doom"], + ["wadl", "application/vnd.sun.wadl+xml"], + ["war", "application/java-archive"], + ["wasm", "application/wasm"], + ["wav", "audio/x-wav"], + ["wax", "audio/x-ms-wax"], + ["wbmp", "image/vnd.wap.wbmp"], + ["wbs", "application/vnd.criticaltools.wbs+xml"], + ["wbxml", "application/wbxml"], + ["wcm", "application/vnd.ms-works"], + ["wdb", "application/vnd.ms-works"], + ["wdp", "image/vnd.ms-photo"], + ["weba", "audio/webm"], + ["webapp", "application/x-web-app-manifest+json"], + ["webm", "video/webm"], + ["webmanifest", "application/manifest+json"], + ["webp", "image/webp"], + ["wg", "application/vnd.pmi.widget"], + ["wgt", "application/widget"], + ["wks", "application/vnd.ms-works"], + ["wm", "video/x-ms-wm"], + ["wma", "audio/x-ms-wma"], + ["wmd", "application/x-ms-wmd"], + ["wmf", "image/wmf"], + ["wml", "text/vnd.wap.wml"], + ["wmlc", "application/wmlc"], + ["wmls", "text/vnd.wap.wmlscript"], + ["wmlsc", "application/vnd.wap.wmlscriptc"], + ["wmv", "video/x-ms-wmv"], + ["wmx", "video/x-ms-wmx"], + ["wmz", "application/x-msmetafile"], + ["woff", "font/woff"], + ["woff2", "font/woff2"], + ["word", "application/msword"], + ["wpd", "application/vnd.wordperfect"], + ["wpl", "application/vnd.ms-wpl"], + ["wps", "application/vnd.ms-works"], + ["wqd", "application/vnd.wqd"], + ["wri", "application/x-mswrite"], + ["wrl", "model/vrml"], + ["wsc", "message/vnd.wfa.wsc"], + ["wsdl", "application/wsdl+xml"], + ["wspolicy", "application/wspolicy+xml"], + ["wtb", "application/vnd.webturbo"], + ["wvx", "video/x-ms-wvx"], + ["x3d", "model/x3d+xml"], + ["x3db", "model/x3d+fastinfoset"], + ["x3dbz", "model/x3d+binary"], + ["x3dv", "model/x3d-vrml"], + ["x3dvz", "model/x3d+vrml"], + ["x3dz", "model/x3d+xml"], + ["x32", "application/x-authorware-bin"], + ["x_b", "model/vnd.parasolid.transmit.binary"], + ["x_t", "model/vnd.parasolid.transmit.text"], + ["xaml", "application/xaml+xml"], + ["xap", "application/x-silverlight-app"], + ["xar", "application/vnd.xara"], + ["xav", "application/xcap-att+xml"], + ["xbap", "application/x-ms-xbap"], + ["xbd", "application/vnd.fujixerox.docuworks.binder"], + ["xbm", "image/x-xbitmap"], + ["xca", "application/xcap-caps+xml"], + ["xcs", "application/calendar+xml"], + ["xdf", "application/xcap-diff+xml"], + ["xdm", "application/vnd.syncml.dm+xml"], + ["xdp", "application/vnd.adobe.xdp+xml"], + ["xdssc", "application/dssc+xml"], + ["xdw", "application/vnd.fujixerox.docuworks"], + ["xel", "application/xcap-el+xml"], + ["xenc", "application/xenc+xml"], + ["xer", "application/patch-ops-error+xml"], + ["xfdf", "application/vnd.adobe.xfdf"], + ["xfdl", "application/vnd.xfdl"], + ["xht", "application/xhtml+xml"], + ["xhtml", "application/xhtml+xml"], + ["xhvml", "application/xv+xml"], + ["xif", "image/vnd.xiff"], + ["xl", "application/excel"], + ["xla", "application/vnd.ms-excel"], + ["xlam", "application/vnd.ms-excel.addin.macroEnabled.12"], + ["xlc", "application/vnd.ms-excel"], + ["xlf", "application/xliff+xml"], + ["xlm", "application/vnd.ms-excel"], + ["xls", "application/vnd.ms-excel"], + ["xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"], + ["xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"], + ["xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"], + ["xlt", "application/vnd.ms-excel"], + ["xltm", "application/vnd.ms-excel.template.macroEnabled.12"], + ["xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"], + ["xlw", "application/vnd.ms-excel"], + ["xm", "audio/xm"], + ["xml", "application/xml"], + ["xns", "application/xcap-ns+xml"], + ["xo", "application/vnd.olpc-sugar"], + ["xop", "application/xop+xml"], + ["xpi", "application/x-xpinstall"], + ["xpl", "application/xproc+xml"], + ["xpm", "image/x-xpixmap"], + ["xpr", "application/vnd.is-xpr"], + ["xps", "application/vnd.ms-xpsdocument"], + ["xpw", "application/vnd.intercon.formnet"], + ["xpx", "application/vnd.intercon.formnet"], + ["xsd", "application/xml"], + ["xsl", "application/xml"], + ["xslt", "application/xslt+xml"], + ["xsm", "application/vnd.syncml+xml"], + ["xspf", "application/xspf+xml"], + ["xul", "application/vnd.mozilla.xul+xml"], + ["xvm", "application/xv+xml"], + ["xvml", "application/xv+xml"], + ["xwd", "image/x-xwindowdump"], + ["xyz", "chemical/x-xyz"], + ["xz", "application/x-xz"], + ["yaml", "text/yaml"], + ["yang", "application/yang"], + ["yin", "application/yin+xml"], + ["yml", "text/yaml"], + ["ymp", "text/x-suse-ymp"], + ["z", "application/x-compress"], + ["z1", "application/x-zmachine"], + ["z2", "application/x-zmachine"], + ["z3", "application/x-zmachine"], + ["z4", "application/x-zmachine"], + ["z5", "application/x-zmachine"], + ["z6", "application/x-zmachine"], + ["z7", "application/x-zmachine"], + ["z8", "application/x-zmachine"], + ["zaz", "application/vnd.zzazz.deck+xml"], + ["zip", "application/zip"], + ["zir", "application/vnd.zul"], + ["zirz", "application/vnd.zul"], + ["zmm", "application/vnd.handheld-entertainment+xml"], + ["zsh", "text/x-scriptzsh"], +]); + +export function guessMimeType(filename: string): string { + try { + const ext = filename.split(".").pop(); + return MIME_TYPES.get(ext!) as string; + } catch (e) { + return "application/octet-stream"; + } +} diff --git a/app/src/media/utils/index.ts b/app/src/media/utils/index.ts new file mode 100644 index 0000000..7a02cf8 --- /dev/null +++ b/app/src/media/utils/index.ts @@ -0,0 +1,21 @@ +import { randomString } from "core/utils"; + +export function getExtension(filename: string): string | undefined { + if (!filename.includes(".")) return; + + const parts = filename.split("."); + return parts[parts.length - 1]; +} + +export function getRandomizedFilename(file: File, length?: number): string; +export function getRandomizedFilename(file: string, length?: number): string; +export function getRandomizedFilename(file: File | string, length = 16): string { + const filename = file instanceof File ? file.name : file; + + if (typeof filename !== "string") { + console.error("Couldn't extract filename from", file); + throw new Error("Invalid file name"); + } + + return [randomString(length), getExtension(filename)].filter(Boolean).join("."); +} diff --git a/app/src/modules/Module.ts b/app/src/modules/Module.ts new file mode 100644 index 0000000..c3364c3 --- /dev/null +++ b/app/src/modules/Module.ts @@ -0,0 +1,112 @@ +import type { Guard } from "auth"; +import { SchemaObject } from "core"; +import type { EventManager } from "core/events"; +import type { Static, TSchema } from "core/utils"; +import type { Connection, EntityManager } from "data"; +import type { Hono } from "hono"; + +export type ModuleBuildContext = { + connection: Connection; + server: Hono; + em: EntityManager; + emgr: EventManager; + guard: Guard; +}; + +export abstract class Module { + private _built = false; + private _schema: SchemaObject>; + private _listener: any = () => null; + + constructor( + initial?: Partial>, + protected _ctx?: ModuleBuildContext + ) { + this._schema = new SchemaObject(this.getSchema(), initial, { + forceParse: this.useForceParse(), + onUpdate: async (c) => { + await this._listener(c); + }, + restrictPaths: this.getRestrictedPaths(), + overwritePaths: this.getOverwritePaths() + }); + } + + setListener(listener: (c: ReturnType<(typeof this)["getSchema"]>) => void | Promise) { + this._listener = listener; + return this; + } + + // @todo: test all getSchema() for additional properties + abstract getSchema(); + + useForceParse() { + return false; + } + + getRestrictedPaths(): string[] | undefined { + return undefined; + } + + /** + * These paths will be overwritten, even when "patch" is called. + * This is helpful if there are keys that contains records, which always be sent in full. + */ + getOverwritePaths(): (RegExp | string)[] | undefined { + return undefined; + } + + get configDefault(): Static> { + return this._schema.default(); + } + + get config(): Static> { + return this._schema.get(); + } + + setContext(ctx: ModuleBuildContext) { + this._ctx = ctx; + return this; + } + + schema() { + return this._schema; + } + + get ctx() { + if (!this._ctx) { + throw new Error("Context not set"); + } + return this._ctx; + } + + async build() { + throw new Error("Not implemented"); + } + + setBuilt() { + this._built = true; + this._schema = new SchemaObject(this.getSchema(), this.toJSON(true), { + onUpdate: async (c) => { + await this._listener(c); + }, + forceParse: this.useForceParse(), + restrictPaths: this.getRestrictedPaths(), + overwritePaths: this.getOverwritePaths() + }); + } + + isBuilt() { + return this._built; + } + + throwIfNotBuilt() { + if (!this._built) { + throw new Error("Config not built: " + this.constructor.name); + } + } + + toJSON(secrets?: boolean): Static> { + return this.config; + } +} diff --git a/app/src/modules/ModuleApi.ts b/app/src/modules/ModuleApi.ts new file mode 100644 index 0000000..4c6aa8d --- /dev/null +++ b/app/src/modules/ModuleApi.ts @@ -0,0 +1,149 @@ +import type { PrimaryFieldType } from "core"; +import { encodeSearch } from "core/utils"; + +export type { PrimaryFieldType }; +export type BaseModuleApiOptions = { + host: string; + basepath?: string; + token?: string; +}; + +export type ApiResponse = { + success: boolean; + status: number; + body: Data; + data?: Data extends { data: infer R } ? R : any; + res: Response; +}; + +export abstract class ModuleApi { + constructor(protected readonly _options: Partial = {}) {} + + protected getDefaultOptions(): Partial { + return {}; + } + + get options(): Options { + return { + host: "http://localhost", + token: undefined, + ...this.getDefaultOptions(), + ...this._options + } as Options; + } + + protected getUrl(path: string) { + return this.options.host + (this.options.basepath + "/" + path).replace(/\/\//g, "/"); + } + + protected async request( + _input: string | (string | number | PrimaryFieldType)[], + _query?: Record | URLSearchParams, + _init?: RequestInit + ): Promise> { + const method = _init?.method ?? "GET"; + const input = Array.isArray(_input) ? _input.join("/") : _input; + let url = this.getUrl(input); + + if (_query instanceof URLSearchParams) { + url += "?" + _query.toString(); + } else if (typeof _query === "object") { + if (Object.keys(_query).length > 0) { + url += "?" + encodeSearch(_query); + } + } + + const headers = new Headers(_init?.headers ?? {}); + headers.set("Accept", "application/json"); + + if (this.options.token) { + //console.log("setting token", this.options.token); + headers.set("Authorization", `Bearer ${this.options.token}`); + } else { + //console.log("no token"); + } + + let body: any = _init?.body; + if (_init && "body" in _init && ["POST", "PATCH"].includes(method)) { + const requestContentType = (headers.get("Content-Type") as string) ?? undefined; + if (!requestContentType || requestContentType.startsWith("application/json")) { + body = JSON.stringify(_init.body); + headers.set("Content-Type", "application/json"); + } + } + + //console.log("url", url); + const res = await fetch(url, { + ..._init, + method, + body, + headers + }); + + let resBody: any; + let resData: any; + + const contentType = res.headers.get("Content-Type") ?? ""; + if (contentType.startsWith("application/json")) { + resBody = await res.json(); + if (typeof resBody === "object") { + resData = "data" in resBody ? resBody.data : resBody; + } + } else if (contentType.startsWith("text")) { + resBody = await res.text(); + } + + return { + success: res.ok, + status: res.status, + body: resBody, + data: resData, + res + }; + } + + protected async get( + _input: string | (string | number | PrimaryFieldType)[], + _query?: Record | URLSearchParams, + _init?: RequestInit + ) { + return this.request(_input, _query, { + ..._init, + method: "GET" + }); + } + + protected async post( + _input: string | (string | number | PrimaryFieldType)[], + body?: any, + _init?: RequestInit + ) { + return this.request(_input, undefined, { + ..._init, + body, + method: "POST" + }); + } + + protected async patch( + _input: string | (string | number | PrimaryFieldType)[], + body?: any, + _init?: RequestInit + ) { + return this.request(_input, undefined, { + ..._init, + body, + method: "PATCH" + }); + } + + protected async delete( + _input: string | (string | number | PrimaryFieldType)[], + _init?: RequestInit + ) { + return this.request(_input, undefined, { + ..._init, + method: "DELETE" + }); + } +} diff --git a/app/src/modules/ModuleManager.ts b/app/src/modules/ModuleManager.ts new file mode 100644 index 0000000..85c6ea5 --- /dev/null +++ b/app/src/modules/ModuleManager.ts @@ -0,0 +1,443 @@ +import { Diff } from "@sinclair/typebox/value"; +import { Guard } from "auth"; +import { DebugLogger, isDebug } from "core"; +import { EventManager } from "core/events"; +import { Default, type Static, objectEach, transformObject } from "core/utils"; +import { type Connection, EntityManager } from "data"; +import { Hono } from "hono"; +import { type Kysely, sql } from "kysely"; +import { CURRENT_VERSION, TABLE_NAME, migrate, migrateSchema } from "modules/migrations"; +import { AppServer } from "modules/server/AppServer"; +import { AppAuth } from "../auth/AppAuth"; +import { AppData } from "../data/AppData"; +import { AppFlows } from "../flows/AppFlows"; +import { AppMedia } from "../media/AppMedia"; +import type { Module, ModuleBuildContext } from "./Module"; + +export const MODULES = { + server: AppServer, + data: AppData, + auth: AppAuth, + media: AppMedia, + flows: AppFlows +} as const; + +// get names of MODULES as an array +export const MODULE_NAMES = Object.keys(MODULES) as ModuleKey[]; + +export type ModuleKey = keyof typeof MODULES; +export type Modules = { + [K in keyof typeof MODULES]: InstanceType<(typeof MODULES)[K]>; +}; + +export type ModuleSchemas = { + [K in keyof typeof MODULES]: ReturnType<(typeof MODULES)[K]["prototype"]["getSchema"]>; +}; + +export type ModuleConfigs = { + [K in keyof ModuleSchemas]: Static; +}; + +export type InitialModuleConfigs = { + version: number; +} & Partial; + +export type ModuleManagerOptions = { + initial?: InitialModuleConfigs; + eventManager?: EventManager; + onUpdated?: ( + module: Module, + config: ModuleConfigs[Module] + ) => Promise; + // base path for the hono instance + basePath?: string; +}; + +type ConfigTable = { + version: number; + type: "config" | "diff" | "backup"; + json: Json; + created_at?: Date; + updated_at?: Date; +}; + +export class ModuleManager { + private modules: Modules; + em!: EntityManager; + server!: Hono; + emgr!: EventManager; + guard!: Guard; + + private _version: number = 0; + private _built = false; + private _fetched = false; + private readonly _provided; + + private logger = new DebugLogger(isDebug() && false); + + constructor( + private readonly connection: Connection, + private options?: Partial + ) { + this.modules = {} as Modules; + this.emgr = new EventManager(); + const context = this.ctx(true); + let initial = {} as Partial; + + if (options?.initial) { + const { version, ...initialConfig } = options.initial; + if (version && initialConfig) { + this._version = version; + initial = initialConfig; + + this._provided = true; + } else { + throw new Error("Initial was provided, but it needs a version!"); + } + } + + for (const key in MODULES) { + const moduleConfig = key in initial ? initial[key] : {}; + const module = new MODULES[key](moduleConfig, context) as Module; + module.setListener(async (c) => { + await this.onModuleConfigUpdated(key, c); + }); + + this.modules[key] = module; + } + } + + /** + * This is set through module's setListener + * It's called everytime a module's config is updated in SchemaObject + * Needs to rebuild modules and save to database + */ + private async onModuleConfigUpdated(key: string, config: any) { + if (this.options?.onUpdated) { + await this.options.onUpdated(key as any, config); + } else { + this.buildModules(); + } + } + + private rebuildServer() { + this.server = new Hono(); + if (this.options?.basePath) { + this.server = this.server.basePath(this.options.basePath); + } + + // @todo: this is a current workaround, controllers must be reworked + objectEach(this.modules, (module) => { + if ("getMiddleware" in module) { + const middleware = module.getMiddleware(); + if (middleware) { + this.server.use(middleware); + } + } + }); + } + + ctx(rebuild?: boolean): ModuleBuildContext { + if (rebuild) { + this.rebuildServer(); + this.em = new EntityManager([], this.connection, [], [], this.emgr); + this.guard = new Guard(); + } + + return { + connection: this.connection, + server: this.server, + em: this.em, + emgr: this.emgr, + guard: this.guard + }; + } + + private get db() { + return this.connection.kysely as Kysely<{ table: ConfigTable }>; + } + + get table() { + return TABLE_NAME as "table"; + } + + private async fetch(): Promise { + this.logger.context("fetch").log("fetching"); + + const startTime = performance.now(); + const result = await this.db + .selectFrom(this.table) + .selectAll() + .where("type", "=", "config") + .orderBy("version", "desc") + .executeTakeFirstOrThrow(); + + this.logger.log("took", performance.now() - startTime, "ms", result).clear(); + return result; + } + + async save() { + this.logger.context("save").log("saving version", this.version()); + const configs = this.configs(); + const version = this.version(); + + const json = JSON.stringify(configs) as any; + const state = await this.fetch(); + + if (state.version !== version) { + // @todo: mark all others as "backup" + this.logger.log("version conflict, storing new version", state.version, version); + await this.db + .insertInto(this.table) + .values({ + version, + type: "config", + json + }) + .execute(); + } else { + this.logger.log("version matches"); + + const diff = Diff(state.json, JSON.parse(json)); + this.logger.log("checking diff", diff); + + if (diff.length > 0) { + // store diff + await this.db + .insertInto(this.table) + .values({ + version, + type: "diff", + json: JSON.stringify(diff) as any + }) + .execute(); + + await this.db + .updateTable(this.table) + .set({ version, json, updated_at: sql`CURRENT_TIMESTAMP` }) + .where((eb) => eb.and([eb("type", "=", "config"), eb("version", "=", version)])) + .execute(); + } else { + this.logger.log("no diff, not saving"); + } + } + + // cleanup + /*this.logger.log("cleaning up"); + const result = await this.db + .deleteFrom(this.table) + .where((eb) => + eb.or([ + // empty migrations + eb.and([ + eb("type", "=", "config"), + eb("version", "<", version), + eb("json", "is", null) + ]), + // past diffs + eb.and([eb("type", "=", "diff"), eb("version", "<", version)]) + ]) + ) + .executeTakeFirst(); + this.logger.log("cleaned up", result.numDeletedRows);*/ + + this.logger.clear(); + return this; + } + + private async migrate() { + this.logger.context("migrate").log("migrating?", this.version(), CURRENT_VERSION); + + if (this.version() < CURRENT_VERSION) { + this.logger.log("there are migrations, verify version"); + + // modules must be built before migration + await this.buildModules({ graceful: true }); + + try { + const state = await this.fetch(); + if (state.version !== this.version()) { + // @todo: potentially drop provided config and use database version + throw new Error( + `Given version (${this.version()}) and fetched version (${state.version}) do not match.` + ); + } + } catch (e: any) { + this.logger.clear(); // fetch couldn't clear + + // if table doesn't exist, migrate schema to version + if (e.message.includes("no such table")) { + this.logger.log("table has to created, migrating schema up to", this.version()); + await migrateSchema(this.version(), { db: this.db }); + } else { + throw new Error(`Version is ${this.version()}, fetch failed: ${e.message}`); + } + } + + this.logger.log("now migrating"); + let version = this.version(); + let configs: any = this.configs(); + //console.log("migrating with", version, configs); + if (Object.keys(configs).length === 0) { + throw new Error("No config to migrate"); + } + + const [_version, _configs] = await migrate(version, configs, { + db: this.db + }); + version = _version; + configs = _configs; + + this.setConfigs(configs); + /* objectEach(configs, (config, key) => { + this.get(key as any).setConfig(config); + }); */ + + this._version = version; + this.logger.log("migrated to", version); + + await this.save(); + } else { + this.logger.log("no migrations needed"); + } + + this.logger.clear(); + } + + private setConfigs(configs: ModuleConfigs): void { + objectEach(configs, (config, key) => { + try { + // setting "noEmit" to true, to not force listeners to update + this.modules[key].schema().set(config as any, true); + } catch (e) { + console.error(e); + throw new Error( + `Failed to set config for module ${key}: ${JSON.stringify(config, null, 2)}` + ); + } + }); + } + + private async buildModules(options?: { graceful?: boolean }) { + this.logger.log("buildModules() triggered", options?.graceful, this._built); + if (options?.graceful && this._built) { + this.logger.log("skipping build (graceful)"); + return; + } + + const ctx = this.ctx(true); + for (const key in this.modules) { + this.logger.log(`building "${key}"`); + await this.modules[key].setContext(ctx).build(); + } + + this._built = true; + this.logger.log("modules built"); + } + + async build() { + this.logger.context("build").log("version", this.version()); + + // if no config provided, try fetch from db + if (this.version() === 0) { + this.logger.context("build no config").log("version is 0"); + try { + const result = await this.fetch(); + + // set version and config from fetched + this._version = result.version; + this.setConfigs(result.json); + } catch (e: any) { + this.logger.clear(); // fetch couldn't clear + + this.logger.context("error handler").log("fetch failed", e.message); + // if table doesn't exist, migrate schema, set default config and latest version + if (e.message.includes("no such table")) { + this.logger.log("migrate schema to", CURRENT_VERSION); + await migrateSchema(CURRENT_VERSION, { db: this.db }); + this._version = CURRENT_VERSION; + + // we can safely build modules, since config version is up to date + // it's up to date because we use default configs (no fetch result) + await this.buildModules(); + await this.save(); + + this.logger.clear(); + return this; + } else { + throw e; + //throw new Error("Issues connecting to the database. Reason: " + e.message); + } + } + this.logger.clear(); + } + + // migrate to latest if needed + await this.migrate(); + + this.logger.log("building"); + await this.buildModules(); + return this; + } + + get(key: K): Modules[K] { + if (!(key in this.modules)) { + throw new Error(`Module "${key}" doesn't exist, cannot get`); + } + return this.modules[key]; + } + + version() { + return this._version; + } + + built() { + return this._built; + } + + configs(): ModuleConfigs { + return transformObject(this.modules, (module) => module.toJSON(true)) as any; + } + + getSchema() { + const schemas = transformObject(this.modules, (module) => module.getSchema()); + + return { + version: this.version(), + ...schemas + }; + } + + toJSON(secrets?: boolean): { version: number } & ModuleConfigs { + const modules = transformObject(this.modules, (module) => { + if (this._built) { + return module.isBuilt() ? module.toJSON(secrets) : module.configDefault; + } + + // returns no config if the all modules are not built + return undefined; + }); + + return { + version: this.version(), + ...modules + } as any; + } +} + +export function getDefaultSchema(pretty = false) { + const schema = { + type: "object", + ...transformObject(MODULES, (module) => module.prototype.getSchema()) + }; + + return JSON.stringify(schema, null, pretty ? 2 : undefined); +} + +export function getDefaultConfig(pretty = false): ModuleConfigs { + const config = transformObject(MODULES, (module) => { + return Default(module.prototype.getSchema(), {}); + }); + + return JSON.stringify(config, null, pretty ? 2 : undefined) as any; +} diff --git a/app/src/modules/SystemApi.ts b/app/src/modules/SystemApi.ts new file mode 100644 index 0000000..7dd056c --- /dev/null +++ b/app/src/modules/SystemApi.ts @@ -0,0 +1,24 @@ +import { ModuleApi } from "./ModuleApi"; +import type { ModuleConfigs, ModuleSchemas } from "./ModuleManager"; + +export type ApiSchemaResponse = { + version: number; + schema: ModuleSchemas; + config: ModuleConfigs; + permissions: string[]; +}; + +export class SystemApi extends ModuleApi { + protected override getDefaultOptions(): Partial { + return { + basepath: "/api/system" + }; + } + + async readSchema(options?: { config?: boolean; secrets?: boolean }) { + return await this.get("schema", { + config: options?.config ? 1 : 0, + secrets: options?.secrets ? 1 : 0 + }); + } +} diff --git a/app/src/modules/index.ts b/app/src/modules/index.ts new file mode 100644 index 0000000..186a689 --- /dev/null +++ b/app/src/modules/index.ts @@ -0,0 +1,21 @@ +import * as prototype from "data/prototype"; +export { prototype }; + +export { AppAuth } from "auth/AppAuth"; +export { AppData } from "data/AppData"; +export { AppMedia, type MediaFieldSchema } from "media/AppMedia"; +export { AppFlows, type AppFlowsSchema } from "flows/AppFlows"; +export { + type ModuleConfigs, + type ModuleSchemas, + MODULE_NAMES, + type ModuleKey +} from "./ModuleManager"; +export { /*Module,*/ type ModuleBuildContext } from "./Module"; + +export { + type PrimaryFieldType, + type BaseModuleApiOptions, + type ApiResponse, + ModuleApi +} from "./ModuleApi"; diff --git a/app/src/modules/migrations.ts b/app/src/modules/migrations.ts new file mode 100644 index 0000000..8a28557 --- /dev/null +++ b/app/src/modules/migrations.ts @@ -0,0 +1,143 @@ +import { _jsonp } from "core/utils"; +import { type Kysely, sql } from "kysely"; +import { set } from "lodash-es"; + +export type MigrationContext = { + db: Kysely; +}; +export type GenericConfigObject = Record; + +export type Migration = { + version: number; + schema?: true; + up: (config: GenericConfigObject, ctx: MigrationContext) => Promise; +}; + +export const migrations: Migration[] = [ + { + version: 1, + schema: true, + up: async (config, { db }) => { + //console.log("config given", config); + await db.schema + .createTable(TABLE_NAME) + .addColumn("id", "integer", (col) => col.primaryKey().notNull().autoIncrement()) + .addColumn("version", "integer", (col) => col.notNull()) + .addColumn("type", "text", (col) => col.notNull()) + .addColumn("json", "text") + .addColumn("created_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) + .addColumn("updated_at", "datetime", (col) => col.defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); + + await db + .insertInto(TABLE_NAME) + .values({ version: 1, type: "config", json: null }) + .execute(); + + return config; + } + }, + { + version: 2, + up: async (config, { db }) => { + return config; + } + }, + { + version: 3, + schema: true, + up: async (config, { db }) => { + await db.schema.alterTable(TABLE_NAME).addColumn("deleted_at", "datetime").execute(); + + return config; + } + }, + { + version: 4, + up: async (config, { db }) => { + return { + ...config, + auth: { + ...config.auth, + basepath: "/api/auth2" + } + }; + } + }, + { + version: 5, + up: async (config, { db }) => { + //console.log("config", _jsonp(config)); + const cors = config.server.cors?.allow_methods ?? []; + set(config.server, "cors.allow_methods", [...new Set([...cors, "PATCH"])]); + return config; + } + }, + { + version: 6, + up: async (config, { db }) => { + return config; + } + } +]; + +export const CURRENT_VERSION = migrations[migrations.length - 1]?.version ?? 0; +export const TABLE_NAME = "__bknd"; + +export async function migrateTo( + current: number, + to: number, + config: GenericConfigObject, + ctx: MigrationContext +): Promise<[number, GenericConfigObject]> { + //console.log("migrating from", current, "to", CURRENT_VERSION, config); + const todo = migrations.filter((m) => m.version > current && m.version <= to); + //console.log("todo", todo.length); + let updated = Object.assign({}, config); + + let i = 0; + let version = current; + for (const migration of todo) { + //console.log("-- running migration", i + 1, "of", todo.length, { version: migration.version }); + try { + updated = await migration.up(updated, ctx); + version = migration.version; + i++; + } catch (e: any) { + console.error(e); + throw new Error(`Migration ${migration.version} failed: ${e.message}`); + } + } + + return [version, updated]; +} + +export async function migrateSchema(to: number, ctx: MigrationContext, current: number = 0) { + console.log("migrating SCHEMA to", to, "from", current); + const todo = migrations.filter((m) => m.version > current && m.version <= to && m.schema); + console.log("todo", todo.length); + + let i = 0; + let version = 0; + for (const migration of todo) { + console.log("-- running migration", i + 1, "of", todo.length); + try { + await migration.up({}, ctx); + version = migration.version; + i++; + } catch (e: any) { + console.error(e); + throw new Error(`Migration ${migration.version} failed: ${e.message}`); + } + } + + return version; +} + +export async function migrate( + current: number, + config: GenericConfigObject, + ctx: MigrationContext +): Promise<[number, GenericConfigObject]> { + return migrateTo(current, CURRENT_VERSION, config, ctx); +} diff --git a/app/src/modules/permissions/index.ts b/app/src/modules/permissions/index.ts new file mode 100644 index 0000000..8b9cb9b --- /dev/null +++ b/app/src/modules/permissions/index.ts @@ -0,0 +1,7 @@ +import { Permission } from "core"; + +export const configRead = new Permission("system.config.read"); +export const configReadSecrets = new Permission("system.config.read.secrets"); +export const configWrite = new Permission("system.config.write"); +export const schemaRead = new Permission("system.schema.read"); +export const build = new Permission("system.build"); diff --git a/app/src/modules/registries.ts b/app/src/modules/registries.ts new file mode 100644 index 0000000..9513794 --- /dev/null +++ b/app/src/modules/registries.ts @@ -0,0 +1,7 @@ +import { MediaAdapterRegistry } from "media"; + +const registries = { + media: MediaAdapterRegistry +} as const; + +export { registries }; diff --git a/app/src/modules/server/AppController.ts b/app/src/modules/server/AppController.ts new file mode 100644 index 0000000..2562814 --- /dev/null +++ b/app/src/modules/server/AppController.ts @@ -0,0 +1,110 @@ +import type { ClassController } from "core"; +import { SimpleRenderer } from "core"; +import { FetchTask, Flow, LogTask } from "flows"; +import { Hono } from "hono"; +import { endTime, startTime } from "hono/timing"; +import type { App } from "../../App"; + +export class AppController implements ClassController { + constructor( + private readonly app: App, + private config: any = {} + ) {} + + getController(): Hono { + const hono = new Hono(); + + // @todo: add test endpoints + + hono + .get("/config", (c) => { + return c.json(this.app.toJSON()); + }) + .get("/ping", (c) => { + //console.log("c", c); + try { + // @ts-ignore @todo: fix with env + const context: any = c.req.raw.cf ? c.req.raw.cf : c.env.cf; + const cf = { + colo: context.colo, + city: context.city, + postal: context.postalCode, + region: context.region, + regionCode: context.regionCode, + continent: context.continent, + country: context.country, + eu: context.isEUCountry, + lat: context.latitude, + lng: context.longitude, + timezone: context.timezone + }; + return c.json({ pong: true, cf, another: 6 }); + } catch (e) { + return c.json({ pong: true, cf: null }); + } + }); + + // test endpoints + if (this.config?.registerTest) { + hono.get("/test/kv", async (c) => { + // @ts-ignore + const cache = c.env!.CACHE as KVNamespace; + startTime(c, "kv-get"); + const value: any = await cache.get("count"); + endTime(c, "kv-get"); + console.log("value", value); + startTime(c, "kv-put"); + if (!value) { + await cache.put("count", "1"); + } else { + await cache.put("count", (Number(value) + 1).toString()); + } + endTime(c, "kv-put"); + + let cf: any = {}; + // @ts-ignore + if ("cf" in c.req.raw) { + cf = { + // @ts-ignore + colo: c.req.raw.cf?.colo + }; + } + + return c.json({ pong: true, value, cf }); + }); + + hono.get("/test/flow", async (c) => { + const first = new LogTask("Task 0"); + const second = new LogTask("Task 1"); + const third = new LogTask("Task 2", { delay: 250 }); + const fourth = new FetchTask("Fetch Something", { + url: "https://jsonplaceholder.typicode.com/todos/1" + }); + const fifth = new LogTask("Task 4"); // without connection + + const flow = new Flow("flow", [first, second, third, fourth, fifth]); + flow.task(first).asInputFor(second); + flow.task(first).asInputFor(third); + flow.task(fourth).asOutputFor(third); + + flow.setRespondingTask(fourth); + + const execution = flow.createExecution(); + await execution.start(); + + const results = flow.tasks.map((t) => t.toJSON()); + + return c.json({ results, response: execution.getResponse() }); + }); + + hono.get("/test/template", async (c) => { + const renderer = new SimpleRenderer({ var: 123 }); + const template = "Variable: {{ var }}"; + + return c.text(await renderer.render(template)); + }); + } + + return hono; + } +} diff --git a/app/src/modules/server/AppServer.ts b/app/src/modules/server/AppServer.ts new file mode 100644 index 0000000..9df8e02 --- /dev/null +++ b/app/src/modules/server/AppServer.ts @@ -0,0 +1,143 @@ +import { Exception, isDebug } from "core"; +import { type Static, StringEnum, Type } from "core/utils"; +import { Hono } from "hono"; +import { cors } from "hono/cors"; +import { timing } from "hono/timing"; +import { Module } from "modules/Module"; + +const serverMethods = ["GET", "POST", "PATCH", "PUT", "DELETE"]; +export const serverConfigSchema = Type.Object( + { + admin: Type.Object( + { + basepath: Type.Optional(Type.String({ default: "", pattern: "^(/.+)?$" })), + color_scheme: Type.Optional(StringEnum(["dark", "light"], { default: "light" })), + logo_return_path: Type.Optional( + Type.String({ + default: "/", + description: "Path to return to after *clicking* the logo" + }) + ) + }, + { default: {}, additionalProperties: false } + ), + cors: Type.Object( + { + origin: Type.String({ default: "*" }), + allow_methods: Type.Array(StringEnum(serverMethods), { + default: serverMethods, + uniqueItems: true + }), + allow_headers: Type.Array(Type.String(), { + default: ["Content-Type", "Content-Length", "Authorization", "Accept"] + }) + }, + { default: {}, additionalProperties: false } + ) + }, + { + additionalProperties: false + } +); + +export type AppServerConfig = Static; + +/*declare global { + interface Request { + cf: IncomingRequestCfProperties; + } +}*/ + +export class AppServer extends Module { + private admin_html?: string; + + override getRestrictedPaths() { + return []; + } + + get client() { + return this.ctx.server; + } + + getSchema() { + return serverConfigSchema; + } + + override async build() { + //this.client.use(timing()); + + /*this.client.use("*", async (c, next) => { + console.log(`[${c.req.method}] ${c.req.url}`); + await next(); + });*/ + this.client.use( + "*", + cors({ + origin: this.config.cors.origin, + allowMethods: this.config.cors.allow_methods, + allowHeaders: this.config.cors.allow_headers + }) + ); + + /*this.client.use(async (c, next) => { + c.res.headers.set("X-Powered-By", "BKND"); + try { + c.res.headers.set("X-Colo", c.req.raw.cf.colo); + } catch (e) {} + await next(); + }); + this.client.use(async (c, next) => { + console.log(`[${c.req.method}] ${c.req.url}`); + await next(); + });*/ + + this.client.onError((err, c) => { + //throw err; + console.error(err); + + if (err instanceof Response) { + return err; + } + + /*if (isDebug()) { + console.log("accept", c.req.header("Accept")); + if (c.req.header("Accept") === "application/json") { + const stack = err.stack; + + if ("toJSON" in err && typeof err.toJSON === "function") { + return c.json({ ...err.toJSON(), stack }, 500); + } + + return c.json({ message: String(err), stack }, 500); + } else { + throw err; + } + }*/ + + if (err instanceof Exception) { + console.log("---is exception", err.code); + return c.json(err.toJSON(), err.code as any); + } + + return c.json({ error: err.message }, 500); + }); + this.setBuilt(); + } + + setAdminHtml(html: string) { + this.admin_html = html; + const basepath = (String(this.config.admin.basepath) + "/").replace(/\/+$/, "/"); + + this.client.get(basepath + "*", async (c, next) => { + return c.html(this.admin_html!); + }); + } + + getAdminHtml() { + return this.admin_html; + } + + override toJSON(secrets?: boolean) { + return this.config; + } +} diff --git a/app/src/modules/server/SystemController.ts b/app/src/modules/server/SystemController.ts new file mode 100644 index 0000000..5777338 --- /dev/null +++ b/app/src/modules/server/SystemController.ts @@ -0,0 +1,311 @@ +/// + +import type { ClassController } from "core"; +import { tbValidator as tb } from "core"; +import { StringEnum, Type, TypeInvalidError } from "core/utils"; +import { type Context, Hono } from "hono"; +import { MODULE_NAMES, type ModuleKey, getDefaultConfig } from "modules/ModuleManager"; +import * as SystemPermissions from "modules/permissions"; +import { generateOpenAPI } from "modules/server/openapi"; +import type { App } from "../../App"; + +const booleanLike = Type.Transform(Type.String()) + .Decode((v) => v === "1") + .Encode((v) => (v ? "1" : "0")); + +export class SystemController implements ClassController { + constructor(private readonly app: App) {} + + get ctx() { + return this.app.modules.ctx(); + } + + private registerConfigController(client: Hono): void { + const hono = new Hono(); + + /*hono.use("*", async (c, next) => { + //this.ctx.guard.throwUnlessGranted(SystemPermissions.configRead); + console.log("perm?", this.ctx.guard.hasPermission(SystemPermissions.configRead)); + return next(); + });*/ + + hono.get( + "/:module?", + tb("param", Type.Object({ module: Type.Optional(StringEnum(MODULE_NAMES)) })), + tb( + "query", + Type.Object({ + secrets: Type.Optional(booleanLike) + }) + ), + async (c) => { + // @todo: allow secrets if authenticated user is admin + const { secrets } = c.req.valid("query"); + const { module } = c.req.valid("param"); + + secrets && this.ctx.guard.throwUnlessGranted(SystemPermissions.configReadSecrets); + + const config = this.app.toJSON(secrets); + + return c.json( + module + ? { + version: this.app.version(), + module, + config: config[module] + } + : config + ); + } + ); + + async function handleConfigUpdateResponse(c: Context, cb: () => Promise) { + try { + return c.json(await cb(), { status: 202 }); + } catch (e) { + console.error(e); + + if (e instanceof TypeInvalidError) { + return c.json({ success: false, errors: e.errors }, { status: 400 }); + } + + return c.json({ success: false }, { status: 500 }); + } + } + + hono.post( + "/set/:module", + tb( + "query", + Type.Object({ + force: Type.Optional(booleanLike) + }) + ), + async (c) => { + const module = c.req.param("module") as any; + const { force } = c.req.valid("query"); + const value = await c.req.json(); + + this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite); + + return await handleConfigUpdateResponse(c, async () => { + // you must explicitly set force to override existing values + // because omitted values gets removed + if (force === true) { + await this.app.mutateConfig(module).set(value); + } else { + await this.app.mutateConfig(module).patch("", value); + } + return { + success: true, + module, + config: this.app.module[module].config + }; + }); + } + ); + + hono.post("/add/:module/:path", async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path") as string; + + this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite); + + const moduleConfig = this.app.mutateConfig(module); + if (moduleConfig.has(path)) { + return c.json({ success: false, path, error: "Path already exists" }, { status: 400 }); + } + console.log("-- add", module, path, value); + + return await handleConfigUpdateResponse(c, async () => { + await moduleConfig.patch(path, value); + return { + success: true, + module, + config: this.app.module[module].config + }; + }); + }); + + hono.patch("/patch/:module/:path", async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path"); + + this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite); + + return await handleConfigUpdateResponse(c, async () => { + await this.app.mutateConfig(module).patch(path, value); + return { + success: true, + module, + config: this.app.module[module].config + }; + }); + }); + + hono.put("/overwrite/:module/:path", async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const value = await c.req.json(); + const path = c.req.param("path"); + + this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite); + + return await handleConfigUpdateResponse(c, async () => { + await this.app.mutateConfig(module).overwrite(path, value); + return { + success: true, + module, + config: this.app.module[module].config + }; + }); + }); + + hono.delete("/remove/:module/:path", async (c) => { + // @todo: require auth (admin) + const module = c.req.param("module") as any; + const path = c.req.param("path")!; + + this.ctx.guard.throwUnlessGranted(SystemPermissions.configWrite); + + return await handleConfigUpdateResponse(c, async () => { + await this.app.mutateConfig(module).remove(path); + return { + success: true, + module, + config: this.app.module[module].config + }; + }); + }); + + client.route("/config", hono); + } + + getController(): Hono { + const hono = new Hono(); + + this.registerConfigController(hono); + + hono.get( + "/schema/:module?", + tb( + "query", + Type.Object({ + config: Type.Optional(booleanLike), + secrets: Type.Optional(booleanLike) + }) + ), + async (c) => { + const module = c.req.param("module") as ModuleKey | undefined; + const { config, secrets } = c.req.valid("query"); + this.ctx.guard.throwUnlessGranted(SystemPermissions.schemaRead); + config && this.ctx.guard.throwUnlessGranted(SystemPermissions.configRead); + secrets && this.ctx.guard.throwUnlessGranted(SystemPermissions.configReadSecrets); + + const { version, ...schema } = this.app.getSchema(); + + if (module) { + return c.json({ + module, + version, + schema: schema[module], + config: config ? this.app.module[module].toJSON(secrets) : undefined + }); + } + + return c.json({ + module, + version, + schema, + config: config ? this.app.toJSON(secrets) : undefined, + permissions: this.app.modules.ctx().guard.getPermissionNames() + }); + } + ); + + hono.post( + "/build", + tb( + "query", + Type.Object({ + sync: Type.Optional(booleanLike), + drop: Type.Optional(booleanLike), + save: Type.Optional(booleanLike) + }) + ), + async (c) => { + const { sync, drop, save } = c.req.valid("query") as Record; + this.ctx.guard.throwUnlessGranted(SystemPermissions.build); + + await this.app.build({ sync, drop, save }); + return c.json({ success: true, options: { sync, drop, save } }); + } + ); + + hono.get("/ping", async (c) => { + //console.log("c", c); + try { + // @ts-ignore @todo: fix with env + const context: any = c.req.raw.cf ? c.req.raw.cf : c.env.cf; + const cf = { + colo: context.colo, + city: context.city, + postal: context.postalCode, + region: context.region, + regionCode: context.regionCode, + continent: context.continent, + country: context.country, + eu: context.isEUCountry, + lat: context.latitude, + lng: context.longitude, + timezone: context.timezone + }; + return c.json({ pong: true }); + } catch (e) { + return c.json({ pong: true }); + } + }); + + hono.get("/info", async (c) => { + return c.json({ + version: this.app.version(), + test: 2, + // @ts-ignore + app: !!c.var.app + }); + }); + + hono.get("/openapi.json", async (c) => { + //const config = this.app.toJSON(); + const config = JSON.parse(getDefaultConfig() as any); + return c.json(generateOpenAPI(config)); + }); + + /*hono.get("/test/sql", async (c) => { + // @ts-ignore + const ai = c.env?.AI as Ai; + const messages = [ + { role: "system", content: "You are a friendly assistant" }, + { + role: "user", + content: "just say hello" + } + ]; + + const stream = await ai.run("@cf/meta/llama-3.1-8b-instruct", { + messages, + stream: true + }); + + return new Response(stream, { + headers: { "content-type": "text/event-stream" } + }); + });*/ + + return hono; + } +} diff --git a/app/src/modules/server/openapi.ts b/app/src/modules/server/openapi.ts new file mode 100644 index 0000000..8db915d --- /dev/null +++ b/app/src/modules/server/openapi.ts @@ -0,0 +1,312 @@ +import { Type } from "core/utils"; +import type { ModuleConfigs } from "modules/ModuleManager"; +import type { OpenAPIV3 as OAS } from "openapi-types"; + +function prefixPaths(paths: OAS.PathsObject, prefix: string): OAS.PathsObject { + const result: OAS.PathsObject = {}; + for (const [path, pathItem] of Object.entries(paths)) { + result[`${prefix}${path}`] = pathItem; + } + return result; +} + +function systemRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } { + const tags = ["system"]; + const paths: OAS.PathsObject = { + "/ping": { + get: { + summary: "Ping", + responses: { + "200": { + description: "Pong", + content: { + "application/json": { + schema: Type.Object({ + pong: Type.Boolean({ default: true }) + }) + } + } + } + }, + tags + } + }, + "/config": { + get: { + summary: "Get config", + responses: { + "200": { + description: "Config", + content: { + "application/json": { + schema: Type.Object({ + version: Type.Number() as any, + server: Type.Object({}), + data: Type.Object({}), + auth: Type.Object({}), + flows: Type.Object({}), + media: Type.Object({}) + }) + } + } + } + }, + tags + } + }, + "/schema": { + get: { + summary: "Get config", + responses: { + "200": { + description: "Config", + content: { + "application/json": { + schema: Type.Object({ + version: Type.Number() as any, + schema: Type.Object({ + server: Type.Object({}), + data: Type.Object({}), + auth: Type.Object({}), + flows: Type.Object({}), + media: Type.Object({}) + }) + }) + } + } + } + }, + tags + } + } + }; + + return { paths: prefixPaths(paths, "/api/system") }; +} + +function dataRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } { + const schemas = { + entityData: Type.Object({ + id: Type.Number() as any + }) + }; + const repoManyResponses: OAS.ResponsesObject = { + "200": { + description: "List of entities", + content: { + "application/json": { + schema: Type.Array(schemas.entityData) + } + } + } + }; + const repoSingleResponses: OAS.ResponsesObject = { + "200": { + description: "Entity", + content: { + "application/json": { + schema: schemas.entityData + } + } + } + }; + const params = { + entity: { + name: "entity", + in: "path", + required: true, + schema: Type.String() + }, + entityId: { + name: "id", + in: "path", + required: true, + schema: Type.Number() as any + } + }; + + const tags = ["data"]; + const paths: OAS.PathsObject = { + "/{entity}": { + get: { + summary: "List entities", + parameters: [params.entity], + responses: repoManyResponses, + tags + }, + post: { + summary: "Create entity", + parameters: [params.entity], + requestBody: { + content: { + "application/json": { + schema: Type.Object({}) + } + } + }, + responses: repoSingleResponses, + tags + } + }, + "/{entity}/{id}": { + get: { + summary: "Get entity", + parameters: [params.entity, params.entityId], + responses: repoSingleResponses, + tags + }, + patch: { + summary: "Update entity", + parameters: [params.entity, params.entityId], + requestBody: { + content: { + "application/json": { + schema: Type.Object({}) + } + } + }, + responses: repoSingleResponses, + tags + }, + delete: { + summary: "Delete entity", + parameters: [params.entity, params.entityId], + responses: { + "200": { + description: "Entity deleted" + } + }, + tags + } + } + }; + + return { paths: prefixPaths(paths, config.data.basepath!) }; +} + +function authRoutes(config: ModuleConfigs): { paths: OAS.Document["paths"] } { + const schemas = { + user: Type.Object({ + id: Type.String(), + email: Type.String(), + name: Type.String() + }) + }; + + const tags = ["auth"]; + const paths: OAS.PathsObject = { + "/password/login": { + post: { + summary: "Login", + requestBody: { + content: { + "application/json": { + schema: Type.Object({ + email: Type.String(), + password: Type.String() + }) + } + } + }, + responses: { + "200": { + description: "User", + content: { + "application/json": { + schema: Type.Object({ + user: schemas.user + }) + } + } + } + }, + tags + } + }, + "/password/register": { + post: { + summary: "Register", + requestBody: { + content: { + "application/json": { + schema: Type.Object({ + email: Type.String(), + password: Type.String() + }) + } + } + }, + responses: { + "200": { + description: "User", + content: { + "application/json": { + schema: Type.Object({ + user: schemas.user + }) + } + } + } + }, + tags + } + }, + "/me": { + get: { + summary: "Get me", + responses: { + "200": { + description: "User", + content: { + "application/json": { + schema: Type.Object({ + user: schemas.user + }) + } + } + } + }, + tags + } + }, + "/strategies": { + get: { + summary: "Get auth strategies", + responses: { + "200": { + description: "Strategies", + content: { + "application/json": { + schema: Type.Object({ + strategies: Type.Object({}) + }) + } + } + } + }, + tags + } + } + }; + + return { paths: prefixPaths(paths, config.auth.basepath!) }; +} + +export function generateOpenAPI(config: ModuleConfigs): OAS.Document { + const system = systemRoutes(config); + const data = dataRoutes(config); + const auth = authRoutes(config); + + return { + openapi: "3.1.0", + info: { + title: "bknd API", + version: "0.0.0" + }, + paths: { + ...system.paths, + ...data.paths, + ...auth.paths + } + }; +} diff --git a/app/src/plugins/cloudflare/image-optimization-plugin.ts b/app/src/plugins/cloudflare/image-optimization-plugin.ts new file mode 100644 index 0000000..4171a36 --- /dev/null +++ b/app/src/plugins/cloudflare/image-optimization-plugin.ts @@ -0,0 +1,83 @@ +import type { App } from "../../App"; + +export type ImageOptimizationPluginOptions = { + accessUrl?: string; + resolvePath?: string; + autoFormat?: boolean; + devBypass?: string; +}; + +export function ImageOptimizationPlugin({ + accessUrl = "/_plugin/image/optimize", + resolvePath = "/api/media/file", + autoFormat = true, + devBypass +}: ImageOptimizationPluginOptions = {}) { + const disallowedAccessUrls = ["/api", "/admin", "/_optimize"]; + if (disallowedAccessUrls.includes(accessUrl) || accessUrl.length < 2) { + throw new Error(`Disallowed accessUrl: ${accessUrl}`); + } + + return (app: App) => { + app.module.server.client.get(`${accessUrl}/:path{.+$}`, async (c) => { + const request = c.req.raw; + const url = new URL(request.url); + + if (devBypass) { + return c.redirect(devBypass + url.pathname + url.search, 302); + } + + const storage = app.module.media?.storage; + if (!storage) { + throw new Error("No media storage configured"); + } + + const path = c.req.param("path"); + if (!path) { + throw new Error("No url provided"); + } + + const imageURL = `${url.origin}${resolvePath}/${path}`; + const metadata = await storage.objectMetadata(path); + + // Cloudflare-specific options are in the cf object. + const params = Object.fromEntries(url.searchParams.entries()); + const options: RequestInitCfPropertiesImage = {}; + + // Copy parameters from query string to request options. + // You can implement various different parameters here. + if ("fit" in params) options.fit = params.fit as any; + if ("width" in params) options.width = Number.parseInt(params.width); + if ("height" in params) options.height = Number.parseInt(params.height); + if ("quality" in params) options.quality = Number.parseInt(params.quality); + + // Your Worker is responsible for automatic format negotiation. Check the Accept header. + if (autoFormat) { + const accept = request.headers.get("Accept")!; + if (/image\/avif/.test(accept)) { + options.format = "avif"; + } else if (/image\/webp/.test(accept)) { + options.format = "webp"; + } + } + + // Build a request that passes through request headers + const imageRequest = new Request(imageURL, { + headers: request.headers + }); + + // Returning fetch() with resizing options will pass through response with the resized image. + const res = await fetch(imageRequest, { cf: { image: options } }); + + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: { + "Cache-Control": "public, max-age=600", + "Content-Type": metadata.type, + "Content-Length": metadata.size.toString() + } + }); + }); + }; +} diff --git a/app/src/ui/Admin.tsx b/app/src/ui/Admin.tsx new file mode 100644 index 0000000..d0fe283 --- /dev/null +++ b/app/src/ui/Admin.tsx @@ -0,0 +1,36 @@ +import { MantineProvider } from "@mantine/core"; +import { Notifications } from "@mantine/notifications"; +import React from "react"; +import { BkndProvider, ClientProvider, useBknd } from "./client"; +import { createMantineTheme } from "./lib/mantine/theme"; +import { BkndModalsProvider } from "./modals"; +import { Routes } from "./routes"; + +export default function Admin({ + baseUrl: baseUrlOverride, + withProvider = false +}: { baseUrl?: string; withProvider?: boolean }) { + const Component = ( + + + + ); + return withProvider ? ( + {Component} + ) : ( + Component + ); +} + +function AdminInternal() { + const b = useBknd(); + const theme = b.app.getAdminConfig().color_scheme; + return ( + + + + + + + ); +} diff --git a/app/src/ui/assets/favicon.ico b/app/src/ui/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..c1a946d533a71c15e51fc4cade87289ca467b57d GIT binary patch literal 15086 zcmeI3`*+mE6~H&yJT}=RyV=ci12qaFP%EgQVoRW{Xhk^@c}Z;%A4d^I@qsT68WgDr zC?W!C#R3W_pz@Gs1lrn`3M%~v`jh`ed)nUK&m@yhzWJ{E-E1D4bDDF8`F_7UGxs@n z?%aC`g$hF9(D?Bo?VCe;E)9jQ357z<&7JS@Q0VXaEv`MkFDeR!R_lP#I!3327HZ?W z`5W2Mx&OAG8y-G+zG;@W^VxYWGj{mhuUjJ>a%)h5nZk-jC2Op2g z&VvnxmVBoH+!ranvFbumduv)jK}bf8DV8^XS8HUFuQZ@9>))x7xP!Z;f&SW`MC8El zQ~fRn@Hq8thu&txE9U)Db;iZneXgi1l#TDJ40P>m)(twAy>|3$O5WL1C$GF!WBOyN z4}1@@2IJ6o7iagmxVS*pY_2i939?=`V}O0|4=I^XMJbJ@jQt#!oEm0Tl8=u!_Sg&TkDuODCdkUl&GvNst}2z6UQf!g zvmJOmwX)jaXYozfqo1QMFEMgt%@gN3&YR{D#XV?d4fX_M{BTY84O-D(KRd?vy(b#w zu0>UH(Iv%lo9f8JPsP*P-E?$Za%r)={6^C3QE*4kCjL0mwFl5yZ1#tT`~K{-vP56d zwc?WQ`z{^M?sw!+Syd<#Cq;~17$=`*9oCi%hW&u{R4^9Oa^ ze5Gvgwqx1LpPW)XQGWClzCqSP#%t6*aV}e2dehf8hGo|g-J_lMG#47+DK-*6jdO^1 z?|RUN4LYMbyGrdx*mGuPOG86bk?aQl9Qn`uje2%XyQSRd^dvoN+twwGZbAqAh8A=l zKlSd#Rptyx=Whh1V@x0D5BP1}RVP2tv(<%*eOspI{`Sx6W%+Ls#=qm79C2B(O#XS9 z-1AV!8Q{*z_IbM6eS8Y9oU?uBT=#>`ePG0>V!3>LiOiT2mF24wvg1I#9Q{km$b@su zZ(fjZrI7+zsWL_%v&GjPkIr8=IpT{)mHIL-tW6jj!5s9+?hE?upVrI8BmK|*Of*cL zR+g1-1A_zTmsX-S({M+W#Q@VGj46u|F!2`yY+TA=NeR5ZtF` zs!rbA8kISUGxwKm`|6D^9AsUKQ*ZiA`r@t$y{+@2g1oplfEzSnQ|MWX-re`^{K0$% zFKiR`hqI6O!NeyMYt=@ckx^Ml*KBZN0 zKcZ*tqtC_7eX*ppKq6%Y(loqCQrd_|VDopUbWvz*#yFaoyg~0{;8d-yf;|vWv6(%z1nG zPr>42y}=>ZIruhr-s>M@;O@ma3!S;b>96_Vw&!SrO#WF}x4UAg+M!!+k4j$K&dxmq z4a5kD5w^~a3chyR+NAMy@|pLpOu-YMrpv!kyE;^Gcj?HJ=b@YfCihMbK5uPC2HdE9 zpax<{)^8}@v)jK(Q9nZs$^~shp%wytr`BiI$*)u4GkSn(c~Pj1zJP!J|JR=!Xk&5Z z^BH>OJ9S#k+NNqNuytBm%jNx3d9`RetL@*~25P?p>nm-*rXMJ{UO3SGuR(uaht{WGtTuN~)l=h-6Z_xtVQw)*d>9)~ z_&$FIp10gk*5e-Vz|xqR-y1yv9x(y(ue4pCr_W;`Z(@U%cH7_gNX*nUcyWhcR#$9} z8anMbdU6@$PwiS0G#9#$oTvWgrH8f1do-lNW;}ThVz^UpEHk-M?m&z&bpkq{y!pKk zR~eoW8=o;dD$l=~F!5@}fy-;})=ETU@D@Jtcyx;2yzuS&7R}iaCw%0YxZxeSjXM@r znmjf+5aMyfaxI>AZF=&=Q#5A2^FV`H|M3;oQsU%}sj+x__xJ0Q#oh9fcVJQ%@Qc=R zA)iU!#f9sArlw-fuPbEc9Tlb)0zMLhTdFaZ-sV!MYuT!M0UEsU{NBO!bCnsgM&{(Y zEsuNC!w+JKYs!dEEy+gOXHAv~; zql3iSgTRO9OB9~F7Wp1@Xz>#z?khL!H1>!%sW&&w zT+R^kmqBT_`~U~+^*TrY>~FSb47EMzA^BeR13Ez+4)&KE5p{Oi!gkkBxBnOe?c~Lm zJRR@Kf3P?F&L6vGu065`C+Z2PPo#!%pyG+tTT&xN9v$0#qxwYTA$!Z;YiX*VO0|p)6*W=tf z@Ic?cp}VDR5Ti4|CQd%ynbgx^_{dl5N$@~@VE>GDmiVd>`?n@D~1w*B>{783dK zo#w*!J~NNIEj}tacQZb~Jl0_>b@jFm5c#?C^}_OdNADKg-%;C0eQCh?-Ru*(ZT` Promise; + actions: ReturnType; + app: AppReduced; +}; + +const BkndContext = createContext(undefined!); +export type { TSchemaActions }; + +export function BkndProvider({ + includeSecrets = false, + children +}: { includeSecrets?: boolean; children: any }) { + const [withSecrets, setWithSecrets] = useState(includeSecrets); + const [schema, setSchema] = useState(); + const client = useClient(); + + async function fetchSchema(_includeSecrets: boolean = false) { + if (withSecrets) return; + const { body } = await client.api.system.readSchema({ + config: true, + secrets: _includeSecrets + }); + console.log("--schema fetched", body); + setSchema(body as any); + setWithSecrets(_includeSecrets); + } + + async function requireSecrets() { + if (withSecrets) return; + await fetchSchema(true); + } + + useEffect(() => { + if (schema?.schema) return; + fetchSchema(includeSecrets); + }, []); + + if (!schema?.schema) return null; + const app = new AppReduced(schema.config as any); + + const actions = getSchemaActions({ client, setSchema }); + + return ( + + {children} + + ); +} + +export function useBknd({ withSecrets }: { withSecrets?: boolean } = {}): BkndContext { + const ctx = useContext(BkndContext); + if (withSecrets) ctx.requireSecrets(); + + return ctx; +} + +/* +type UseSchemaForType = { + version: number; + schema: ModuleSchemas[Key]; + config: ModuleConfigs[Key]; +}; + +export function useSchemaFor(module: Key): UseSchemaForType { + //const app = useApp(); + const { version, schema, config } = useSchema(); + return { + version, + schema: schema[module], + config: config[module] + }; +}*/ diff --git a/app/src/ui/client/ClientProvider.tsx b/app/src/ui/client/ClientProvider.tsx new file mode 100644 index 0000000..cd31f42 --- /dev/null +++ b/app/src/ui/client/ClientProvider.tsx @@ -0,0 +1,82 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { createContext, useContext, useEffect, useState } from "react"; +import { AppQueryClient } from "./utils/AppQueryClient"; + +const ClientContext = createContext<{ baseUrl: string; client: AppQueryClient }>({ + baseUrl: undefined +} as any); + +export const queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + refetchOnWindowFocus: false + } + } +}); + +export const ClientProvider = ({ children, baseUrl }: { children?: any; baseUrl?: string }) => { + const [actualBaseUrl, setActualBaseUrl] = useState(null); + + try { + const _ctx_baseUrl = useBaseUrl(); + if (_ctx_baseUrl) { + console.warn("wrapped many times"); + setActualBaseUrl(_ctx_baseUrl); + } + } catch (e) { + console.error("error", e); + } + + useEffect(() => { + // Only set base URL if running on the client side + if (typeof window !== "undefined") { + setActualBaseUrl(baseUrl || window.location.origin); + } + }, [baseUrl]); + + if (!actualBaseUrl) { + // Optionally, return a fallback during SSR rendering + return null; // or a loader/spinner if desired + } + + console.log("client provider11 with", { baseUrl, fallback: actualBaseUrl }); + const client = createClient(actualBaseUrl); + + return ( + + + {children} + + + ); +}; + +export function createClient(baseUrl: string = window.location.origin) { + return new AppQueryClient(baseUrl); +} + +export function createOrUseClient(baseUrl: string = window.location.origin) { + const context = useContext(ClientContext); + if (!context) { + console.warn("createOrUseClient returned a new client"); + return createClient(baseUrl); + } + + return context.client; +} + +export const useClient = () => { + const context = useContext(ClientContext); + if (!context) { + throw new Error("useClient must be used within a ClientProvider"); + } + + console.log("useClient", context.baseUrl); + return context.client; +}; + +export const useBaseUrl = () => { + const context = useContext(ClientContext); + return context.baseUrl; +}; diff --git a/app/src/ui/client/index.ts b/app/src/ui/client/index.ts new file mode 100644 index 0000000..7c914a7 --- /dev/null +++ b/app/src/ui/client/index.ts @@ -0,0 +1,4 @@ +export { ClientProvider, useClient, useBaseUrl } from "./ClientProvider"; +export { BkndProvider, useBknd } from "./BkndProvider"; + +export { useAuth } from "./schema/auth/use-auth"; diff --git a/app/src/ui/client/schema/actions.ts b/app/src/ui/client/schema/actions.ts new file mode 100644 index 0000000..fc10000 --- /dev/null +++ b/app/src/ui/client/schema/actions.ts @@ -0,0 +1,190 @@ +import { set } from "lodash-es"; +import type { ModuleConfigs } from "../../../modules"; +import type { AppQueryClient } from "../utils/AppQueryClient"; + +export type SchemaActionsProps = { + client: AppQueryClient; + setSchema: React.Dispatch>; +}; + +export type TSchemaActions = ReturnType; + +export function getSchemaActions({ client, setSchema }: SchemaActionsProps) { + const baseUrl = client.baseUrl; + const token = client.auth().state()?.token; + return { + set: async ( + module: keyof ModuleConfigs, + value: ModuleConfigs[Module], + force?: boolean + ) => { + const res = await fetch( + `${baseUrl}/api/system/config/set/${module}?force=${force ? 1 : 0}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}` + }, + body: JSON.stringify(value) + } + ); + if (res.ok) { + const data = (await res.json()) as any; + console.log("update config set", module, data); + if (data.success) { + setSchema((prev) => { + if (!prev) return prev; + return { + ...prev, + config: { + ...prev.config, + [module]: data.config + } + }; + }); + } + + return data.success; + } + + return false; + }, + patch: async ( + module: keyof ModuleConfigs, + path: string, + value: any + ): Promise => { + const res = await fetch(`${baseUrl}/api/system/config/patch/${module}/${path}`, { + method: "PATCH", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}` + }, + body: JSON.stringify(value) + }); + if (res.ok) { + const data = (await res.json()) as any; + console.log("update config patch", module, path, data); + if (data.success) { + setSchema((prev) => { + if (!prev) return prev; + return { + ...prev, + config: { + ...prev.config, + [module]: data.config + } + }; + }); + } + + return data.success; + } + + return false; + }, + overwrite: async ( + module: keyof ModuleConfigs, + path: string, + value: any + ) => { + const res = await fetch(`${baseUrl}/api/system/config/overwrite/${module}/${path}`, { + method: "PUT", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}` + }, + body: JSON.stringify(value) + }); + if (res.ok) { + const data = (await res.json()) as any; + console.log("update config overwrite", module, path, data); + if (data.success) { + setSchema((prev) => { + if (!prev) return prev; + return { + ...prev, + config: { + ...prev.config, + [module]: data.config + } + }; + }); + } + + return data.success; + } + + return false; + }, + add: async ( + module: keyof ModuleConfigs, + path: string, + value: any + ) => { + const res = await fetch(`${baseUrl}/api/system/config/add/${module}/${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}` + }, + body: JSON.stringify(value) + }); + if (res.ok) { + const data = (await res.json()) as any; + console.log("update config add", module, data); + + if (data.success) { + setSchema((prev) => { + if (!prev) return prev; + return { + ...prev, + config: { + ...prev.config, + [module]: data.config + } + }; + }); + } + + return data.success; + } + + return false; + }, + remove: async ( + module: keyof ModuleConfigs, + path: string + ) => { + const res = await fetch(`${baseUrl}/api/system/config/remove/${module}/${path}`, { + method: "DELETE", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}` + } + }); + if (res.ok) { + const data = (await res.json()) as any; + console.log("update config remove", module, data); + + if (data.success) { + setSchema((prev) => { + if (!prev) return prev; + return { + ...prev, + config: { + ...prev.config, + [module]: data.config + } + }; + }); + } + + return data.success; + } + + return false; + } + }; +} diff --git a/app/src/ui/client/schema/auth/use-auth.ts b/app/src/ui/client/schema/auth/use-auth.ts new file mode 100644 index 0000000..42276da --- /dev/null +++ b/app/src/ui/client/schema/auth/use-auth.ts @@ -0,0 +1,107 @@ +import { Api } from "Api"; +import type { AuthResponse } from "auth"; +import type { AppAuthSchema } from "auth/auth-schema"; +import type { ApiResponse } from "modules/ModuleApi"; +import { useEffect, useState } from "react"; +import { + createClient, + createOrUseClient, + queryClient, + useBaseUrl, + useClient +} from "../../ClientProvider"; + +type LoginData = { + email: string; + password: string; + [key: string]: any; +}; + +type UseAuth = { + data: (AuthResponse & { verified: boolean }) | undefined; + user: AuthResponse["user"] | undefined; + token: AuthResponse["token"] | undefined; + verified: boolean; + login: (data: LoginData) => Promise>; + register: (data: LoginData) => Promise>; + logout: () => void; + verify: () => void; + setToken: (token: string) => void; +}; + +// @todo: needs to use a specific auth endpoint to get strategy information +export const useAuth = (options?: { baseUrl?: string }): UseAuth => { + const ctxBaseUrl = useBaseUrl(); + //const client = useClient(); + const client = createOrUseClient(options?.baseUrl ? options?.baseUrl : ctxBaseUrl); + const authState = client.auth().state(); + const [authData, setAuthData] = useState(authState); + const verified = authState?.verified ?? false; + + async function login(input: LoginData) { + const res = await client.auth().login(input); + if (res.res.ok && res.data && "user" in res.data) { + setAuthData(res.data); + } + return res; + } + + async function register(input: LoginData) { + const res = await client.auth().register(input); + if (res.res.ok && res.data && "user" in res.data) { + setAuthData(res.data); + } + return res; + } + + function setToken(token: string) { + setAuthData(client.auth().setToken(token) as any); + } + + async function logout() { + await client.auth().logout(); + setAuthData(undefined); + queryClient.clear(); + } + + async function verify() { + await client.auth().verify(); + setAuthData(client.auth().state()); + } + + return { + data: authData, + user: authData?.user, + token: authData?.token, + verified, + login, + register, + logout, + setToken, + verify + }; +}; + +export const useAuthStrategies = (options?: { baseUrl?: string }): { + strategies: AppAuthSchema["strategies"]; + loading: boolean; +} => { + const [strategies, setStrategies] = useState(); + const ctxBaseUrl = useBaseUrl(); + const api = new Api({ + host: options?.baseUrl ? options?.baseUrl : ctxBaseUrl, + tokenStorage: "localStorage" + }); + + useEffect(() => { + (async () => { + const res = await api.auth.strategies(); + console.log("res", res); + if (res.res.ok) { + setStrategies(res.body.strategies); + } + })(); + }, [options?.baseUrl]); + + return { strategies, loading: !strategies }; +}; diff --git a/app/src/ui/client/schema/auth/use-bknd-auth.ts b/app/src/ui/client/schema/auth/use-bknd-auth.ts new file mode 100644 index 0000000..87990dd --- /dev/null +++ b/app/src/ui/client/schema/auth/use-bknd-auth.ts @@ -0,0 +1,33 @@ +import { useBknd } from "ui/client"; + +export function useBkndAuth() { + //const client = useClient(); + const { config, app, schema, actions: bkndActions } = useBknd(); + + const actions = { + roles: { + add: async (name: string, data: any = {}) => { + console.log("add role", name, data); + return await bkndActions.add("auth", `roles.${name}`, data); + }, + patch: async (name: string, data: any) => { + console.log("patch role", name, data); + return await bkndActions.patch("auth", `roles.${name}`, data); + }, + delete: async (name: string) => { + console.log("delete role", name); + if (window.confirm(`Are you sure you want to delete the role "${name}"?`)) { + return await bkndActions.remove("auth", `roles.${name}`); + } + } + } + }; + const $auth = {}; + + return { + $auth, + config: config.auth, + schema: schema.auth, + actions + }; +} diff --git a/app/src/ui/client/schema/data/use-bknd-data.ts b/app/src/ui/client/schema/data/use-bknd-data.ts new file mode 100644 index 0000000..0768d7f --- /dev/null +++ b/app/src/ui/client/schema/data/use-bknd-data.ts @@ -0,0 +1,115 @@ +import { Type, TypeInvalidError, parse, transformObject } from "core/utils"; +import type { Entity } from "data"; +import { AppData } from "data/AppData"; +import { + type TAppDataEntity, + type TAppDataEntityFields, + type TAppDataField, + type TAppDataRelation, + entitiesSchema, + entityFields, + fieldsSchema, + relationsSchema +} from "data/data-schema"; +import { useBknd } from "ui/client"; +import type { TSchemaActions } from "ui/client/schema/actions"; + +export function useBkndData() { + const { config, app, schema, actions: bkndActions } = useBknd(); + + // @todo: potentially store in ref, so it doesn't get recomputed? or use memo? + const entities = transformObject(config.data.entities ?? {}, (entity, name) => { + return AppData.constructEntity(name, entity); + }); + + const actions = { + entity: { + add: async (name: string, data: TAppDataEntity) => { + console.log("create entity", { data }); + const validated = parse(entitiesSchema, data, { + skipMark: true, + forceParse: true + }); + console.log("validated", validated); + // @todo: check for existing? + return await bkndActions.add("data", `entities.${name}`, validated); + }, + patch: (entityName: string) => { + const entity = entities[entityName]; + if (!entity) { + throw new Error(`Entity "${entityName}" not found`); + } + + return { + config: async (partial: Partial): Promise => { + console.log("patch config", entityName, partial); + return await bkndActions.patch("data", `entities.${entityName}.config`, partial); + }, + fields: entityFieldActions(bkndActions, entityName) + }; + } + }, + relations: { + add: async (relation: TAppDataRelation) => { + console.log("create relation", { relation }); + const name = crypto.randomUUID(); + const validated = parse(Type.Union(relationsSchema), relation, { + skipMark: true, + forceParse: true + }); + console.log("validated", validated); + return await bkndActions.add("data", `relations.${name}`, validated); + } + } + }; + const $data = { + entity: (name: string) => entities[name] + }; + + return { + $data, + entities, + relations: app.relations, + config: config.data, + schema: schema.data, + actions + }; +} + +function entityFieldActions(bkndActions: TSchemaActions, entityName: string) { + return { + add: async (name: string, field: TAppDataField) => { + console.log("create field", { name, field }); + const validated = parse(fieldsSchema, field, { + skipMark: true, + forceParse: true + }); + console.log("validated", validated); + return await bkndActions.add("data", `entities.${entityName}.fields.${name}`, validated); + }, + patch: () => null, + set: async (fields: TAppDataEntityFields) => { + console.log("set fields", entityName, fields); + try { + const validated = parse(entityFields, fields, { + skipMark: true, + forceParse: true + }); + const res = await bkndActions.overwrite( + "data", + `entities.${entityName}.fields`, + validated + ); + console.log("res", res); + //bkndActions.set("data", "entities", fields); + } catch (e) { + console.error("error", e); + if (e instanceof TypeInvalidError) { + alert("Error updating fields: " + e.firstToString()); + } else { + alert("An error occured, check console. There will be nice error handling soon."); + } + } + } + }; +} diff --git a/app/src/ui/client/schema/flows/use-flows.ts b/app/src/ui/client/schema/flows/use-flows.ts new file mode 100644 index 0000000..cdaa220 --- /dev/null +++ b/app/src/ui/client/schema/flows/use-flows.ts @@ -0,0 +1,23 @@ +import { type Static, parse } from "core/utils"; +import { type TAppFlowSchema, flowSchema } from "flows/flows-schema"; +import { useBknd } from "../../BkndProvider"; +import { useClient } from "../../ClientProvider"; + +export function useFlows() { + const client = useClient(); + const { config, app, actions: bkndActions } = useBknd(); + + const actions = { + flow: { + create: async (name: string, data: TAppFlowSchema) => { + console.log("would create", name, data); + const parsed = parse(flowSchema, data, { skipMark: true, forceParse: true }); + console.log("parsed", parsed); + const res = await bkndActions.add("flows", `flows.${name}`, parsed); + console.log("res", res); + } + } + }; + + return { flows: app.flows, config: config.flows, actions }; +} diff --git a/app/src/ui/client/schema/system/use-bknd-system.ts b/app/src/ui/client/schema/system/use-bknd-system.ts new file mode 100644 index 0000000..fe4c98c --- /dev/null +++ b/app/src/ui/client/schema/system/use-bknd-system.ts @@ -0,0 +1,40 @@ +import { useBknd } from "ui/client"; + +export function useBkndSystem() { + const { config, schema, actions: bkndActions } = useBknd(); + const theme = config.server.admin.color_scheme ?? "light"; + + const actions = { + theme: { + set: async (scheme: "light" | "dark") => { + return await bkndActions.patch("server", "admin", { + color_scheme: scheme + }); + }, + toggle: async () => { + return await bkndActions.patch("server", "admin", { + color_scheme: theme === "light" ? "dark" : "light" + }); + } + } + }; + const $system = {}; + + return { + $system, + config: config.server, + schema: schema.server, + theme, + actions + }; +} + +export function useBkndSystemTheme() { + const $sys = useBkndSystem(); + + return { + theme: $sys.theme, + set: $sys.actions.theme.set, + toggle: () => $sys.actions.theme.toggle() + }; +} diff --git a/app/src/ui/client/use-theme.ts b/app/src/ui/client/use-theme.ts new file mode 100644 index 0000000..844dcfd --- /dev/null +++ b/app/src/ui/client/use-theme.ts @@ -0,0 +1,8 @@ +import { useBknd } from "ui"; + +export function useTheme(): { theme: "light" | "dark" } { + const b = useBknd(); + const theme = b.app.getAdminConfig().color_scheme as any; + + return { theme }; +} diff --git a/app/src/ui/client/utils/AppQueryClient.ts b/app/src/ui/client/utils/AppQueryClient.ts new file mode 100644 index 0000000..76b5896 --- /dev/null +++ b/app/src/ui/client/utils/AppQueryClient.ts @@ -0,0 +1,211 @@ +import { + type QueryObserverOptions, + type UseQueryResult, + keepPreviousData, + useMutation, + useQuery +} from "@tanstack/react-query"; +import type { AuthResponse } from "auth"; +import type { EntityData, RepoQuery, RepositoryResponse } from "data"; +import { Api } from "../../../Api"; +import type { ApiResponse } from "../../../modules/ModuleApi"; +import { queryClient } from "../ClientProvider"; + +export class AppQueryClient { + api: Api; + constructor(public baseUrl: string) { + this.api = new Api({ + host: baseUrl, + tokenStorage: "localStorage" + }); + } + + queryOptions(options?: Partial): Partial { + return { + staleTime: 1000 * 60 * 5, + placeholderData: keepPreviousData, + ...options + }; + } + + auth = () => { + return { + state: (): (AuthResponse & { verified: boolean }) | undefined => { + return this.api.getAuthState() as any; + }, + login: async (data: { email: string; password: string }): Promise< + ApiResponse + > => { + return await this.api.auth.loginWithPassword(data); + }, + register: async (data: any): Promise> => { + return await this.api.auth.registerWithPassword(data); + }, + logout: async () => { + this.api.updateToken(undefined); + return true; + }, + setToken: (token) => { + this.api.updateToken(token); + return this.api.getAuthState(); + }, + verify: async () => { + console.log("verifiying"); + const res = await this.api.auth.me(); + console.log("verifying result", res); + if (!res.res.ok) { + this.api.markAuthVerified(false); + this.api.updateToken(undefined); + } else { + this.api.markAuthVerified(true); + } + } + }; + }; + + media = (options?: Partial) => { + const queryOptions = this.queryOptions(options); + return { + api: () => { + return this.api.media; + }, + list: (query: Partial = { limit: 10 }): UseQueryResult => { + return useQuery({ + ...(queryOptions as any), // @todo: fix typing + queryKey: ["data", "entity", "media", { query }], + queryFn: async () => { + return await this.api.data.readMany("media", query); + } + }); + }, + deleteFile: async (filename: string | { path: string }) => { + const res = await this.api.media.deleteFile( + typeof filename === "string" ? filename : filename.path + ); + + if (res.res.ok) { + queryClient.invalidateQueries({ queryKey: ["data", "entity", "media"] }); + return true; + } + + return false; + } + }; + }; + + query = (options?: Partial) => { + const queryOptions = this.queryOptions(options); + return { + data: { + entity: (name: string) => { + return { + readOne: ( + id: number, + query: Partial> = {} + ): any => { + return useQuery({ + ...queryOptions, + queryKey: ["data", "entity", name, id, { query }], + queryFn: async () => { + return await this.api.data.readOne(name, id, query); + } + }); + }, + readMany: ( + query: Partial = { limit: 10, offset: 0 } + ): UseQueryResult => { + return useQuery({ + ...(queryOptions as any), // @todo: fix typing + queryKey: ["data", "entity", name, { query }], + queryFn: async () => { + return await this.api.data.readMany(name, query); + } + }); + }, + readManyByReference: ( + id: number, + reference: string, + referenced_entity?: string, // required for query invalidation + query: Partial = { limit: 10, offset: 0 } + ): UseQueryResult> => { + return useQuery({ + ...(queryOptions as any), // @todo: fix typing + queryKey: [ + "data", + "entity", + referenced_entity ?? reference, + { name, id, reference, query } + ], + queryFn: async () => { + return await this.api.data.readManyByReference( + name, + id, + reference, + query + ); + } + }); + }, + count: ( + where: RepoQuery["where"] = {} + ): UseQueryResult> => { + return useQuery({ + ...(queryOptions as any), // @todo: fix typing + queryKey: ["data", "entity", name, "fn", "count", { where }], + queryFn: async () => { + return await this.api.data.count(name, where); + } + }); + } + }; + } + } + }; + }; + + // @todo: centralize, improve + __invalidate = (...args: any[]) => { + console.log("___invalidate", ["data", "entity", ...args]); + queryClient.invalidateQueries({ queryKey: ["data", "entity", ...args] }); + }; + + // @todo: must return response... why? + mutation = { + data: { + entity: (name: string) => { + return { + update: (id: number): any => { + return useMutation({ + mutationFn: async (input: EntityData) => { + return await this.api.data.updateOne(name, id, input); + }, + onSuccess: async () => { + await queryClient.invalidateQueries({ queryKey: ["data", "entity", name] }); + } + }); + }, + create: (): any => { + return useMutation({ + mutationFn: async (input: EntityData) => { + return await this.api.data.createOne(name, input); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["data", "entity", name] }); + } + }); + }, + delete: (id: number): any => { + return useMutation({ + mutationFn: async () => { + return await this.api.data.deleteOne(name, id); + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["data", "entity", name] }); + } + }); + } + }; + } + } + }; +} diff --git a/app/src/ui/client/utils/AppReduced.ts b/app/src/ui/client/utils/AppReduced.ts new file mode 100644 index 0000000..3c95934 --- /dev/null +++ b/app/src/ui/client/utils/AppReduced.ts @@ -0,0 +1,83 @@ +import type { App } from "App"; +import type { Entity, EntityRelation } from "data"; +import { AppData } from "data/AppData"; +import { RelationAccessor } from "data/relations/RelationAccessor"; +import { Flow, TaskMap } from "flows"; + +export type AppType = ReturnType; + +/** + * Reduced version of the App class for frontend use + */ +export class AppReduced { + // @todo: change to record + private _entities: Entity[] = []; + private _relations: EntityRelation[] = []; + private _flows: Flow[] = []; + + constructor(protected appJson: AppType) { + console.log("received appjson", appJson); + + this._entities = Object.entries(this.appJson.data.entities ?? {}).map(([name, entity]) => { + return AppData.constructEntity(name, entity); + }); + + this._relations = Object.entries(this.appJson.data.relations ?? {}).map(([, relation]) => { + return AppData.constructRelation(relation, this.entity.bind(this)); + }); + + for (const [name, obj] of Object.entries(this.appJson.flows.flows ?? {})) { + // @ts-ignore + // @todo: fix constructing flow + const flow = Flow.fromObject(name, obj, TaskMap); + + this._flows.push(flow); + } + } + + get entities(): Entity[] { + return this._entities; + } + + // @todo: change to record + entity(_entity: Entity | string): Entity { + const name = typeof _entity === "string" ? _entity : _entity.name; + const entity = this._entities.find((entity) => entity.name === name); + if (!entity) { + throw new Error(`Entity "${name}" not found`); + } + + return entity; + } + + get relations(): RelationAccessor { + return new RelationAccessor(this._relations); + } + + get flows(): Flow[] { + return this._flows; + } + + get config() { + return this.appJson; + } + + getAdminConfig() { + return this.appJson.server.admin; + } + + getSettingsPath(path: string[] = []): string { + const { basepath } = this.getAdminConfig(); + const base = `~/${basepath}/settings`.replace(/\/+/g, "/"); + return [base, ...path].join("/"); + } + + getAbsolutePath(path?: string): string { + const { basepath } = this.getAdminConfig(); + return (path ? `~/${basepath}/${path}` : `~/${basepath}`).replace(/\/+/g, "/"); + } + + getAuthConfig() { + return this.appJson.auth; + } +} diff --git a/app/src/ui/client/utils/theme-switcher.ts b/app/src/ui/client/utils/theme-switcher.ts new file mode 100644 index 0000000..ed774e4 --- /dev/null +++ b/app/src/ui/client/utils/theme-switcher.ts @@ -0,0 +1,28 @@ +import { useState } from "react"; +export type AppTheme = "light" | "dark" | string; + +export function useSetTheme(initialTheme: AppTheme = "light") { + const [theme, _setTheme] = useState(initialTheme); + + const $html = document.querySelector("#bknd-admin")!; + function setTheme(newTheme: AppTheme) { + $html?.classList.remove("dark", "light"); + $html?.classList.add(newTheme); + _setTheme(newTheme); + + // @todo: just a quick switcher config update test + fetch("/api/system/config/patch/server/admin", { + method: "PATCH", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ color_scheme: newTheme }) + }) + .then((res) => res.json()) + .then((data) => { + console.log("theme updated", data); + }); + } + + return { theme, setTheme }; +} diff --git a/app/src/ui/components/Context.tsx b/app/src/ui/components/Context.tsx new file mode 100644 index 0000000..edcf3ed --- /dev/null +++ b/app/src/ui/components/Context.tsx @@ -0,0 +1,17 @@ +import { useBaseUrl } from "../client/ClientProvider"; + +export function Context() { + const baseurl = useBaseUrl(); + + return ( +
+ {JSON.stringify( + { + baseurl + }, + null, + 2 + )} +
+ ); +} diff --git a/app/src/ui/components/buttons/Button.tsx b/app/src/ui/components/buttons/Button.tsx new file mode 100644 index 0000000..c9df2b6 --- /dev/null +++ b/app/src/ui/components/buttons/Button.tsx @@ -0,0 +1,75 @@ +import type React from "react"; +import { forwardRef } from "react"; +import { twMerge } from "tailwind-merge"; +import { Link } from "ui/components/wouter/Link"; + +const sizes = { + small: "px-2 py-1.5 rounded-md gap-1.5 text-sm", + default: "px-3 py-2.5 rounded-md gap-2.5", + large: "px-4 py-3 rounded-md gap-3 text-lg" +}; + +const iconSizes = { + small: 15, + default: 18, + large: 22 +}; + +const styles = { + default: "bg-primary/5 hover:bg-primary/10 link text-primary/70", + primary: "bg-primary hover:bg-primary/80 link text-background", + ghost: "bg-transparent hover:bg-primary/5 link text-primary/70", + outline: "border border-primary/70 bg-transparent hover:bg-primary/5 link text-primary/70", + red: "dark:bg-red-950 dark:hover:bg-red-900 bg-red-100 hover:bg-red-200 link text-primary/70", + subtlered: + "dark:text-red-950 text-red-700 dark:hover:bg-red-900 bg-transparent hover:bg-red-50 link" +}; + +export type BaseProps = { + className?: string; + children?: React.ReactNode; + IconLeft?: React.ComponentType; + IconRight?: React.ComponentType; + iconSize?: number; + iconProps?: Record; + size?: keyof typeof sizes; + variant?: keyof typeof styles; + labelClassName?: string; +}; + +const Base = ({ + children, + size, + variant, + IconLeft, + IconRight, + iconSize = iconSizes[size ?? "default"], + iconProps, + labelClassName, + ...props +}: BaseProps) => ({ + ...props, + className: twMerge( + "flex flex-row flex-nowrap items-center font-semibold disabled:opacity-50 cursor-pointer disabled:cursor-not-allowed", + sizes[size ?? "default"], + styles[variant ?? "default"], + props.className + ), + children: ( + <> + {IconLeft && } + {children && {children}} + {IconRight && } + + ) +}); + +export type ButtonProps = React.ComponentPropsWithoutRef<"button"> & BaseProps; +export const Button = forwardRef((props, ref) => ( + + + )} + + +); diff --git a/app/src/ui/components/display/Logo.tsx b/app/src/ui/components/display/Logo.tsx new file mode 100644 index 0000000..89ad5bd --- /dev/null +++ b/app/src/ui/components/display/Logo.tsx @@ -0,0 +1,31 @@ +import { useBknd } from "../../client/BkndProvider"; + +export function Logo({ scale = 0.2, fill }: { scale?: number; fill?: string }) { + const { app } = useBknd(); + const theme = app.getAdminConfig().color_scheme; + const svgFill = fill ? fill : theme === "light" ? "black" : "white"; + + const dim = { + width: Math.round(578 * scale), + height: Math.round(188 * scale) + } as const; + + return ( +
+ + + + +
+ ); +} diff --git a/app/src/ui/components/form/FloatingSelect/FloatingSelect.tsx b/app/src/ui/components/form/FloatingSelect/FloatingSelect.tsx new file mode 100644 index 0000000..7276c83 --- /dev/null +++ b/app/src/ui/components/form/FloatingSelect/FloatingSelect.tsx @@ -0,0 +1,55 @@ +import { FloatingIndicator, Input, UnstyledButton } from "@mantine/core"; +import { useState } from "react"; +import { twMerge } from "tailwind-merge"; + +export type FloatingSelectProps = { + data: string[]; + description?: string; + label?: string; +}; + +export function FloatingSelect({ data, label, description }: FloatingSelectProps) { + const [rootRef, setRootRef] = useState(null); + const [controlsRefs, setControlsRefs] = useState>({}); + const [active, setActive] = useState(0); + + const setControlRef = (index: number) => (node: HTMLButtonElement) => { + controlsRefs[index] = node; + setControlsRefs(controlsRefs); + }; + + const controls = data.map((item, index) => ( + + )); + + return ( + + {label && ( +
+ {label} + {description && {description}} +
+ )} +
+ {controls} + + +
+ {/*Input error*/} + + ); +} diff --git a/app/src/ui/components/form/Formy.tsx b/app/src/ui/components/form/Formy.tsx new file mode 100644 index 0000000..af2eb49 --- /dev/null +++ b/app/src/ui/components/form/Formy.tsx @@ -0,0 +1,176 @@ +import { Switch } from "@mantine/core"; +import { getBrowser } from "core/utils"; +import type { Field } from "data"; +import { forwardRef, useEffect, useImperativeHandle, useRef, useState } from "react"; +import { TbCalendar, TbChevronDown, TbInfoCircle } from "react-icons/tb"; +import { twMerge } from "tailwind-merge"; +import { useEvent } from "../../hooks/use-event"; +import { IconButton } from "../buttons/IconButton"; + +export const Group: React.FC & { error?: boolean }> = ({ + error, + ...props +}) => ( +
+); + +export const formElementFactory = (element: string, props: any) => { + switch (element) { + case "date": + return DateInput; + case "boolean": + return BooleanInput; + case "textarea": + return Textarea; + default: + return Input; + } +}; + +export const Label: React.FC> = (props) =>
+ +
+ + + +
+ Result + 

+    

+
+    
+  
+
\ No newline at end of file
diff --git a/examples/sw/main.ts b/examples/sw/main.ts
new file mode 100644
index 0000000..e92828f
--- /dev/null
+++ b/examples/sw/main.ts
@@ -0,0 +1,19 @@
+if ("serviceWorker" in navigator) {
+   navigator.serviceWorker.getRegistrations().then((registrations) => {
+      for (const registration of registrations) {
+         console.log("[-] unregister Service Worker");
+         registration.unregister();
+      }
+
+      navigator.serviceWorker
+         .register("./sw.ts?t=" + Date.now(), {
+            type: "module"
+         })
+         .then(() => console.log("[+] service Worker registered"))
+         .catch((err) => console.error("[!] service Worker registration failed:", err));
+
+      navigator.serviceWorker.ready.then(() => {
+         console.log("[√] service worker is ready and controlling the page.");
+      });
+   });
+}
diff --git a/examples/sw/package.json b/examples/sw/package.json
new file mode 100644
index 0000000..f017dd8
--- /dev/null
+++ b/examples/sw/package.json
@@ -0,0 +1,19 @@
+{
+  "name": "sw",
+  "private": true,
+  "scripts": {
+    "dev": "vite dev",
+    "db": "turso dev --db-file test.db",
+    "bknd": "bknd run --db-url http://localhost:8080",
+    "bknd:local": "bknd run --db-url file:test.db",
+    "db:check": "sqlite3 test.db \"PRAGMA wal_checkpoint(FULL);\""
+  },
+  "type": "module",
+  "dependencies": {
+    "hono": "^4.6.9",
+    "bknd": "workspace:*"
+  },
+  "devDependencies": {
+    "vite": "^5.4.10"
+  }
+}
diff --git a/examples/sw/sw.ts b/examples/sw/sw.ts
new file mode 100644
index 0000000..77d53e5
--- /dev/null
+++ b/examples/sw/sw.ts
@@ -0,0 +1,35 @@
+// To support types
+// https://github.com/microsoft/TypeScript/issues/14877
+
+declare const self: ServiceWorkerGlobalScope;
+
+import { App } from "bknd";
+
+async function getBknd() {
+   const bknd = App.create({
+      connection: {
+         type: "libsql",
+         config: {
+            url: "http://localhost:8080"
+         }
+      }
+   });
+   await bknd.build();
+   return bknd;
+}
+
+self.addEventListener("fetch", async (e) => {
+   // only intercept api requests
+   if (e.request.url.includes("/api/")) {
+      e.respondWith(
+         (async () => {
+            try {
+               const bknd = await getBknd();
+               return bknd.modules.server.fetch(e.request);
+            } catch (e) {
+               return new Response(e.message, { status: 500 });
+            }
+         })()
+      );
+   }
+});
diff --git a/examples/sw/tsconfig.json b/examples/sw/tsconfig.json
new file mode 100644
index 0000000..69a9583
--- /dev/null
+++ b/examples/sw/tsconfig.json
@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "lib": ["ES2020", "DOM", "WebWorker"],
+    "moduleResolution": "bundler"
+  },
+  "include": ["./"],
+  "exclude": ["node_modules"]
+}
\ No newline at end of file
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..fe1b2e7
--- /dev/null
+++ b/package.json
@@ -0,0 +1,50 @@
+{
+  "name": "bknd",
+  "private": true,
+  "sideEffects": false,
+  "type": "module",
+  "scripts": {
+    "test": "ALL_TESTS=1 bun test --bail",
+    "test:coverage": "bun test --coverage",
+    "types": "bun run --filter './packages/**' types",
+    "build": "bun run clean:dist && bun run --cwd app build:all && bun build:packages",
+    "build:packages": "bun run --filter './packages/{cli,plasmic}' build",
+    "git:pre-commit": "bun run test",
+    "lint": "bunx @biomejs/biome lint --changed .",
+    "updater": "bun x npm-check-updates -ui",
+    "clean:dist": "find packages -name 'dist' -type d -exec rm -rf {} +",
+    "ci": "find . -name 'node_modules' -type d -exec rm -rf {} + && bun install",
+    "npm:local": "verdaccio --config verdaccio.yml"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@biomejs/biome": "1.8.3",
+    "@clack/prompts": "^0.7.0",
+    "@cloudflare/workers-types": "^4.20240620.0",
+    "@tsconfig/strictest": "^2.0.5",
+    "@types/lodash-es": "^4.17.12",
+    "bun-types": "^1.1.18",
+    "dotenv": "^16.4.5",
+    "esbuild": "^0.23.0",
+    "esbuild-plugin-tsc": "^0.4.0",
+    "miniflare": "^3.20240806.0",
+    "mitata": "^0.1.11",
+    "picocolors": "^1.0.1",
+    "semver": "^7.6.2",
+    "sql-formatter": "^15.3.2",
+    "tsd": "^0.31.1",
+    "tsup": "^8.1.0",
+    "typescript": "^5.5.3",
+    "verdaccio": "^5.32.1",
+    "wrangler": "^3.71.0"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "workspaces": [
+    "app",
+    "docs",
+    "packages/*",
+    "examples/*"
+  ]
+}
\ No newline at end of file
diff --git a/packages/cli/package.json b/packages/cli/package.json
new file mode 100644
index 0000000..4f1fcca
--- /dev/null
+++ b/packages/cli/package.json
@@ -0,0 +1,33 @@
+{
+  "name": "bknd-cli",
+  "type": "module",
+  "bin": "./dist/cli/index.js",
+  "version": "0.0.7",
+  "scripts": {
+    "build": "rm -rf dist && mkdir -p dist/{static,cli} && cp -r ../../app/dist/static/ dist/static && cp -r ../../app/dist/cli/ dist/cli",
+    "cli": "node dist/cli/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "@libsql/client": "^0.14.0",
+    "hono": "^4.6.7"
+  },
+  "tsup": {
+    "entry": [
+      "src/index.ts"
+    ],
+    "minify": true,
+    "outDir": "dist",
+    "format": [
+      "esm"
+    ],
+    "platform": "neutral",
+    "splitting": false,
+    "loader": {
+      ".md": "copy"
+    }
+  }
+}
diff --git a/packages/plasmic/.gitignore b/packages/plasmic/.gitignore
new file mode 100644
index 0000000..9b1ee42
--- /dev/null
+++ b/packages/plasmic/.gitignore
@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store
diff --git a/packages/plasmic/README.md b/packages/plasmic/README.md
new file mode 100644
index 0000000..8b1029b
--- /dev/null
+++ b/packages/plasmic/README.md
@@ -0,0 +1,15 @@
+# bknd-plasmic
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.4. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.
diff --git a/packages/plasmic/components/Image.tsx b/packages/plasmic/components/Image.tsx
new file mode 100644
index 0000000..5c02e68
--- /dev/null
+++ b/packages/plasmic/components/Image.tsx
@@ -0,0 +1,243 @@
+"use client";
+
+import type { CodeComponentMeta } from "@plasmicapp/host";
+//import { PlasmicCanvasContext } from "@plasmicapp/loader-react";
+import { useContext, useEffect, useRef, useState } from "react";
+
+type PlasmicImageProps = {
+   asset: {
+      aspectRatio?: any;
+      dataUri: string;
+      name: string;
+      type: string;
+      uid: number;
+      uuid: string;
+      width: number;
+      height: number;
+   };
+   uid: number;
+};
+
+type ImageProps = {
+   className?: string;
+   src?: string | PlasmicImageProps;
+   alt?: string;
+   width?: number | string;
+   height?: number | string;
+   ratio?: number;
+   backgroundColor?: string;
+   forceLoad?: boolean;
+   transformations?: string;
+   transitionSpeed?: number;
+   loadTreshold?: number;
+};
+
+function numeric(value: number | string): number {
+   return typeof value === "number" ? value : Number.parseFloat(value);
+}
+
+function getDimensionDefaults(
+   width: number | string | undefined,
+   height: number | string | undefined,
+   ratio: number | undefined,
+) {
+   let _width = width;
+   let _height = height;
+   let _ratio = ratio;
+
+   if (_width && ratio) {
+      _height = (1 / ratio) * numeric(_width);
+   } else if (_height && ratio) {
+      _width = ratio * numeric(_height);
+   }
+
+   if (_width && _height && !_ratio) {
+      _ratio = numeric(_width) / numeric(_height);
+   }
+
+   return { width: _width, height: _height, ratio: _ratio };
+}
+
+function getPlaceholderStyle(
+   width: number | string | undefined,
+   height: number | string | undefined,
+   ratio: number | undefined,
+) {
+   let paddingBottom = 0;
+   if (width && height) {
+      paddingBottom = (1 / (numeric(width) / numeric(height))) * 100;
+      //paddingBottom = `${numeric(width)}px / ${numeric(height)}px * 100%}`;
+      //
+   } else if (ratio) {
+      paddingBottom = (1 / ratio) * 100;
+   }
+
+   return {
+      paddingBottom: paddingBottom + "%",
+   };
+}
+
+export const Image: React.FC = ({
+   className,
+   src,
+   alt = "",
+   width,
+   height,
+   ratio,
+   backgroundColor = "rgba(225, 225, 225, 0.2)",
+   forceLoad = false,
+   transformations = "",
+   transitionSpeed = 200,
+   loadTreshold = 0.1,
+   ...rest
+}) => {
+   const inEditor = false; // !!useContext(PlasmicCanvasContext);
+   const [loaded, setLoaded] = useState(false);
+   const [isInView, setIsInView] = useState(inEditor ?? forceLoad);
+   const [transitioned, setTransitioned] = useState(forceLoad);
+   const imgRef = useRef(null);
+
+   if (src) {
+      if (typeof src === "object") {
+         src = src.asset.dataUri;
+      }
+
+      if (/cloudinary/.test(src)) {
+         if (transformations) {
+            src = src.replace("/upload", "/upload/" + transformations);
+         }
+      }
+   }
+
+   //console.log("after:src", src);
+
+   useEffect(() => {
+      if (forceLoad) {
+         setIsInView(true);
+         return;
+      }
+
+      const observer = new IntersectionObserver(
+         (entries) => {
+            entries.forEach((entry) => {
+               if (entry.isIntersecting) {
+                  setIsInView(true);
+                  observer.disconnect();
+               }
+            });
+         },
+         { threshold: loadTreshold },
+      );
+      if (imgRef.current) {
+         observer.observe(imgRef.current);
+      }
+
+      return () => {
+         observer.disconnect();
+      };
+   }, [forceLoad]);
+
+   const onLoad = () => {
+      setTimeout(() => {
+         setLoaded(true);
+      }, 0);
+
+      setTimeout(() => {
+         setTransitioned(true);
+      }, transitionSpeed);
+   };
+
+   const {
+      width: _width,
+      height: _height,
+      ratio: _ratio,
+   } = getDimensionDefaults(width, height, ratio);
+
+   const imgStyle: any = {
+      objectFit: "cover",
+      transition: `opacity ${transitionSpeed}ms linear`,
+      position: "relative",
+      maxWidth: "100%",
+      maxHeight: "100%",
+      width: _width || "100%",
+      height: "auto",
+      //height: _height || "auto",
+      //height: !transitioned ? _height || "auto" : "auto",
+      opacity: forceLoad || loaded ? 1 : 0,
+   };
+
+   const placeholderStyle: any = {
+      position: "absolute",
+      maxWidth: "100%",
+      maxHeight: "100%",
+      backgroundColor,
+      width: _width || "100%",
+      height: 0,
+      //height: transitioned ? "auto" : 0,
+      ...getPlaceholderStyle(_width, _height, _ratio),
+   };
+
+   const wrapperStyle: any = {
+      position: "relative",
+      width: _width,
+      ...getPlaceholderStyle(_width, _height, _ratio),
+      height: 0,
+      margin: 0,
+      lineHeight: 0,
+      //height: _height,
+      maxWidth: "100%",
+      maxHeight: "100%",
+   };
+   if (loaded) {
+      wrapperStyle.height = "auto";
+      wrapperStyle.paddingBottom = 0;
+   }
+
+   if (!src) return 
;
+
+   return (
+      

+         

+         {isInView && (
+            {alt}
+         )}
+      

+   );
+};
+
+export const ImageMeta: CodeComponentMeta> = {
+   name: "ImageLazy",
+   importPath: import.meta.dir,
+   props: {
+      src: {
+         type: "imageUrl",
+         displayName: "Image",
+      },
+      alt: "string",
+      width: "number",
+      height: "number",
+      ratio: "number",
+      forceLoad: "boolean",
+      transformations: "string",
+      //backgroundColor: "color",
+      transitionSpeed: {
+         type: "number",
+         helpText: "How fast image should fade in. Default is 200 (ms).",
+      },
+      loadTreshold: {
+         type: "number",
+         displayName: "Treshold",
+         //defaultValue: 0.1,
+         helpText:
+            "Number between 0 and 1. Default is 0.1. Determines how much of the image must be in viewport before it gets loaded",
+      },
+   },
+};
diff --git a/packages/plasmic/components/LazyRender.tsx b/packages/plasmic/components/LazyRender.tsx
new file mode 100644
index 0000000..2de5e06
--- /dev/null
+++ b/packages/plasmic/components/LazyRender.tsx
@@ -0,0 +1,111 @@
+import type { CodeComponentMeta } from "@plasmicapp/host";
+import { useEffect, useRef, useState } from "react";
+
+interface LazyRenderProps {
+   className?: string;
+   forceLoad?: boolean;
+   forceFallback?: boolean;
+   threshold?: number;
+   fallback?: React.ReactNode;
+   delay?: number;
+   children?: React.ReactNode;
+   onBecomesVisible?: () => void;
+}
+
+const DefaultFallback = () => 
asdf
;
+
+export const LazyRender: React.FC = ({
+   className = "",
+   children,
+   forceLoad = false,
+   forceFallback = false,
+   threshold = 0.1,
+   delay = 0,
+   fallback = ,
+   onBecomesVisible,
+}) => {
+   const [isVisible, setIsVisible] = useState(forceLoad);
+   const ref = useRef(null);
+   /* console.log("props", {
+      delay,
+      threshold,
+      fallback,
+      isVisible,
+      forceLoad,
+      forceFallback,
+      children,
+   }); */
+
+   useEffect(() => {
+      if (forceLoad || forceFallback) {
+         setIsVisible(true);
+         return;
+      }
+
+      const observerOptions: IntersectionObserverInit = {
+         threshold: threshold < 1 ? threshold : 0.1,
+      };
+
+      const observerCallback: IntersectionObserverCallback = (entries) => {
+         entries.forEach((entry) => {
+            if (entry.isIntersecting && !isVisible) {
+               setTimeout(() => {
+                  setIsVisible(true);
+                  onBecomesVisible?.();
+                  if (ref.current) observer.unobserve(ref.current);
+               }, delay);
+            }
+         });
+      };
+
+      const observer = new IntersectionObserver(observerCallback, observerOptions);
+
+      if (ref.current) observer.observe(ref.current);
+
+      return () => {
+         if (ref.current) observer.unobserve(ref.current);
+      };
+   }, [forceLoad, threshold, forceFallback, delay]);
+
+   return (
+      

+         {isVisible && !forceFallback ? children : fallback}
+      

+   );
+};
+
+export const LazyRenderMeta: CodeComponentMeta> = {
+   name: "LazyRender",
+   importPath: import.meta.dir,
+   props: {
+      forceLoad: {
+         type: "boolean",
+         defaultValue: false,
+      },
+      forceFallback: {
+         type: "boolean",
+         defaultValue: false,
+      },
+      threshold: {
+         type: "number",
+         defaultValue: 0.1,
+      },
+      fallback: {
+         type: "slot",
+         //allowedComponents: ["*"],
+      },
+      delay: {
+         type: "number",
+         defaultValue: 0,
+      },
+      onBecomesVisible: {
+         type: "code",
+         lang: "javascript",
+      },
+
+      children: {
+         type: "slot",
+         //allowedComponents: ["*"],
+      },
+   },
+};
diff --git a/packages/plasmic/components/WouterLink.tsx b/packages/plasmic/components/WouterLink.tsx
new file mode 100644
index 0000000..a7472dc
--- /dev/null
+++ b/packages/plasmic/components/WouterLink.tsx
@@ -0,0 +1,23 @@
+import type { CodeComponentMeta } from "@plasmicapp/host";
+import { Link } from "wouter";
+
+export function WouterLink({ href, className, children, ...props }) {
+   return (
+      
+         {children}
+      
+   );
+}
+
+export const WouterLinkMeta: CodeComponentMeta = {
+   name: "WouterLink",
+   importPath: import.meta.dir,
+   props: {
+      href: {
+         type: "href",
+      },
+      children: {
+         type: "slot",
+      },
+   },
+};
diff --git a/packages/plasmic/components/data/BkndData.tsx b/packages/plasmic/components/data/BkndData.tsx
new file mode 100644
index 0000000..f5fef9d
--- /dev/null
+++ b/packages/plasmic/components/data/BkndData.tsx
@@ -0,0 +1,385 @@
+import { type CodeComponentMeta, DataProvider, usePlasmicCanvasContext } from "@plasmicapp/host";
+import type { RepoQuery } from "bknd/data";
+import { useEntities, useEntity } from "bknd/ui";
+import { encodeSearch } from "bknd/utils";
+import { useContext, useEffect, useState } from "react";
+import { usePlasmicBkndContext } from "../../contexts/BkndContext";
+
+type BkndEntitiesProps = {
+   children?: React.ReactNode;
+   loading?: React.ReactNode;
+   error?: React.ReactNode;
+   empty?: React.ReactNode;
+   setControlContextData?: (ctxData: {
+      entities: string[];
+      fields: string[];
+      references: string[];
+   }) => void;
+   className?: string;
+   limit?: number;
+   offset?: number;
+   withRefs?: string[];
+   joinRefs?: string[];
+   dataName?: string;
+   entityId?: number;
+   entity?: string;
+   sortBy: string;
+   sortDir: "asc" | "desc";
+   where?: string;
+   mode?: "fetch" | "react-query";
+   noLayout?: boolean;
+   preview?: boolean;
+   previewSlot?: "loading" | "error" | "empty";
+};
+
+const LoadingComponent = ({ loading }: { loading?: React.ReactNode }) => {
+   return loading ? <>{loading} : <>Loading...;
+};
+
+const ErrorComponent = ({ error }: { error?: React.ReactNode }) => {
+   return error ? <>{error} : <>Error;
+};
+
+const EmptyComponent = ({ empty }: { empty?: React.ReactNode }) => {
+   return empty ? <>{empty} : <>No data;
+};
+
+export function BkndData({
+   children,
+   loading,
+   error,
+   empty,
+   entity,
+   setControlContextData,
+   dataName,
+   limit,
+   offset,
+   entityId,
+   where,
+   withRefs,
+   joinRefs,
+   sortBy = "id",
+   sortDir = "asc",
+   mode = "fetch",
+   noLayout,
+   preview,
+   previewSlot,
+   ...props
+}: BkndEntitiesProps) {
+   const inEditor = !!usePlasmicCanvasContext();
+   const plasmicContext = usePlasmicBkndContext();
+
+   if (inEditor && preview) {
+      let Component: React.ReactNode;
+      switch (previewSlot) {
+         case "loading":
+            Component = ;
+            break;
+         case "error":
+            Component = ;
+            break;
+         case "empty":
+            Component = ;
+            break;
+      }
+
+      if (Component) {
+         return noLayout ? Component : 
{Component}
;
+      }
+   }
+
+   let _where: any = undefined;
+   if (where) {
+      if (typeof where === "string") {
+         try {
+            _where = JSON.parse(where);
+         } catch (e) {}
+      } else {
+         _where = where;
+      }
+   }
+
+   const query = {
+      limit: entityId ? undefined : limit,
+      offset: entityId ? undefined : offset,
+      where: _where,
+      sort: { by: sortBy, dir: sortDir },
+      with: withRefs,
+      join: joinRefs
+   };
+
+   console.log("---context", plasmicContext);
+   if (plasmicContext.appConfig?.data?.entities) {
+      const { entities, relations } = plasmicContext.appConfig.data;
+      console.log("entities", entities);
+      //setControlContextData?.({ entities, fields: ["id"] });
+
+      let fields: string[] = ["id"];
+      let references: string[] = [];
+
+      if (entity && entity in entities) {
+         fields = Object.keys(entities[entity].fields!);
+
+         if (relations) {
+            const rels = Object.values(relations).filter(
+               // biome-ignore lint/suspicious/noDoubleEquals: 
+               (r: any) => r.source == entity
+            );
+            // @ts-ignore
+            references = rels?.map((r) => r.config?.mappedBy ?? r.target);
+            //console.log("relations", relations, references);
+         }
+      }
+
+      setControlContextData?.({ entities: Object.keys(entities), fields, references });
+   }
+
+   if (!entity) {
+      return 
Select an entity
;
+   }
+
+   const modeProps: ModeProps = {
+      loading,
+      error,
+      empty,
+      dataName: dataName ?? entity ?? "data",
+      entityId,
+      entity,
+      query,
+      children
+   };
+
+   const Component =
+      mode === "react-query" ?  : ;
+   return noLayout ? Component : 
{Component}
;
+}
+
+type ModeProps = {
+   entity: string;
+   dataName: string;
+   children?: React.ReactNode;
+   loading?: React.ReactNode;
+   error?: React.ReactNode;
+   empty?: React.ReactNode;
+   entityId?: number;
+   query?: Partial;
+};
+
+const ModeFetch = ({
+   children,
+   loading,
+   error,
+   empty,
+   dataName,
+   entityId,
+   entity,
+   query
+}: ModeProps) => {
+   const [data, setData] = useState([]);
+   const [isLoading, setLoading] = useState(true);
+   const [hasError, setError] = useState();
+   const plasmicContext = usePlasmicBkndContext();
+   const basepath = "/api/data";
+   const path = entityId ? `${basepath}/${entity}/${entityId}` : `${basepath}/${entity}`;
+   console.log("query", path, query);
+   const url = `${plasmicContext.baseUrl}${path}?${encodeSearch(query)}`;
+   useEffect(() => {
+      (async () => {
+         try {
+            const res = await fetch(url);
+            const result = (await res.json()) as any;
+            //console.log("result", result);
+            setData(result.data);
+            setLoading(false);
+            setError(undefined);
+         } catch (e) {
+            console.error(e);
+            setError(String(e));
+            setLoading(false);
+         }
+      })();
+   }, [url]);
+
+   console.log("--data", { name: dataName ?? entity ?? "data", data, isLoading, hasError });
+
+   if (isLoading) {
+      return ;
+   }
+
+   if (hasError) {
+      return ;
+   }
+
+   if (data.length === 0) {
+      return ;
+   }
+   console.log("--here1");
+
+   return (
+      
+         {children}
+      
+   );
+};
+
+const ModeReactQuery = (props: ModeProps) => {
+   return props.entityId ? (
+      
+   ) : (
+      
+   );
+};
+
+const ModeReactQuerySingle = ({
+   children,
+   loading,
+   error,
+   dataName,
+   entityId,
+   empty,
+   entity
+}: ModeProps) => {
+   const container = useEntity(entity, entityId);
+   const { isLoading, isError } = container.status.fetch;
+
+   if (isLoading) {
+      return ;
+   }
+
+   if (isError) {
+      return ;
+   }
+
+   if (!container.data) {
+      return ;
+   }
+
+   return (
+      
+         {children}
+      
+   );
+};
+
+const ModeReactQueryMultiple = ({
+   children,
+   loading,
+   error,
+   empty,
+   dataName,
+   entity,
+   query
+}: ModeProps) => {
+   const container = useEntities(entity, query);
+   const { isLoading, isError } = container.status.fetch;
+
+   if (isLoading) {
+      return ;
+   }
+
+   if (isError) {
+      return ;
+   }
+
+   if (!container.data || container.data.length === 0) {
+      return ;
+   }
+
+   return (
+      
+         {children}
+      
+   );
+};
+
+export const BkndDataMeta: CodeComponentMeta> = {
+   name: "BKND Data",
+   section: "BKND",
+   importPath: import.meta.dir,
+   providesData: true,
+   props: {
+      entity: {
+         type: "choice",
+         options: (props, ctx) => ctx.entities
+      },
+      dataName: {
+         type: "string"
+      },
+      entityId: {
+         type: "number"
+      },
+      limit: {
+         type: "number",
+         defaultValue: 10,
+         // @ts-ignore
+         hidden: (props) => !!props.entityId,
+         min: 0
+      },
+      offset: {
+         type: "number",
+         defaultValue: 0,
+         // @ts-ignore
+         hidden: (props) => !!props.entityId,
+         min: 0
+      },
+      withRefs: {
+         displayName: "With",
+         type: "choice",
+         multiSelect: true,
+         options: (props, ctx) => ctx.references
+      },
+      joinRefs: {
+         displayName: "Join",
+         type: "choice",
+         multiSelect: true,
+         options: (props, ctx) => ctx.references
+      },
+      where: {
+         type: "code",
+         lang: "json"
+      },
+      sortBy: {
+         type: "choice",
+         options: (props, ctx) => ctx.fields
+      },
+      sortDir: {
+         type: "choice",
+         options: ["asc", "desc"],
+         defaultValue: "asc"
+      },
+      children: {
+         type: "slot"
+      },
+      loading: {
+         type: "slot"
+      },
+      error: {
+         type: "slot"
+      },
+      empty: {
+         type: "slot"
+      },
+      mode: {
+         type: "choice",
+         options: ["fetch", "react-query"],
+         defaultValue: "fetch",
+         advanced: true
+      },
+      noLayout: {
+         type: "boolean",
+         defaultValue: true,
+         advanced: true
+      },
+      preview: {
+         type: "boolean",
+         defaultValue: false,
+         advanced: true
+      },
+      previewSlot: {
+         type: "choice",
+         options: ["loading", "error", "empty"],
+         hidden: (props: any) => props.preview !== true,
+         advanced: true
+      }
+   }
+};
diff --git a/packages/plasmic/components/index.ts b/packages/plasmic/components/index.ts
new file mode 100644
index 0000000..e162ba7
--- /dev/null
+++ b/packages/plasmic/components/index.ts
@@ -0,0 +1,4 @@
+export { BkndData, BkndDataMeta } from "./data/BkndData";
+export { WouterLink, WouterLinkMeta } from "./WouterLink";
+export { Image, ImageMeta } from "./Image";
+export { LazyRender, LazyRenderMeta } from "./LazyRender";
diff --git a/packages/plasmic/contexts/BkndContext.tsx b/packages/plasmic/contexts/BkndContext.tsx
new file mode 100644
index 0000000..a81fbde
--- /dev/null
+++ b/packages/plasmic/contexts/BkndContext.tsx
@@ -0,0 +1,143 @@
+import { DataProvider, GlobalActionsProvider, usePlasmicCanvasContext } from "@plasmicapp/host";
+import type { AppConfig } from "bknd";
+import { ClientProvider, useAuth, useBaseUrl } from "bknd/ui";
+import { createContext, useContext, useEffect, useMemo, useState } from "react";
+
+// Users will be able to set these props in Studio.
+interface BkndGlobalContextProps {
+   // You might use this to override the auth URL to a test or local URL.
+   baseUrl?: string;
+   appConfig?: AppConfig;
+   auth: any; // @todo: add typings
+}
+
+type BkndContextProps = {
+   baseUrl?: string;
+   initialAuth?: any;
+};
+
+const BkndContextContext = createContext({} as any);
+
+function getBaseUrlFromWindow() {
+   if (typeof window === "undefined") {
+      return "";
+   }
+
+   const protocol = window.location.protocol;
+   const host = window.location.host;
+
+   return `${protocol}//${host}`;
+}
+
+// @todo: it's an issue that we need auth, so we cannot make baseurl adjustable (maybe add an option to useAuth with a specific base url?)
+export const BkndContext = ({
+   children,
+   baseUrl,
+   initialAuth
+}: React.PropsWithChildren) => {
+   const auth = useAuth();
+   const baseurl = useBaseUrl();
+
+   const [data, setData] = useState({
+      baseUrl: baseurl,
+      /*baseUrl: (baseUrl && baseUrl.length > 0 ? baseUrl : getBaseUrlFromWindow()).replace(
+         /\/+$/,
+         ""
+      ),*/
+      auth: auth ?? initialAuth,
+      appConfig: undefined
+   });
+   const inEditor = !!usePlasmicCanvasContext();
+   console.log("context:user", data);
+
+   useEffect(() => {
+      setData((prev) => ({ ...prev, auth: auth }));
+   }, [auth.user]);
+
+   useEffect(() => {
+      (async () => {
+         if (inEditor) {
+            const res = await fetch(`${baseurl}/api/system/config`);
+            const result = (await res.json()) as BkndGlobalContextProps["appConfig"];
+            console.log("appconfig", result);
+            setData((prev) => ({ ...prev, appConfig: result }));
+         }
+      })();
+   }, [inEditor]);
+
+   const actions = useMemo(
+      () => ({
+         login: async (data: any) => {
+            console.log("login", data);
+            const result = await auth.login(data);
+            console.log("login:result", result);
+            if (result.res.ok && "user" in result.data) {
+               //result.data.
+               return result.data;
+            } else {
+               console.log("login failed", result);
+            }
+
+            return false;
+         },
+         register: async (data: any) => {
+            console.log("register", data);
+            const result = await auth.register(data);
+            console.log("register:result", result);
+            if (result.res.ok && "user" in result.data) {
+               //result.data.
+               return result.data;
+            }
+
+            return false;
+         },
+         logout: async () => {
+            await auth.logout();
+            console.log("logged out");
+            return true;
+         },
+         setToken: auth.setToken
+      }),
+      [baseUrl]
+   );
+
+   console.log("plasmic.bknd.context", data);
+   return (
+      
+         
+            
+               {/*{children}*/}
+               {children}
+            
+         
+      
+   );
+};
+
+export function usePlasmicBkndContext() {
+   const context = useContext(BkndContextContext);
+   return context;
+}
+
+export const BkndContextMeta = {
+   name: "BkndContext",
+   props: { baseUrl: { type: "string" }, initialAuth: { type: "object" } },
+   providesData: true,
+   globalActions: {
+      login: {
+         parameters: [{ name: "data", type: "object" }]
+      },
+      register: {
+         parameters: [{ name: "data", type: "object" }]
+      },
+      logout: {
+         parameters: []
+      },
+      setToken: {
+         parameters: [{ name: "token", type: "string" }]
+      },
+      sayHi: {
+         parameters: [{ name: "message", type: "string" }]
+      }
+   }
+};
diff --git a/packages/plasmic/contexts/index.ts b/packages/plasmic/contexts/index.ts
new file mode 100644
index 0000000..5947f0b
--- /dev/null
+++ b/packages/plasmic/contexts/index.ts
@@ -0,0 +1 @@
+export { BkndContext, BkndContextMeta } from "./BkndContext";
diff --git a/packages/plasmic/index.ts b/packages/plasmic/index.ts
new file mode 100644
index 0000000..05e7de8
--- /dev/null
+++ b/packages/plasmic/index.ts
@@ -0,0 +1,4 @@
+export { loader as loadBkndComponents, CatchAllPage, createWouterPlasmicApp } from "./loader";
+
+export * from "./components";
+export * from "./contexts";
diff --git a/packages/plasmic/loader.tsx b/packages/plasmic/loader.tsx
new file mode 100644
index 0000000..f81547f
--- /dev/null
+++ b/packages/plasmic/loader.tsx
@@ -0,0 +1,109 @@
+import { PlasmicCanvasHost, type registerComponent } from "@plasmicapp/host";
+import {
+   type ComponentRenderData,
+   PlasmicComponent,
+   type PlasmicComponentLoader,
+   PlasmicRootProvider
+} from "@plasmicapp/loader-react";
+import { forwardRef, useEffect, useState } from "react";
+import { Link, Route, Router, Switch } from "wouter";
+import {
+   BkndData,
+   BkndDataMeta,
+   Image,
+   ImageMeta,
+   LazyRender,
+   LazyRenderMeta,
+   WouterLink,
+   WouterLinkMeta
+} from "./components";
+import { BkndContext, BkndContextMeta } from "./contexts";
+
+export function loader(PLASMIC: PlasmicComponentLoader) {
+   PLASMIC.registerComponent(BkndData, BkndDataMeta);
+   PLASMIC.registerComponent(WouterLink, WouterLinkMeta);
+   PLASMIC.registerComponent(Image, ImageMeta);
+   PLASMIC.registerComponent(LazyRender, LazyRenderMeta);
+   PLASMIC.registerGlobalContext(BkndContext, BkndContextMeta as any);
+}
+
+const CustomLink = forwardRef((props, ref) => {
+   //console.log("rendering custom link", props);
+   //return null;
+   if ("data-replace" in props) {
+      return ;
+   }
+   //return ;
+   // @ts-ignore it's because of the link
+   return ;
+});
+
+const Wrapper = ({ children }) => {
+   return (
+      

+         
{children}

+      

+   );
+};
+
+export function CatchAllPage({
+   PLASMIC,
+   prefix = ""
+}: { PLASMIC: PlasmicComponentLoader; prefix?: string }) {
+   const [loading, setLoading] = useState(true);
+   const [pageData, setPageData] = useState(null);
+
+   //const params = useParams();
+   const pathname = location.pathname.replace(prefix, "");
+   const path = pathname.length === 0 ? "/" : pathname;
+   //console.log("path", path, params);
+   useEffect(() => {
+      async function load() {
+         const pageData = await PLASMIC.maybeFetchComponentData(path);
+         //console.log("pageData", pageData);
+         setPageData(pageData);
+         setLoading(false);
+      }
+      load();
+   }, []);
+
+   if (loading) {
+      return Loading ...;
+   }
+   if (!pageData) {
+      return Not found;
+   }
+
+   const pageMeta = pageData.entryCompMetas[0];
+
+   // The page will already be cached from the `load` call above.
+   return (
+      
+         
+      
+   );
+}
+
+export function createWouterPlasmicApp(PLASMIC: PlasmicComponentLoader, prefix = "") {
+   return function App() {
+      return (
+         
+            
+               
+                }
+               />
+            
+         
+      );
+   };
+}
diff --git a/packages/plasmic/package.json b/packages/plasmic/package.json
new file mode 100644
index 0000000..2ad935c
--- /dev/null
+++ b/packages/plasmic/package.json
@@ -0,0 +1,48 @@
+{
+   "name": "@bknd/plasmic",
+   "type": "module",
+   "sideEffects": false,
+   "scripts": {
+      "build": "rm -rf dist && bun tsup && bun run build:types",
+      "watch": "bun tsup --watch --onSuccess 'bun run build:types'",
+      "build:only": "rm -rf dist && bun tsup",
+      "types": "bun tsc -p tsconfig.json --noEmit --skipLibCheck",
+      "build:types": "bun tsc --emitDeclarationOnly",
+      "updater": "bun x npm-check-updates -ui"
+   },
+   "dependencies": {
+      "wouter": "^3.3.5"
+   },
+   "devDependencies": {
+      "@types/bun": "latest",
+      "typescript": "^5.0.0"
+   },
+   "peerDependencies": {
+      "@plasmicapp/host": ">=1.0.0",
+      "bknd": "workspace:*",
+      "react": ">=18",
+      "react-dom": ">=18"
+   },
+   "tsup": {
+      "entry": ["index.ts"],
+      "minify": true,
+      "clean": true,
+      "external": ["react", "react-dom", "@plasmicapp/host", "@plasmicapp/loader-react", "@plasmicapp/loader-core"],
+      "format": ["esm"],
+      "platform": "browser",
+      "shims": true,
+      "bundle": true,
+      "metafile": true,
+      "splitting": false,
+      "sourceMap": true,
+      "outDir": "dist"
+   },
+   "exports": {
+      ".": {
+         "types": "./dist/index.d.ts",
+         "import": "./dist/index.js",
+         "require": "./dist/index.js"
+      }
+   },
+   "files": ["dist"]
+}
diff --git a/packages/plasmic/tsconfig.json b/packages/plasmic/tsconfig.json
new file mode 100644
index 0000000..7f2aec6
--- /dev/null
+++ b/packages/plasmic/tsconfig.json
@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "jsx": "react-jsx",
+    "allowJs": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": false,
+    "verbatimModuleSyntax": true,
+    "strict": true,
+    "outDir": "dist",
+    "declaration": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitAny": false,
+    "noPropertyAccessFromIndexSignature": false
+  },
+  "include": ["index.ts", "loader.tsx", "components", "contexts"],
+  "exclude": ["@bknd/app", "@bknd/core", "dist", "node_modules", "build.ts"]
+}
diff --git a/tmp/event_manager_returning_test.patch b/tmp/event_manager_returning_test.patch
new file mode 100644
index 0000000..1e194ef
--- /dev/null
+++ b/tmp/event_manager_returning_test.patch
@@ -0,0 +1,150 @@
+Subject: [PATCH] event manager returning test
+---
+Index: app/__test__/core/EventManager.spec.ts
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/app/__test__/core/EventManager.spec.ts b/app/__test__/core/EventManager.spec.ts
+--- a/app/__test__/core/EventManager.spec.ts	(revision f06777256f332766de4bc76c23183725c8c7d310)
++++ b/app/__test__/core/EventManager.spec.ts	(date 1731498680965)
+@@ -1,8 +1,8 @@
+ import { describe, expect, test } from "bun:test";
+-import { Event, EventManager, NoParamEvent } from "../../src/core/events";
++import { Event, EventManager, type ListenerHandler, NoParamEvent } from "../../src/core/events";
+ 
+ class SpecialEvent extends Event<{ foo: string }> {
+-   static slug = "special-event";
++   static override slug = "special-event";
+ 
+    isBar() {
+       return this.params.foo === "bar";
+@@ -10,7 +10,19 @@
+ }
+ 
+ class InformationalEvent extends NoParamEvent {
+-   static slug = "informational-event";
++   static override slug = "informational-event";
++}
++
++class ReturnEvent extends Event<{ foo: string }, number> {
++   static override slug = "return-event";
++   static override returning = true;
++
++   override setValidatedReturn(value: number) {
++      if (typeof value !== "number") {
++         throw new Error("Invalid return value");
++      }
++      this.params.foo = value.toString();
++   }
+ }
+ 
+ describe("EventManager", async () => {
+@@ -43,4 +55,22 @@
+ 
+       expect(true).toBe(true);
+    });
++
++   test.only("piping", async () => {
++      const emgr = new EventManager();
++      emgr.registerEvents([ReturnEvent, InformationalEvent]);
++
++      type T = ListenerHandler;
++
++      // @ts-expect-error InformationalEvent has no return value
++      emgr.onEvent(InformationalEvent, async (event, name) => {
++         console.log("Event: ", name, event.params);
++         return 1;
++      });
++
++      emgr.onEvent(ReturnEvent, async (event, name) => {
++         console.log("Event: ", name, event.params);
++         return 1;
++      });
++   });
+ });
+Index: app/src/core/events/EventManager.ts
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/app/src/core/events/EventManager.ts b/app/src/core/events/EventManager.ts
+--- a/app/src/core/events/EventManager.ts	(revision f06777256f332766de4bc76c23183725c8c7d310)
++++ b/app/src/core/events/EventManager.ts	(date 1731498680971)
+@@ -6,7 +6,7 @@
+ }
+ 
+ export type EventClass = {
+-   new (params: any): Event;
++   new (params: any): Event;
+    slug: string;
+ };
+ 
+@@ -137,6 +137,9 @@
+          throw new Error(`Event "${slug}" not registered`);
+       }
+ 
++      // @ts-expect-error returning is static
++      const returning = Boolean(event.constructor.returning);
++
+       const listeners = this.listeners.filter((listener) => listener.event.slug === slug);
+       //console.log("---!-- emitting", slug, listeners.length);
+ 
+Index: app/src/core/events/EventListener.ts
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/app/src/core/events/EventListener.ts b/app/src/core/events/EventListener.ts
+--- a/app/src/core/events/EventListener.ts	(revision f06777256f332766de4bc76c23183725c8c7d310)
++++ b/app/src/core/events/EventListener.ts	(date 1731498680968)
+@@ -4,10 +4,10 @@
+ export const ListenerModes = ["sync", "async"] as const;
+ export type ListenerMode = (typeof ListenerModes)[number];
+ 
+-export type ListenerHandler = (
++export type ListenerHandler> = (
+    event: E,
+-   slug: string,
+-) => Promise | void;
++   slug: string
++) => E extends Event ? R | Promise : never;
+ 
+ export class EventListener {
+    mode: ListenerMode = "async";
+Index: app/src/core/events/Event.ts
+IDEA additional info:
+Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
+<+>UTF-8
+===================================================================
+diff --git a/app/src/core/events/Event.ts b/app/src/core/events/Event.ts
+--- a/app/src/core/events/Event.ts	(revision f06777256f332766de4bc76c23183725c8c7d310)
++++ b/app/src/core/events/Event.ts	(date 1731498680973)
+@@ -1,17 +1,25 @@
+-export abstract class Event {
++export abstract class Event {
+    /**
+     * Unique event slug
+     * Must be static, because registering events is done by class
+     */
+    static slug: string = "untitled-event";
+    params: Params;
++   _returning!: Returning;
++   static returning: boolean = false;
++
++   setValidatedReturn(value: Returning): void {
++      if (typeof value !== "undefined") {
++         throw new Error("Invalid event return value");
++      }
++   }
+ 
+    constructor(params: Params) {
+       this.params = params;
+    }
+ }
+ 
+-// @todo: current workaround: potentially there is none and that's the way
++// @todo: current workaround: potentially there is "none" and that's the way
+ export class NoParamEvent extends Event {
+    static override slug: string = "noparam-event";
+ 
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000..8e09a00
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,20 @@
+{
+  "include": ["packages/app/env.d.ts", "./scripts/**/*.ts"],
+  "exclude": ["node_modules", "dist/**/*", "build", "public"],
+  "compilerOptions": {
+    "lib": ["DOM", "DOM.Iterable", "ES2022"],
+    "types": ["bun-types"],
+    "isolatedModules": true,
+    "esModuleInterop": true,
+    "jsx": "react-jsx",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "resolveJsonModule": true,
+    "target": "ES2022",
+    "strict": true,
+    "allowJs": true,
+    "noImplicitAny": false,
+    "forceConsistentCasingInFileNames": true,
+    "verbatimModuleSyntax": true
+  }
+}
diff --git a/verdaccio.yml b/verdaccio.yml
new file mode 100644
index 0000000..8c95671
--- /dev/null
+++ b/verdaccio.yml
@@ -0,0 +1,14 @@
+storage: .verdaccio/storage
+auth:
+  htpasswd:
+    file: .verdaccio/htpasswd
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+packages:
+  '**':
+    access: $all
+    publish: $authenticated
+    proxy: npmjs
+log: { type: stdout, format: pretty, level: http }
+