From 001880f5d2ed567fab668b12212930af612ca078 Mon Sep 17 00:00:00 2001 From: Miro Dojkic Date: Fri, 26 Mar 2021 11:55:29 +0100 Subject: [PATCH 1/9] =?UTF-8?q?Decouple=20local=20access=20manager=20from?= =?UTF-8?q?=20proxy=20=F0=9F=9A=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/app.js | 2 +- server/repository/index.js | 4 +- server/repository/proxy.js | 7 +-- .../shared/content-plugins/elementRegistry.js | 2 +- server/shared/storage/proxy/index.js | 27 ++++++--- server/shared/storage/proxy/mw.js | 9 +-- .../providers/cloudfront/access-manager.js | 0 .../{cloudfront.js => cloudfront/index.js} | 0 .../{local.js => local/access-manager.js} | 33 +--------- .../storage/proxy/providers/local/index.js | 60 +++++++++++++++++++ 10 files changed, 93 insertions(+), 51 deletions(-) create mode 100644 server/shared/storage/proxy/providers/cloudfront/access-manager.js rename server/shared/storage/proxy/providers/{cloudfront.js => cloudfront/index.js} (100%) rename server/shared/storage/proxy/providers/{local.js => local/access-manager.js} (56%) create mode 100644 server/shared/storage/proxy/providers/local/index.js diff --git a/server/app.js b/server/app.js index 8ce6b2069..2b8881103 100644 --- a/server/app.js +++ b/server/app.js @@ -8,7 +8,7 @@ const helmet = require('helmet'); const origin = require('./shared/origin'); const path = require('path'); const storage = require('./repository/storage'); -const storageProxy = require('./repository/proxy'); +const storageProxy = require('./shared/storage/proxy'); // eslint-disable-next-line require-sort/require-sort require('express-async-errors'); diff --git a/server/repository/index.js b/server/repository/index.js index 797711075..38e3dd4a1 100644 --- a/server/repository/index.js +++ b/server/repository/index.js @@ -8,11 +8,11 @@ const feed = require('./feed'); const multer = require('multer'); const path = require('path'); const processQuery = require('../shared/util/processListQuery'); -const proxy = require('./proxy'); +const proxyAccessManager = require('./proxy'); const { Repository } = require('../shared/database'); const router = require('express').Router(); const storage = require('./storage'); -const { setSignedCookies } = require('../shared/storage/proxy/mw')(storage, proxy); +const { setSignedCookies } = require('../shared/storage/proxy/mw')(storage, proxyAccessManager); /* eslint-disable require-sort/require-sort */ const activity = require('../activity'); diff --git a/server/repository/proxy.js b/server/repository/proxy.js index 161f85c71..009c04bf8 100644 --- a/server/repository/proxy.js +++ b/server/repository/proxy.js @@ -1,14 +1,13 @@ 'use strict'; -const BaseProxy = require('../shared/storage/proxy'); -const { proxy: config } = require('../../config/server').storage; const path = require('path'); +const proxy = require('../shared/storage/proxy'); const storageCookies = { REPOSITORY: 'Storage-Repository' }; -class Proxy extends BaseProxy { +class RepositoryProxyAccessManager extends proxy.AccessManager { getSignedCookies(repositoryId, maxAge) { const resource = path.join('repository', `${repositoryId}`); return { @@ -31,4 +30,4 @@ class Proxy extends BaseProxy { } } -module.exports = new Proxy(config); +module.exports = new RepositoryProxyAccessManager(proxy.config); diff --git a/server/shared/content-plugins/elementRegistry.js b/server/shared/content-plugins/elementRegistry.js index 83b76b422..060f332d8 100644 --- a/server/shared/content-plugins/elementRegistry.js +++ b/server/shared/content-plugins/elementRegistry.js @@ -8,7 +8,7 @@ const elementsList = require('../../../config/shared/core-elements'); const hooks = require('./elementHooks'); const pick = require('lodash/pick'); const storage = require('../../repository/storage'); -const storageProxy = require('../../repository/proxy'); +const storageProxy = require('../../shared/storage/proxy'); const toCase = require('to-case'); const EXTENSIONS_LIST = '../../../extensions/content-elements/index'; diff --git a/server/shared/storage/proxy/index.js b/server/shared/storage/proxy/index.js index 652815d32..9de71a00e 100644 --- a/server/shared/storage/proxy/index.js +++ b/server/shared/storage/proxy/index.js @@ -2,6 +2,7 @@ const autobind = require('auto-bind'); const path = require('path'); +const { proxy: config } = require('../../../../config/server').storage; class Proxy { constructor(config) { @@ -22,32 +23,40 @@ class Proxy { return this.provider.isSelfHosted; } + get config() { + return this.provider.config; + } + get path() { return this.isSelfHosted && this.provider.path; } - getSignedCookies(resource, maxAge) { - return this.provider.getSignedCookies(resource, maxAge); + get AccessManager() { + return this.provider.AccessManager; + } + + getSignedCookies(resource, maxAge, accessManager) { + return this.provider.getSignedCookies(resource, maxAge, accessManager); } - verifyCookies(cookies, resource) { - return this.provider.verifyCookies(cookies, resource); + verifyCookies(cookies, resource, accessManager) { + return this.provider.verifyCookies(cookies, resource, accessManager); } - hasCookies(cookies) { - return this.provider.hasCookies(cookies); + hasCookies(cookies, ...params) { + return this.provider.hasCookies(cookies, ...params); } getFileUrl(key) { return this.provider.getFileUrl(key); } - getCookieNames() { - return this.provider.getCookieNames(); + getCookieNames(accessManager) { + return this.provider.getCookieNames(accessManager); } } -module.exports = Proxy; +module.exports = new Proxy(config); function loadProvider(name) { try { diff --git a/server/shared/storage/proxy/mw.js b/server/shared/storage/proxy/mw.js index ca46aaca3..45706ceda 100644 --- a/server/shared/storage/proxy/mw.js +++ b/server/shared/storage/proxy/mw.js @@ -3,12 +3,13 @@ const { FORBIDDEN } = require('http-status-codes'); const miss = require('mississippi'); const path = require('path'); +const proxy = require('.'); const router = require('express').Router(); -module.exports = (storage, proxy) => { +module.exports = (storage, proxyAccessManager) => { function getFile(req, res, next) { const key = req.params[0]; - const hasValidCookies = proxy.verifyCookies(req.cookies, key); + const hasValidCookies = proxy.verifyCookies(req.cookies, key, proxyAccessManager); if (!hasValidCookies) return res.status(FORBIDDEN).end(); res.type(path.extname(key)); miss.pipe(storage.createReadStream(key), res, err => { @@ -19,9 +20,9 @@ module.exports = (storage, proxy) => { function setSignedCookies(req, res, next) { const repositoryId = req.repository.id; - if (proxy.hasCookies(req.cookies, repositoryId)) return next(); + if (proxy.hasCookies(req.cookies, repositoryId, proxyAccessManager)) return next(); const maxAge = 1000 * 60 * 60; // 1 hour in ms - const cookies = proxy.getSignedCookies(repositoryId, maxAge); + const cookies = proxy.getSignedCookies(repositoryId, maxAge, proxyAccessManager); Object.entries(cookies).forEach(([cookie, value]) => { res.cookie(cookie, value, { maxAge, httpOnly: true }); }); diff --git a/server/shared/storage/proxy/providers/cloudfront/access-manager.js b/server/shared/storage/proxy/providers/cloudfront/access-manager.js new file mode 100644 index 000000000..e69de29bb diff --git a/server/shared/storage/proxy/providers/cloudfront.js b/server/shared/storage/proxy/providers/cloudfront/index.js similarity index 100% rename from server/shared/storage/proxy/providers/cloudfront.js rename to server/shared/storage/proxy/providers/cloudfront/index.js diff --git a/server/shared/storage/proxy/providers/local.js b/server/shared/storage/proxy/providers/local/access-manager.js similarity index 56% rename from server/shared/storage/proxy/providers/local.js rename to server/shared/storage/proxy/providers/local/access-manager.js index 0086993c1..ea98c52f0 100644 --- a/server/shared/storage/proxy/providers/local.js +++ b/server/shared/storage/proxy/providers/local/access-manager.js @@ -2,36 +2,18 @@ const every = require('lodash/every'); const NodeRSA = require('node-rsa'); -const { origin } = require('../../../../../config/server'); -const urlJoin = require('url-join'); -const { validateConfig } = require('../../validation'); -const yup = require('yup'); -const PROXY_PATH = '/proxy'; const storageCookies = { SIGNATURE: 'Storage-Signature', EXPIRES: 'Storage-Expires' }; -const schema = yup.object().shape({ - privateKey: yup.string().pkcs1().required() -}); - -class Local { +class LocalAccessManager { constructor(config) { - config = validateConfig(config, schema); - this.signer = new NodeRSA(config.privateKey, 'private'); - this.isSelfHosted = true; - this.path = PROXY_PATH; - } - - static create(config) { - return new this(config); } - getSignedCookies(resource, maxAge) { - const expires = getExpirationTime(maxAge); + getSignedCookies(resource, expires) { const signature = this.signer.encrypt({ resource, expires }, 'base64'); return { [storageCookies.SIGNATURE]: signature, @@ -52,18 +34,9 @@ class Local { return every(storageCookies, cookie => cookies[cookie]); } - getFileUrl(key) { - return urlJoin(origin, this.path, key); - } - getCookieNames() { return Object.values(storageCookies); } } -module.exports = { create: Local.create.bind(Local) }; - -function getExpirationTime(maxAge) { - // Expiration unix timestamp in ms - return new Date().getTime() + maxAge; -} +module.exports = LocalAccessManager; diff --git a/server/shared/storage/proxy/providers/local/index.js b/server/shared/storage/proxy/providers/local/index.js new file mode 100644 index 000000000..a8ea9f1f7 --- /dev/null +++ b/server/shared/storage/proxy/providers/local/index.js @@ -0,0 +1,60 @@ +'use strict'; + +const AccessManager = require('./access-manager'); +const last = require('lodash/last'); +const { origin } = require('../../../../../../config/server'); +const urlJoin = require('url-join'); +const { validateConfig } = require('../../../validation'); +const yup = require('yup'); + +const PROXY_PATH = '/proxy'; + +const schema = yup.object().shape({ + privateKey: yup.string().pkcs1().required() +}); + +class Local { + constructor(config) { + this.config = validateConfig(config, schema); + this.accessManager = new AccessManager(this.config); + this.isSelfHosted = true; + this.path = PROXY_PATH; + } + + static create(config) { + return new this(config); + } + + get AccessManager() { + return AccessManager; + } + + getSignedCookies(resource, maxAge, accessManager = this.accessManager) { + const expires = getExpirationTime(maxAge); + return accessManager.getSignedCookies(resource, expires); + } + + verifyCookies(cookies, key, accessManager = this.accessManager) { + return accessManager.verifyCookies(cookies, key); + } + + hasCookies(cookies, ...params) { + const accessManager = last(params) || this.accessManager; + return accessManager.hasCookies(cookies, ...params); + } + + getCookieNames(accessManager = this.accessManager) { + return accessManager.getCookieNames(); + } + + getFileUrl(key) { + return urlJoin(origin, this.path, key); + } +} + +module.exports = { create: Local.create.bind(Local) }; + +function getExpirationTime(maxAge) { + // Expiration unix timestamp in ms + return new Date().getTime() + maxAge; +} From 22d6e64aa252c73029fd61ea78b2cc6d1667366d Mon Sep 17 00:00:00 2001 From: Miro Dojkic Date: Fri, 26 Mar 2021 13:41:19 +0100 Subject: [PATCH 2/9] =?UTF-8?q?Enable=20saving=20file=20in=20the=20storage?= =?UTF-8?q?=20folder=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/asset.js | 13 ++++---- client/components/common/mixins/upload.js | 3 +- server/repository/index.js | 2 -- server/router.js | 2 ++ server/shared/storage/helpers.js | 2 +- server/shared/storage/storage.controller.js | 33 ++++++++++++--------- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/client/api/asset.js b/client/api/asset.js index 2ca18bc65..3674a997e 100644 --- a/client/api/asset.js +++ b/client/api/asset.js @@ -1,16 +1,17 @@ import request from './request'; const urls = { - base: repositoryId => `/repositories/${repositoryId}/assets` + base: 'assets' }; -function getUrl(repositoryId, key) { - const params = { key }; - return request.get(urls.base(repositoryId), { params }).then(res => res.data.url); +function getUrl(folder, key) { + const params = { key: `${folder}/${key}` }; + return request.get(urls.base, { params }).then(res => res.data.url); } -function upload(repositoryId, data) { - return request.post(urls.base(repositoryId), data).then(res => res.data); +function upload(folder, data) { + if (folder) data.append('folder', folder); + return request.post(urls.base, data).then(res => res.data); } export default { diff --git a/client/components/common/mixins/upload.js b/client/components/common/mixins/upload.js index 2da93b7e5..b4a591f82 100644 --- a/client/components/common/mixins/upload.js +++ b/client/components/common/mixins/upload.js @@ -18,7 +18,8 @@ export default { }, upload: loader(function (e) { this.createFileForm(e); - return this.$storageService.upload(this.repositoryId, this.form) + const folder = `repository/${this.repositoryId}`; + return this.$storageService.upload(folder, this.form) .then(data => { const { name } = this.form.get('file'); this.$emit('upload', { ...data, name }); diff --git a/server/repository/index.js b/server/repository/index.js index 38e3dd4a1..5865bbad7 100644 --- a/server/repository/index.js +++ b/server/repository/index.js @@ -19,7 +19,6 @@ const activity = require('../activity'); const comment = require('../comment'); const revision = require('../revision'); const contentElement = require('../content-element'); -const storageRouter = require('../shared/storage/storage.router'); /* eslint-enable */ // NOTE: disk storage engine expects an object to be passed as the first argument @@ -59,7 +58,6 @@ mount(router, '/:repositoryId', activity); mount(router, '/:repositoryId', revision); mount(router, '/:repositoryId', contentElement); mount(router, '/:repositoryId', comment); -mount(router, '/:repositoryId', storageRouter); function mount(router, mountPath, subrouter) { return router.use(path.join(mountPath, subrouter.path), subrouter.router); diff --git a/server/router.js b/server/router.js index 17d76631d..31fbfe1dc 100644 --- a/server/router.js +++ b/server/router.js @@ -5,6 +5,7 @@ const { authenticate } = require('./shared/auth'); const express = require('express'); const { extractAuthData } = require('./shared/auth/mw'); const repository = require('./repository'); +const storage = require('./shared/storage/storage.router'); const tag = require('./tag'); const user = require('./user'); @@ -24,6 +25,7 @@ authConfig.oidc.enabled && (() => { // Protected routes: router.use(authenticate('jwt')); router.use(repository.path, repository.router); +router.use(storage.path, storage.router); router.use(tag.path, tag.router); module.exports = router; diff --git a/server/shared/storage/helpers.js b/server/shared/storage/helpers.js index b21c99804..1903e9291 100644 --- a/server/shared/storage/helpers.js +++ b/server/shared/storage/helpers.js @@ -4,7 +4,7 @@ const { elementRegistry } = require('../content-plugins'); const get = require('lodash/get'); const config = require('../../../config/server').storage; const Promise = require('bluebird'); -const proxy = require('../../repository/proxy'); +const proxy = require('../storage/proxy'); const set = require('lodash/set'); const toPairs = require('lodash/toPairs'); const values = require('lodash/values'); diff --git a/server/shared/storage/storage.controller.js b/server/shared/storage/storage.controller.js index 5310c48e3..39a8f6657 100644 --- a/server/shared/storage/storage.controller.js +++ b/server/shared/storage/storage.controller.js @@ -1,6 +1,5 @@ 'use strict'; -const { getFileUrl, getPath, saveFile } = require('../../repository/storage'); const { readFile, sha256 } = require('./util'); const config = require('../../../config/server').storage; const fecha = require('fecha'); @@ -9,49 +8,55 @@ const JSZip = require('jszip'); const mime = require('mime-types'); const path = require('path'); const pickBy = require('lodash/pickBy'); +const Storage = require('./'); const getStorageUrl = key => `${config.protocol}${key}`; +const storage = new Storage(config); function getUrl(req, res) { const { query: { key } } = req; - return getFileUrl(key).then(url => res.json({ url })); + return storage.getFileUrl(key).then(url => res.json({ url })); } -async function upload({ file, body, user, repository }, res) { +async function upload({ file, body, user }, res) { + const { folder, unpack } = body; const { name } = path.parse(file.originalname); - const { id: repositoryId } = repository; - if (body.unpack) { + if (unpack) { const timestamp = fecha.format(new Date(), 'YYYY-MM-DDTHH:mm:ss'); const root = `${timestamp}__${user.id}__${name}`; - const assets = await uploadArchiveContent(repositoryId, file, root); + const assets = await uploadArchiveContent(folder, file, root); return res.json({ root, assets }); } - const asset = await uploadFile(repositoryId, file, name); + const asset = await uploadFile(folder, file, name); return res.json(asset); } module.exports = { getUrl, upload }; -async function uploadFile(repositoryId, file, name) { +async function uploadFile(folder, file, name) { const buffer = await readFile(file); const hash = sha256(file.originalname, buffer); const extension = path.extname(file.originalname); const fileName = `${hash}___${name}${extension}`; - const key = path.join(getPath(repositoryId), fileName); - await saveFile(key, buffer, { ContentType: file.mimetype }); - const publicUrl = await getFileUrl(key); + const keyComponents = [folder, fileName].filter(Boolean); + const key = path.join(...keyComponents); + await storage.saveFile(key, buffer, { ContentType: file.mimetype }); + const publicUrl = await storage.getFileUrl(key); return { key, publicUrl, url: getStorageUrl(key) }; } -async function uploadArchiveContent(repositoryId, archive, name) { +async function uploadArchiveContent(folder, archive, name) { const buffer = await readFile(archive); const content = await JSZip.loadAsync(buffer); const files = pickBy(content.files, it => !it.dir); const keys = await Promise.all(Object.keys(files).map(async src => { - const key = path.join(getPath(repositoryId), name, src); + const keyComponents = [folder, name, src].filter(Boolean); + const key = path.join(...keyComponents); const file = await content.file(src).async('uint8array'); const mimeType = mime.lookup(src); - await saveFile(key, Buffer.from(file), { ContentType: mimeType }); + await storage.saveFile(key, Buffer.from(file), { + ContentType: mimeType + }); return [key, getStorageUrl(key)]; })); return fromPairs(keys); From ac0d881facaa24c2073cac7df18b705ba625ebc3 Mon Sep 17 00:00:00 2001 From: Miro Dojkic Date: Fri, 26 Mar 2021 20:35:38 +0100 Subject: [PATCH 3/9] =?UTF-8?q?Parametrize=20upload=20mixin=20as=20provide?= =?UTF-8?q?r=20component=20=E2=99=BB=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/asset.js | 5 +- client/components/common/FileInput.vue | 79 ++++++++++--------- client/components/common/InputAsset.vue | 2 + .../mixins/{upload.js => UploadProvider.vue} | 24 ++++-- .../components/common/tce-core/UploadBtn.vue | 23 +++--- .../tce-audio/edit/Toolbar.vue | 1 + .../tce-image/edit/Toolbar.vue | 1 + .../content-elements/tce-pdf/edit/Toolbar.vue | 1 + .../tce-video/edit/Toolbar.vue | 1 + 9 files changed, 81 insertions(+), 56 deletions(-) rename client/components/common/mixins/{upload.js => UploadProvider.vue} (72%) diff --git a/client/api/asset.js b/client/api/asset.js index 3674a997e..339182fe5 100644 --- a/client/api/asset.js +++ b/client/api/asset.js @@ -5,12 +5,11 @@ const urls = { }; function getUrl(folder, key) { - const params = { key: `${folder}/${key}` }; + const params = { key: folder ? `${folder}/${key}` : key }; return request.get(urls.base, { params }).then(res => res.data.url); } -function upload(folder, data) { - if (folder) data.append('folder', folder); +function upload(data) { return request.post(urls.base, data).then(res => res.data); } diff --git a/client/components/common/FileInput.vue b/client/components/common/FileInput.vue index 4a902042a..e9a606446 100644 --- a/client/components/common/FileInput.vue +++ b/client/components/common/FileInput.vue @@ -1,49 +1,56 @@ diff --git a/client/components/common/InputAsset.vue b/client/components/common/InputAsset.vue index fc79b9ef9..b0c857928 100644 --- a/client/components/common/InputAsset.vue +++ b/client/components/common/InputAsset.vue @@ -15,6 +15,7 @@ :uploading.sync="uploading" :validate="{ ext: extensions }" :confirm-deletion="false" + :folder="folder" :label="uploadLabel" class="upload-btn" />