commit 30d6918b83cb76463dd8a297c67fa189f5513ad9 Author: Christoph Wiechert Date: Sun Apr 23 13:41:29 2017 +0200 init diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9765ae8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.idea +data +temp +node_modules +app/node_modules +npm-debug.log +scripts +docs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..5395b40 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# EditorConfig helps developers define and maintain consistent +# coding styles between different editors and IDEs +# editorconfig.org + +root = true + +[*] + +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md, *.pug] +trim_trailing_whitespace = false + +[*.yml] +indent_style = space diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..175190a --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +data +temp +node_modules +npm-debug.log diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..02a819f --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v7 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..75a2fb3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM node:7-alpine + +ENV PSITRANSFER_UPLOAD_DIR=/data \ + NODE_ENV=production + +MAINTAINER Christoph Wiechert + +WORKDIR /app + +ADD *.js package.json README.md /app/ +ADD lib /app/lib +ADD app /app/app +ADD public /app/public + +# Rebuild the frontend apps +RUN cd app && \ + NODE_ENV=dev npm install --quiet 1>/dev/null && \ + npm run build && \ + cd .. && rm -rf app + +# Install backend dependencies +RUN mkdir /data && \ + chown node /data && \ + npm install --quiet 1>/dev/null + +EXPOSE 3000 +VOLUME ["/data"] + +USER node + +HEALTHCHECK CMD wget -O /dev/null -q http://localhost:3000 + +CMD ["node", "app.js"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a62afad --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright 2017 Christoph Wiechert + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7ed778b --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +# PsiTransfer + +Simple open source self-hosted file sharing solution. + +* Supports many and very big files (Streams ftw) +* Resumable up- and downloads ([TUS](https://tus.io)) +* Set an expire-time for your upload bucket +* One-time downloads +* Password protected downloads ([AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)) +* Requires Node >=7.4 + +![Screenshot](https://raw.githubusercontent.com/psi-4ward/psitransfer/docs/psitransfer.gif) + +**Demo**: https://transfer.psi.cx + +## Quickstart + +### Docker (recommended) +```bash +$ docker run -p 0.0.0.0:3000:3000 -v $PWD/data:/data psitrax/psitransfer +# data volume needs UID 1000 +$ sudo chown -R 1000 $PWD/data +``` + +### Manual + +```bash +# Be sure to have NodeJS >= 7.4 +$ node -v +v7.4.0 + +# Download and extract latest release package from +# https://github.com/psi-4ward/psitransfer/releases + +# Install dependencies and start the app +$ NODE_ENV=production npm install +$ npm start +``` + +### Configuration + +There are some configs in `config.js` like port and data-dir. +You can: +* Edit the `config.js` **(not recommend)** +* Add a `config.production.js` where `production` is the value from `NODE_ENV` + See `config.dev.js` +* Define environment Variables like `PSITRANSFER_UPLOAD_DIR` + +### Customization + +`public/upload.html` and `download.html` are kept simple. +You can alter these files and add your logo and styles. +The following elements are mandatory: +`common.js` and respectively `upload.js`, `download.js` as well as `
`, `
` +Please keep a footnote like *Powered by PsiTransfer* :) + +### Debug + +Psitransfer uses [debug](https://github.com/visionmedia/debug): + +```bash +DEBUG=psitransfer:* npm start +``` + +## License + +[BSD](LICENSE) + diff --git a/app.js b/app.js new file mode 100644 index 0000000..fecb48f --- /dev/null +++ b/app.js @@ -0,0 +1,29 @@ +'use strict'; +const config = require('./config'); +const app = require('./lib/endpoints'); + +/** + * Naming: + * sid: Group of files + * key: File + * fid: {sid}++{key} + */ + +const server = app.listen(config.port, config.iface, () => { + console.log(`PsiTransfer listening on http://${config.iface}:${config.port}`); +}); + + +// graceful shutdown +function shutdown() { + console.log('PsiTransfer shutting down...'); + server.close(() => { + process.exit(0); + }); + setTimeout(function() { + console.log('Could not close connections in time, forcefully shutting down'); + process.exit(0); + }, 180 * 1000); +} +process.on('SIGTERM', shutdown); +process.on('SIGINT', shutdown); diff --git a/app/.babelrc b/app/.babelrc new file mode 100644 index 0000000..f64b829 --- /dev/null +++ b/app/.babelrc @@ -0,0 +1,15 @@ +{ + "presets": [ + [ + "env", + { + "modules": false + } + ], + "stage-2" + ], + "plugins": [ + "transform-runtime" + ], + "comments": false +} diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 0000000..060b35b --- /dev/null +++ b/app/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +node_modules/ +dist/ +npm-debug.log diff --git a/app/README.md b/app/README.md new file mode 100644 index 0000000..1b63158 --- /dev/null +++ b/app/README.md @@ -0,0 +1,14 @@ +# PsiTransfer Upload / Download App + +## Build Setup + +``` bash +# install dependencies +npm install + +# serve with hot reload at localhost:8080 +npm run dev + +# build for production with minification +npm run build +``` diff --git a/app/package.json b/app/package.json new file mode 100644 index 0000000..6c1c300 --- /dev/null +++ b/app/package.json @@ -0,0 +1,35 @@ +{ + "name": "psitransfer", + "description": "A Vue.js project", + "version": "1.0.0", + "author": "Christoph Wiechert ", + "private": true, + "scripts": { + "dev": "cross-env NODE_ENV=development webpack-dev-server --inline --hot --host 0.0.0.0", + "build": "cross-env NODE_ENV=production webpack --progress --hide-modules" + }, + "dependencies": { + "babel-polyfill": "^6.23.0", + "crypto-js": "^3.1.9-1", + "drag-drop": "^2.13.2", + "tus-js-client": "^1.4.3", + "uuid": "^3.0.1", + "vue": "^2.2.6", + "vuex": "^2.3.1" + }, + "devDependencies": { + "babel-core": "^6.24.1", + "babel-loader": "^6.4.1", + "babel-plugin-transform-runtime": "^6.23.0", + "babel-preset-env": "^1.3.2", + "babel-preset-stage-2": "^6.22.0", + "cross-env": "^4.0.0", + "css-loader": "^0.28.0", + "file-loader": "^0.11.1", + "pug": "^2.0.0-beta.12", + "vue-loader": "^11.3.4", + "vue-template-compiler": "^2.2.6", + "webpack": "^2.4.1", + "webpack-dev-server": "^2.4.2" + } +} diff --git a/app/src/Download.vue b/app/src/Download.vue new file mode 100644 index 0000000..860f91c --- /dev/null +++ b/app/src/Download.vue @@ -0,0 +1,140 @@ + + + + diff --git a/app/src/Upload.vue b/app/src/Upload.vue new file mode 100644 index 0000000..2b400da --- /dev/null +++ b/app/src/Upload.vue @@ -0,0 +1,80 @@ + + + diff --git a/app/src/Upload/Files.vue b/app/src/Upload/Files.vue new file mode 100644 index 0000000..b69440b --- /dev/null +++ b/app/src/Upload/Files.vue @@ -0,0 +1,74 @@ + + + + + + diff --git a/app/src/Upload/Settings.vue b/app/src/Upload/Settings.vue new file mode 100644 index 0000000..1f27fd6 --- /dev/null +++ b/app/src/Upload/Settings.vue @@ -0,0 +1,63 @@ + + + diff --git a/app/src/Upload/store.js b/app/src/Upload/store.js new file mode 100644 index 0000000..247f46e --- /dev/null +++ b/app/src/Upload/store.js @@ -0,0 +1,41 @@ +'use strict'; +import Vue from 'vue'; +import Vuex from 'vuex'; +Vue.use(Vuex); + +import config from './store/config.js'; +import upload from './store/upload.js'; + +export default new Vuex.Store({ + modules: { + config, + upload + }, + + state: { + error: '', + // disable all input fields + disabled: false, + /* States: + * new: can modify settings and add/remove files + * uploading: probably let user pause/cancel upload + * uploaded: show download link + * uploadError: show retry btn */ + state: 'new' + }, + + mutations: { + ERROR(state, msg) { + state.error = msg; + state.disabled = true; + }, + DISABLE(state) { + state.disabled = true; + }, + STATE(state, val) { + state.state = val; + if(val !== 'new') state.disabled = true; + }, + }, + +}); diff --git a/app/src/Upload/store/config.js b/app/src/Upload/store/config.js new file mode 100644 index 0000000..858669b --- /dev/null +++ b/app/src/Upload/store/config.js @@ -0,0 +1,39 @@ +'use strict'; +import Vue from 'vue'; + +export default { + namespaced: true, + + state: {}, + + mutations: { + SET(state, val) { + for (let k in val) { + Vue.set(state, k, val[k]); + } + } + }, + + actions: { + fetch({commit, }) { + const xhr = new XMLHttpRequest(); + xhr.open('GET', '/config.json'); + xhr.onload = () => { + if(xhr.status === 200) { + try { + const conf = JSON.parse(xhr.responseText); + commit('SET', conf); + commit('upload/RETENTION', conf.defaultRetention, {root:true}); + } + catch(e) { + commit('ERROR', `Config parse Error: ${e.message}`, {root: true}); + } + } + else { + commit('ERROR', `Config load error: ${xhr.status} ${xhr.statusText}`, {root: true}); + } + }; + xhr.send(); + } + } +} diff --git a/app/src/Upload/store/upload.js b/app/src/Upload/store/upload.js new file mode 100644 index 0000000..d4aa8a6 --- /dev/null +++ b/app/src/Upload/store/upload.js @@ -0,0 +1,184 @@ +'use strict'; +import tus from "tus-js-client"; +import uuid from 'uuid/v4'; +import md5 from 'crypto-js/md5'; + +function humanFileSize(fileSizeInBytes) { + let i = -1; + const byteUnits = [' kB', ' MB', ' GB', ' TB', 'PB', 'EB', 'ZB', 'YB']; + do { + fileSizeInBytes = fileSizeInBytes / 1024; + i++; + } + while(fileSizeInBytes > 1024); + return Math.max(fileSizeInBytes, 0.01).toFixed(2) + byteUnits[i]; +} + +let onOnlineHandler = null; +let onOnlineHandlerAttached = false; + +export default { + namespaced: true, + + state: { + retention: null, + password: '', + files: [], + sid: md5(uuid()).toString().substr(0, 12), + bytesUploaded: 0, + + }, + + getters: { + shareUrl: state => { + return document.location.protocol + '//' + document.location.host + '/' + state.sid; + }, + percentUploaded: state => { + return Math.min( + Math.round(state.files.reduce((sum, file) => sum += file.progress.percentage, 0) / state.files.length), 100); + } + }, + + mutations: { + RETENTION(state, seconds) { + state.retention = seconds; + }, + PASSWORD(state, pwd) { + state.password = pwd; + }, + ADD_FILE(state, file) { + state.files.splice(0, 0, file); + }, + REMOVE_FILE(state, file) { + let index = state.files.indexOf(file); + if(index > -1) state.files.splice(index, 1); + }, + UPDATE_FILE(state, payload) { + for(let k in payload.data) { + payload.file[k] = payload.data[k]; + } + }, + NEW_SESSION(state) { + state.password = ''; + state.files.splice(0, state.files.length); + state.sid = md5(uuid()).toString().substr(0, 12); + } + }, + + + actions: { + addFiles({commit, state}, files) { + if(state.disabled) return; + for(let i = 0; i < files.length; i++) { + // wrap, don't change the HTML5-File-API object + commit('ADD_FILE', { + _File: files[i], + name: files[i].name, + comment: '', + progress: {percentage: 0, humanSize: 0}, + uploaded: false, + error: false, + humanSize: humanFileSize(files[i].size), + _retryDelay: 500, + _retries: 0 + }); + } + }, + + upload({commit, dispatch, state}) { + commit('STATE', 'uploading', {root:true}); + commit('ERROR', '', {root:true}); + + if(onOnlineHandler === null) { + onOnlineHandler = function() { + onOnlineHandlerAttached = false; + commit('ERROR', false, {root: true}); + dispatch('upload'); + } + } + if(onOnlineHandlerAttached) window.removeEventListener('online', onOnlineHandler); + + // upload all files in parallel + state.files.forEach(file => { + file.error = ''; + file._retries = 0; + file._retryDelay = 500; + + const _File = file._File; + let tusUploader = new tus.Upload(_File, { + metadata: { + sid: state.sid, + retention: state.retention, + password: state.password, + name: file.name, + comment: file.comment, + type: file._File.type + }, + resume: true, + endpoint: "/files/", + fingerprint: (file) => { + // include sid to prevent duplicate file detection on different session + return ["tus", state.sid, file.name, file.type, file.size, file.lastModified].join("-"); + }, + retryDelays: null, + onError(error) { + // browser is offline + if(!navigator.onLine) { + commit('ERROR', 'You are offline. Your uploads will resume as soon as you are back online.', {root: true}); + if(!onOnlineHandlerAttached) { + onOnlineHandlerAttached = true; + // attach onOnline handler + window.addEventListener('online', onOnlineHandler); + } + } + // Client Error + else if(error && error.originalRequest && + error.originalRequest.status >= 400 && error.originalRequest.status < 500) + { + commit('UPDATE_FILE', {file, data: {error: error.message || error.toString()}}); + } + // Generic Error + else { + if(file._retries > 30) { + commit('UPDATE_FILE', {file, data: {error: error.message || error.toString()}}); + if(state.files.every(f => f.error)) { + commit('STATE', 'uploadError', {root: true}); + commit('ERROR', 'Upload failed.', {root: true}); + } + return; + } + + file._retryDelay = Math.min(file._retryDelay*1.7, 10000); + file._retries++; + if(console) console.log(error.message || error.toString(), '; will retry in', file._retryDelay, 'ms'); + setTimeout(() => tusUploader.start(), file._retryDelay); + } + }, + onProgress(bytesUploaded, bytesTotal) { + // uploaded=total gets also emitted on error + if(bytesUploaded === bytesTotal) return; + + file.error = ''; + file._retries = 0; + file._retryDelay = 500; + const percentage = Math.round(bytesUploaded / bytesTotal * 10000) / 100; + commit('UPDATE_FILE', { + file, + data: {progress: {percentage, humanSize: humanFileSize(bytesUploaded)}} + }); + }, + onSuccess() { + localStorage.removeItem(tusUploader._fingerprint); + commit('UPDATE_FILE', {file, data: { + uploaded:true, + progress: {percentage: 100, humanFileSize: file.humanSize} + }}); + if(state.files.every(f => f.uploaded)) commit('STATE', 'uploaded', {root: true}); + } + }); + tusUploader.start(); + }); + } + } + +} diff --git a/app/src/common/Clipboard.vue b/app/src/common/Clipboard.vue new file mode 100644 index 0000000..e01ee43 --- /dev/null +++ b/app/src/common/Clipboard.vue @@ -0,0 +1,62 @@ + + + + + + diff --git a/app/src/common/FileIcon.vue b/app/src/common/FileIcon.vue new file mode 100644 index 0000000..8ebfbaa --- /dev/null +++ b/app/src/common/FileIcon.vue @@ -0,0 +1,37 @@ + + + + diff --git a/app/src/download.js b/app/src/download.js new file mode 100644 index 0000000..3945638 --- /dev/null +++ b/app/src/download.js @@ -0,0 +1,10 @@ +import 'babel-polyfill'; +import Vue from 'vue'; +import Download from './Download.vue'; + +new Vue({ + el: '#download', + render: h => h(Download) +}); + +window.PSITRANSFER_VERSION = PSITRANSFER_VERSION; diff --git a/app/src/upload.js b/app/src/upload.js new file mode 100644 index 0000000..797877b --- /dev/null +++ b/app/src/upload.js @@ -0,0 +1,17 @@ +"use strict"; + +import 'babel-polyfill'; +import Vue from 'vue'; +import Upload from './Upload.vue'; +import store from './Upload/store.js'; + +new Vue({ + el: '#upload', + store, + render: h => h(Upload), + beforeCreate() { + this.$store.dispatch('config/fetch'); + } +}); + +window.PSITRANSFER_VERSION = PSITRANSFER_VERSION; diff --git a/app/webpack.config.js b/app/webpack.config.js new file mode 100644 index 0000000..eac4a7f --- /dev/null +++ b/app/webpack.config.js @@ -0,0 +1,97 @@ +const path = require('path'); +const webpack = require('webpack'); +const execSync = require('child_process').execSync; + +let commitShaId; + +try { + commitShaId = '#'+execSync('git rev-parse HEAD').toString().substr(0,10); +} catch (e) {} + +module.exports = { + entry: { + upload: './src/upload.js', + download: './src/download.js', + }, + output: { + path: path.resolve(__dirname, '../public/app'), + publicPath: '/app/', + filename: '[name].js' + }, + plugins: [ + new webpack.optimize.CommonsChunkPlugin({ + filename: "common.js", + name: "common" + }), + new webpack.DefinePlugin({ + 'process.env': { + NODE_ENV: process.env.NODE_ENV !== 'development' ? '"production"' : '"development"', + }, + PSITRANSFER_VERSION: '"' + (process.env.PSITRANSFER_VERSION || commitShaId || 'dev') + '"' + }), + ], + module: { + rules: [ + { + test: /\.vue$/, + loader: 'vue-loader', + options: { + loaders: {} + } + }, + { + test: /\.js$/, + loader: 'babel-loader', + exclude: /node_modules/ + }, + { + test: /\.(png|jpg|gif|svg)$/, + loader: 'file-loader', + options: { + name: '[name].[ext]?[hash]' + } + } + ] + }, + resolve: { + alias: { + 'vue$': 'vue/dist/vue.common.js' + } + }, + devServer: { + historyApiFallback: true, + noInfo: true, + proxy: [ + // Proxy requests to BE in DEV mode + // https://webpack.github.io/docs/webpack-dev-server.html#proxy + { + // everything except of js, html, css + context: ['/**', '!/**.js', '!/**.html', '!/**.css'], + target: 'http://localhost:3000/' + } + ] + }, + performance: { + hints: false + }, + devtool: '#eval-source-map' +}; + +if (process.env.NODE_ENV !== 'development') { + module.exports.devtool = '#source-map'; + let commit; + + // http://vue-loader.vuejs.org/en/workflow/production.html + module.exports.plugins = [ + ...module.exports.plugins, + new webpack.optimize.UglifyJsPlugin({ + sourceMap: true, + compress: { + warnings: false + } + }), + new webpack.LoaderOptionsPlugin({ + minimize: true + }) + ]; +} diff --git a/config.dev.js b/config.dev.js new file mode 100644 index 0000000..2c0d181 --- /dev/null +++ b/config.dev.js @@ -0,0 +1,5 @@ +'use strict'; + +module.exports = { + accessLog: 'dev' +}; diff --git a/config.js b/config.js new file mode 100644 index 0000000..e5a3049 --- /dev/null +++ b/config.js @@ -0,0 +1,51 @@ +'use strict'; +const path = require('path'); +const fsp = require('fs-promise'); + +// Default Config +// Do not edit this, generate a config..js for your NODE_ENV +// or use ENV-VARS like PSITRANSFER_PORT=8000 +const config = { + uploadDir: path.resolve(__dirname + '/data'), + port: 3000, + iface: '0.0.0.0', + // retention options in seconds:label; first one is the default + retentions: { + "one-time": "one time download", + 3600: "1 Hour", + 21600: "6 Hours", + 86400: "1 Day", + 259200: "3 Days", + 604800: "1 Week", + 1209600: "2 Weeks", + 2419200: "4 Weeks", + 4838400: "8 Weeks" + }, + defaultRetention: 604800, + mailTemplate: 'mailto:?subject=File Transfer&body=You can download the files here: %%URL%%', + // see https://github.com/expressjs/morgan + // set to false to disable logging + accessLog: ':date[iso] :method :url :status :response-time :remote-addr' +}; + + +// Load NODE_ENV specific config +const envConfFile = path.resolve(__dirname, `config.${process.env.NODE_ENV}.js`); +if(process.env.NODE_ENV && fsp.existsSync(envConfFile)) { + Object.assign(config, require(envConfFile)); +} + +// Load config from ENV VARS +let envName; +for (let k in config) { + envName = 'PSITRANSFER_'+ k.replace(/([A-Z])/g, $1 => "_" + $1).toUpperCase(); + if(process.env[envName]) { + if(typeof config[k] === 'number') { + config[k] = parseInt(process.env[envName], 10); + } else { + config[k] = process.env[envName]; + } + } +} + +module.exports = config; diff --git a/docs/psitransfer.gif b/docs/psitransfer.gif new file mode 100644 index 0000000..5afe866 Binary files /dev/null and b/docs/psitransfer.gif differ diff --git a/lib/db.js b/lib/db.js new file mode 100644 index 0000000..f11f2fe --- /dev/null +++ b/lib/db.js @@ -0,0 +1,118 @@ +'use strict'; +const fsp = require('fs-promise'); +const path = require('path'); +const debug = require('debug')('psitransfer:db'); + +module.exports = class DB { + + constructor(uploadDir, store) { + this.initialized = false; + this.db = {}; + this.expireTimers = {}; + + this.uploadDir = uploadDir; + this.store = store; + + // delete expired files + const gc = () => { + let sid,f,expires; + for (sid of Object.keys(this.db)) { + for (f of this.db[sid]) { + // no removal of one-time downloads + if(!Number.isInteger(f.metadata.retention)) return; + + expires = (+f.metadata.createdAt) + (+f.metadata.retention * 1000) - Date.now(); + if(expires <= 0) { + debug(`Expired ${sid}++${f.key}`); + this.remove(sid, f.key).catch(e => console.error(e)); + } + } + } + }; + setInterval(gc, 60*1000); + + } + + + init() { + if(this.initialized) return; + this.initialized = true; + + try { + this._sync(); + } catch(e) { + this.initialized = false; + e.message = `db initialization failed with error ${e.message}`; + throw e; + } + } + + + /** + * @private + */ + _sync() { + fsp.ensureDirSync(this.uploadDir); + + fsp.readdirSync(this.uploadDir).forEach((sid) => { + this._import(sid); + }); + } + + + /** + * @private + */ + _import(sid) { + const p = path.resolve(this.uploadDir, sid); + const stat = fsp.statSync(p); + if(!stat.isDirectory()) return; + + fsp.readdirSync(p).forEach(async (key) => { + if(path.extname(key) !== '') { + return Promise.resolve(); + } + try { + let info = await this.store.info(`${sid}++${key}`); + this.add(sid, key, info); + } catch(e) { + console.error(e); + } + }); + } + + + add(sid, key, data) { + if(!this.initialized) throw new Error('DB not initialized_'); + if(!this.db[sid]) this.db[sid] = []; + data.key = key; + + const old = this.db[sid].findIndex(i => i.key === key); + if(old !== -1) { + this.db[sid].splice(old, 1, data); + debug(`Updated ${sid}++${key}`); + } else { + this.db[sid].push(data); + debug(`Added ${sid}++${key}`); + } + } + + + async remove(sid, key) { + if(!this.initialized) throw new Error('DB not initialized'); + debug(`Remove ${sid}++${key}`); + await this.store.del(sid + '++' + key); + const i = this.get(sid).find(item => item.key === key); + this.get(sid).splice(i, 1); + if(this.get(sid).length === 0) { + delete this.db[sid]; + await fsp.rmdir(path.resolve(this.uploadDir, sid)); + } + } + + + get(sid) { + return this.db[sid]; + } + +}; diff --git a/lib/endpoints.js b/lib/endpoints.js new file mode 100644 index 0000000..734f17a --- /dev/null +++ b/lib/endpoints.js @@ -0,0 +1,145 @@ +'use strict'; +const config = require('../config'); +const tusboy = require('tusboy').default; +const express = require('express'); +const morgan = require('morgan'); +const compression = require('compression'); +const Store = require('./store'); +const uuid = require('uuid/v4'); +const path = require('path'); +const fs = require("fs"); +const tusMeta = require('tus-metadata'); +const assert = require('assert'); +const AES = require("crypto-js/aes"); +const debug = require('debug')('psitransfer:main'); + +const errorPage = fs.readFileSync(path.join(__dirname, '../public/html/error.html')).toString(); +const store = new Store(config.uploadDir); +const Db = require('./db'); +const db = new Db(config.uploadDir, store); +db.init(); +const app = express(); + +app.disable('x-powered-by'); +app.use(compression()); + +if(config.accessLog) { + app.use(morgan(config.accessLog)); +} + +// Static files +app.use('/app', express.static(path.join(__dirname, '../public/app'))); +app.use('/assets', express.static(path.join(__dirname, '../public/assets'))); + + +// Upload App +app.get('/', (req, res) => { + res.sendFile(path.join(__dirname, '../public/html/upload.html')); +}); + +// Config +app.get('/config.json', (req, res) => { + res.json({ + retentions: config.retentions, + defaultRetention: config.defaultRetention, + mailTemplate: config.mailTemplate + }); +}); + + +// List files / Download App +app.get('/:sid', (req, res, next) => { + if(req.url.endsWith('.json')) { + const sid = req.params.sid.substr(0, req.params.sid.length-5); + if(!db.get(sid)) return res.status(404).end(); + + res.json(db.get(sid).map(data => { + const item = Object.assign(data, {url: `/files/${sid}++${data.key}`}); + if(item.metadata.password) { + return AES.encrypt(JSON.stringify(data), item.metadata.password).toString(); + } else { + return item; + } + })); + } else { + if(!db.get(req.params.sid)) return next(); + res.sendFile(path.join(__dirname, '../public/html/download.html')); + } +}); + + +// Download single file +app.get('/files/:fid', async(req, res, next) => { + // let tusboy handle HEAD with Tus Header + if(req.method === 'HEAD' && req.get('Tus-Resumable')) return next(); + + debug(`Download ${req.params.fid}`); + try { + const info = await store.info(req.params.fid); // throws on 404 + res.download(store.getFilename(req.params.fid), info.metadata.name); + + // remove one-time files after download + if(info.metadata.retention === 'one-time') { + res.on('finish', async () => { + await db.remove(info.metadata.sid, info.metadata.key); + }); + } + } catch(e) { + res.status(404).send(errorPage.replace('%%ERROR%%', e.message)); + } +}); + + +// Upload file +app.use('/files', + function(req, res, next) { + if(req.method === 'GET') return res.status(405).end(); + + if(req.method === 'POST') { + // validate meta-data + // !! tusMeta.encode supports only strings !! + const meta = tusMeta.decode(req.get('Upload-Metadata')); + + try { + assert(meta.name, 'tus meta prop missing: name'); + assert(meta.sid, 'tus meta prop missing: sid'); + assert(meta.retention, 'tus meta prop missing: retention'); + assert(Object.keys(config.retentions).indexOf(meta.retention) >= 0, + `invalid tus meta prop retention. Value ${meta.retention} not in [${Object.keys(config.retentions).join(',')}]`); + + meta.key = uuid(); + meta.createdAt = Date.now().toString(); + + // store changed metadata for tusboy + req.headers['upload-metadata'] = tusMeta.encode(meta); + // for tusboy getKey() + req.FID = meta.sid + '++' + meta.key; + + db.add(meta.sid, meta.key, { + "isPartial": true, + metadata: meta + }); + } + catch(e) { + return res.status(400).end(e.message); + } + } + + next(); + }, + + // let tusboy handle the upload + tusboy(store, { + getKey: req => req.FID, + afterComplete: (req, upload, fid) => { + db.add(upload.metadata.sid, upload.metadata.key, upload); + debug(`Completed upload ${fid}, size=${upload.size} name=${upload.metadata.name}`); + }, + }) +); + +app.use((req, res, next) => { + res.status(404).send(errorPage.replace('%%ERROR%%', 'Download bucket not found.')); +}); + +module.exports = app; diff --git a/lib/store.js b/lib/store.js new file mode 100644 index 0000000..bc51c4d --- /dev/null +++ b/lib/store.js @@ -0,0 +1,113 @@ +'use strict'; +const fsp = require('fs-promise'); +const path = require('path'); +const Transform = require('stream').Transform; +const debug = require('debug')('psitransfer:store'); +const httpErrors = require('http-errors'); + +class StreamLen extends Transform { + constructor(options) { + super(options); + this.bytes = 0; + } + + _transform(chunk, encoding, cb) { + this.bytes += chunk.length; + this.push(chunk); + cb(); + } +} + +// TODO ???: make tus-store complaint: https://github.com/blockai/abstract-tus-store +class Store { + + constructor(targetDir) { + this.dir = path.normalize(targetDir); + } + + + getFilename(fid) { + let p = path.resolve(this.dir, fid.replace('++', '/')); + if(!p.startsWith(this.dir)) { + throw new Error('file name not in jail path. aborting'); + } + return p; + } + + + async create(fid, opts = {}) { + debug(`New File ${this.getFilename(fid)}`); + await fsp.ensureDir(path.dirname(this.getFilename(fid))); + await fsp.writeJson(this.getFilename(fid) + '.json', Object.assign(opts, { + isPartial: true + })); + return {uploadId: fid}; + } + + + async info(fid) { + try { + const info = await fsp.readJson(this.getFilename(fid) + '.json'); + const stat = await fsp.stat(this.getFilename(fid)); + info.size = stat.size; + info.offset = stat.size; + debug(`Fetched Fileinfo ${this.getFilename(fid)}`); + return info; + } catch(e) { + if(e.code === 'ENOENT') { + throw httpErrors.NotFound(); + } + throw e; + } + } + + + async append(fid, readStream, offset) { + debug(`Append Data to ${this.getFilename(fid)}`); + const uploadSize = new StreamLen(); + const ws = fsp.createWriteStream(this.getFilename(fid), {flags: 'a', start: offset}); + + const ret = new Promise((resolve, reject) => { + ws.on('finish', async() => { + const info = await this.info(fid); + if(info.size >= info.uploadLength) delete info.isPartial; + await fsp.writeJson(this.getFilename(fid) + '.json', info); + debug(`Finished appending Data to ${this.getFilename(fid)}`); + return resolve({ offset: info.offset, upload: info }); + }); + ws.on('error', reject); + // End writeStream on connection abort and wait for drain + readStream.on('close', () => { + ws.on('drain', () => { + ws.end(); + }); + }); + }); + + readStream.pipe(uploadSize).pipe(ws); + return ret; + } + + + createReadStream(fid, start, end, cb) { + debug(`Create ReadStream for ${this.getFilename(fid)}`); + this.info(fid).then(info => { + let contentLength = info.size; + if(start > 0) { + if(!end) end = info.size - 1; + contentLength = end - start + 1 + } + cb({ contentLength, metadata: info.metadata, info }); + }); + return fsp.createReadStream(this.getFilename(fid), {start, end}); + } + + + async del(fid) { + debug(`Delete ${this.getFilename(fid)}`); + await fsp.unlink(this.getFilename(fid) + '.json'); + await fsp.unlink(this.getFilename(fid)); + } +} + +module.exports = Store; diff --git a/package.json b/package.json new file mode 100644 index 0000000..5b74e5b --- /dev/null +++ b/package.json @@ -0,0 +1,35 @@ +{ + "name": "psitransfer", + "version": "1.0.0", + "description": "Open self-hoste file-sharing solution", + "keywords": ["share","upload","transfer","files","wetransfer"], + "repository": "psi-4ward/psitransfer", + "bugs": "https://github.com/psi-4ward/psitransfer/issues", + "main": "app.js", + "dependencies": { + "compression": "^1.6.2", + "crypto-js": "^3.1.9-1", + "debug": "^2.6.0", + "express": "^4.14.0", + "fs-promise": "^2.0.2", + "http-errors": "^1.5.1", + "morgan": "^1.7.0", + "tus-metadata": "^1.0.2", + "tusboy": "^1.1.1", + "uuid": "^3.0.1" + }, + "devDependencies": { + }, + "scripts": { + "start": "NODE_ENV=production node app.js", + "dev": "NODE_ENV=dev DEBUG=psitransfer:* nodemon -i app -i dist -i data app.js", + "debug": "node --inspect app.js" + }, + "engines": { + "node": ">= 7.4.0", + "npm": ">= 3" + }, + "author": "Christoph Wiechert ", + "contributors": [], + "license": "BSD-2-Clause" +} diff --git a/public/app/.gitignore b/public/app/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/public/app/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/public/assets/favicon.ico b/public/assets/favicon.ico new file mode 100644 index 0000000..591d9ca Binary files /dev/null and b/public/assets/favicon.ico differ diff --git a/public/html/download.html b/public/html/download.html new file mode 100644 index 0000000..51fe7e0 --- /dev/null +++ b/public/html/download.html @@ -0,0 +1,45 @@ + + + + + PsiTransfer + + + + + + + +
+

+ + PsiTransfer +

+
+
+
+ + + + + diff --git a/public/html/error.html b/public/html/error.html new file mode 100644 index 0000000..07c9244 --- /dev/null +++ b/public/html/error.html @@ -0,0 +1,51 @@ + + + + + PsiTransfer + + + + + + + +
+

+ + PsiTransfer +

+
+

+ + + %%ERROR%% + +

+

+ new upload +

+
+ + + diff --git a/public/html/upload.html b/public/html/upload.html new file mode 100644 index 0000000..d39d0bd --- /dev/null +++ b/public/html/upload.html @@ -0,0 +1,69 @@ + + + + + PsiTransfer + + + + + + + +
+
+

+ + PsiTransfer +

+
+
+
+ + + + + + diff --git a/tmp/scripts/traffic-limit.sh b/tmp/scripts/traffic-limit.sh new file mode 100755 index 0000000..bad00b9 --- /dev/null +++ b/tmp/scripts/traffic-limit.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# http://www.codeoriented.com/how-to-limit-network-bandwidth-for-a-specific-tcp-port-on-ubuntu-linux/ + +IF=$2 +BAND=$1 +PORT=$3 + +if [ -z "$3" ] ; then + echo $0 10kbps wlan0 8080 + exit 1 +fi + +tc qdisc del root dev $IF +tc qdisc add dev $IF root handle 1:0 htb default 10 +tc class add dev $IF parent 1:0 classid 1:10 htb rate $BAND prio 0 +tc filter add dev $IF parent 1:0 prio 0 protocol ip u32 match ip protocol 4 0xff match ip dport $PORT 0xffff flowid 1:10 +tc qdisc show dev $IF diff --git a/tmp/test/db.spec.js b/tmp/test/db.spec.js new file mode 100644 index 0000000..7a3d169 --- /dev/null +++ b/tmp/test/db.spec.js @@ -0,0 +1,116 @@ +"use strict"; +const fsp = require("fs-promise"); +const path = require("path"); +const Db = require("./db"); +const Store = require("./store"); +const uploadDir = path.resolve(__dirname, "data"); + +describe("psitransfer db", () => { + let db; + let store; + beforeEach(() => { + // TODO: atomic name + store = new Store(uploadDir); + db = new Db( + uploadDir, + store + ); + }); + + afterEach((next) => { + fsp.remove(uploadDir, next); + }); + + it("should properly construct", () => { + expect(db.initialized).toBeFalsy(); + expect(db.db).toEqual({}); + expect(db.expireTimers).toEqual({}); + expect(db.store).toBe(store); + expect(db.uploadDir).toEqual(uploadDir); + }); + + it("should call sync on init", () => { + spyOn(db, "sync"); + db.initialized = false; + db.init(); + expect(db.sync).toHaveBeenCalled(); + }); + + it("shouldn't call sync on initialize if already bootstrapped ", () => { + spyOn(db, "sync"); + db.initialized = true; + db.init(); + expect(db.sync).not.toHaveBeenCalled(); + }); + + describe("testing CRUD", () => { + let sid; + let uuid; + let metaData; + + beforeEach(() => { + sid = "221813e1688d"; + uuid = "e40bc20e-5be3-4906-903c-895f05e49efe"; + metaData = { + uploadLength: 0, + metadata: { + sid, + retention: "259200", + password: "", + name: "test.txt", + key: uuid, + createdAt: "" + Date.now() + }, + size: 0, + offset: 0 + }; + + fsp.ensureDirSync(uploadDir); + fsp.ensureDirSync(path.resolve(uploadDir, sid)); + fsp.writeFileSync(path.resolve(uploadDir, sid, uuid), ""); + fsp.writeFileSync(path.resolve(uploadDir, sid, uuid + ".json"), JSON.stringify(metaData)); + }); + + it("should sync upload dir", () => { + spyOn(db, "import"); + db.sync(); + expect(db.import).toHaveBeenCalledWith(sid); + }); + + it("should import existing files", async() => { + spyOn(db.store, "info").and.returnValue(Promise.resolve(metaData)); + spyOn(db, "add"); + db.initialized = true; + await db.import(sid); + expect(db.add).toHaveBeenCalledWith(sid, uuid, metaData); + }); + + it("should remove files", async() => { + db.initialized = true; + db.db[sid] = [ metaData ]; + spyOn(store, "del").and.returnValue(Promise.resolve()); + spyOn(fsp, "rmdir").and.returnValue(Promise.resolve()); + + await db.remove(sid, uuid); + + expect(store.del).toHaveBeenCalledWith(sid + '++' + uuid); + expect(db.db[sid]).not.toBeDefined(); + expect(fsp.rmdir).toHaveBeenCalledWith(path.resolve(uploadDir, sid)); + }); + + it("should add new files to sid", () => { + db.initialized = true; + spyOn(db, "registerRemove"); + db.add(sid, uuid, metaData); + expect(db.registerRemove).toHaveBeenCalled(); + expect(db.db[sid]).toEqual([metaData]); + }); + + it("should update already existing files", () => { + db.initialized = true; + db.add(sid, uuid, Object.assign(metaData, { bacon: "yammie"})); + db.add(sid, uuid, metaData); + expect(db.db[sid]).toEqual([metaData]); + }); + }); +}); diff --git a/tmp/test/jasmine.json b/tmp/test/jasmine.json new file mode 100644 index 0000000..1f94680 --- /dev/null +++ b/tmp/test/jasmine.json @@ -0,0 +1,6 @@ +{ + "spec_dir": "lib", + "spec_files": [ + "**/*.[sS]pec.js" + ] +} diff --git a/tmp/test/pkgs b/tmp/test/pkgs new file mode 100644 index 0000000..c5bd69f --- /dev/null +++ b/tmp/test/pkgs @@ -0,0 +1,5 @@ + "jasmine": "^2.5.2", + "nodemon": "^1.11.0", + "request": "^2.79.0" + + "test": "node spec-runner.js" diff --git a/tmp/test/spec-runner.js b/tmp/test/spec-runner.js new file mode 100644 index 0000000..ef46c57 --- /dev/null +++ b/tmp/test/spec-runner.js @@ -0,0 +1,6 @@ +'use strict'; +const Jasmine = require('jasmine'); + +const jasmine = new Jasmine(); +jasmine.loadConfigFile('jasmine.json'); +jasmine.execute(); diff --git a/tmp/test/store.spec.js b/tmp/test/store.spec.js new file mode 100644 index 0000000..5f152c3 --- /dev/null +++ b/tmp/test/store.spec.js @@ -0,0 +1,148 @@ +"use strict"; +const fsp = require("fs-promise"); +const path = require("path"); +const Store = require("./store"); +const httpErrors = require('http-errors'); + +const uploadDir = path.resolve(__dirname, "data"); + +describe("psitransfer store", () => { + + let store; + let sid; + let uuid; + let fid; + let metaData; + let info; + let stat; + + beforeEach(() => { + store = new Store(uploadDir); + + sid = "221813e1688d"; + uuid = "e40bc20e-5be3-4906-903c-895f05e49efe"; + + fsp.ensureDirSync(path.resolve(uploadDir, sid)); + + fid = `${sid}++${uuid}`; + metaData = { + uploadLength: 0, + metadata: { + sid, + retention: "259200", + password: "", + name: "test.txt", + key: uuid, + createdAt: "" + Date.now() + }, + size: 0, + offset: 0 + }; + + info = { + offset: 100, + uploadLength: 100, + metadata: { + sid: 'fea60a1beba6', + retention: '259200', + password: '', + name: 'bacon.ham', + key: '1215182b-ca57-4212-9e87-c7028190ff69', + createdAt: '1483890816120' + }, + isPartial: true, + }; + + stat = { + size: 287 + }; + }); + + afterEach((next) => { + fsp.remove(uploadDir, next); + }); + + it("should properly construct", () => { + expect(store.dir).toEqual(uploadDir); + }); + + it("should create new files", async() => { + + let fileName = store.getFilename(fid); + + spyOn(store, "getFilename").and.callThrough(); + spyOn(fsp, "ensureDir").and.returnValue(Promise.resolve()); + spyOn(fsp, "writeJson").and.returnValue(Promise.resolve()); + + expect(await store.create(fid, metaData)).toEqual({ + uploadId: fid + }); + + expect(store.getFilename).toHaveBeenCalled(); + expect(fsp.ensureDir).toHaveBeenCalled(); + expect(fsp.writeJson).toHaveBeenCalled(); + }); + + it("should evaluate file info", async() => { + + spyOn(fsp, "readJson").and.returnValue(Promise.resolve(info)); + spyOn(fsp, "stat").and.returnValue(Promise.resolve(stat)); + expect(await store.info(fid)).toEqual(Object.assign({}, info, stat)); + }); + + it("should throw an http error on file info if file doesn't exist", (next) => { + store.info(fid) + // TODO use always when available + // .always(() => next()); + .then(() => { + // if this test fails you accidently created that file and didn't cleanup + expect(true).toBeFalsy(); + next(); + }) + .catch((e) => { + expect(e).toEqual(jasmine.any(httpErrors.NotFound().constructor)); + next(); + }); + }); + + it("should append file content", async() => { + let readStream = fsp.createReadStream('/dev/urandom', {start: 0, end: 99}); + + spyOn(store, "info").and.returnValue(Promise.resolve(info)); + spyOn(fsp, "writeJson").and.returnValue(Promise.resolve()); + spyOn(fsp, "createWriteStream").and.callThrough(); + + let retVal = await store.append(fid, readStream, 0); + expect(retVal).toEqual({offset: 100, upload: info}); + + expect(store.info).toHaveBeenCalledWith(fid); + expect(fsp.writeJson).toHaveBeenCalled(); + }); + + it("should create a read stream", (next) => { + let cb = jasmine.createSpy('cb'); + spyOn(store, "info").and.returnValue(Promise.resolve(info)); + spyOn(store, "getFilename"); + spyOn(fsp, "createReadStream"); + + store.createReadStream(fid, 0, 100, cb); + + expect(store.getFilename).toHaveBeenCalledTimes(2); + expect(store.info).toHaveBeenCalled(); + expect(fsp.createReadStream).toHaveBeenCalled(); + setTimeout(() => { + expect(cb).toHaveBeenCalled(); + next(); + }, 0); + }); + + it("should delete files", async() => { + spyOn(fsp, "unlink"); + spyOn(store, "getFilename"); + + await store.del(fid); + + expect(fsp.unlink).toHaveBeenCalledTimes(2); + expect(store.getFilename).toHaveBeenCalledTimes(3); + }); +});