init

lib/db.js

@@ -0,0 +1,118 @@
+'use strict';
+const fsp = require('fs-promise');
+const path = require('path');
+const debug = require('debug')('psitransfer:db');
+
+module.exports = class DB {
+
+  constructor(uploadDir, store) {
+    this.initialized = false;
+    this.db = {};
+    this.expireTimers = {};
+
+    this.uploadDir = uploadDir;
+    this.store = store;
+
+    // delete expired files
+    const gc = () => {
+      let sid,f,expires;
+      for (sid of Object.keys(this.db)) {
+        for (f of this.db[sid]) {
+          // no removal of one-time downloads
+          if(!Number.isInteger(f.metadata.retention)) return;
+
+          expires = (+f.metadata.createdAt) + (+f.metadata.retention * 1000) - Date.now();
+          if(expires <= 0) {
+            debug(`Expired ${sid}++${f.key}`);
+            this.remove(sid, f.key).catch(e => console.error(e));
+          }
+        }
+      }
+    };
+    setInterval(gc, 60*1000);
+
+  }
+
+
+  init() {
+    if(this.initialized) return;
+    this.initialized = true;
+
+    try {
+      this._sync();
+    } catch(e) {
+      this.initialized = false;
+      e.message = `db initialization failed with error ${e.message}`;
+      throw e;
+    }
+  }
+
+
+  /**
+   * @private
+   */
+  _sync() {
+    fsp.ensureDirSync(this.uploadDir);
+
+    fsp.readdirSync(this.uploadDir).forEach((sid) => {
+      this._import(sid);
+    });
+  }
+
+
+  /**
+   * @private
+   */
+  _import(sid) {
+    const p = path.resolve(this.uploadDir, sid);
+    const stat = fsp.statSync(p);
+    if(!stat.isDirectory()) return;
+
+    fsp.readdirSync(p).forEach(async (key) => {
+      if(path.extname(key) !== '') {
+        return Promise.resolve();
+      }
+      try {
+        let info = await this.store.info(`${sid}++${key}`);
+        this.add(sid, key, info);
+      } catch(e) {
+        console.error(e);
+      }
+    });
+  }
+
+
+  add(sid, key, data) {
+    if(!this.initialized) throw new Error('DB not initialized_');
+    if(!this.db[sid]) this.db[sid] = [];
+    data.key = key;
+
+    const old = this.db[sid].findIndex(i => i.key === key);
+    if(old !== -1) {
+      this.db[sid].splice(old, 1, data);
+      debug(`Updated ${sid}++${key}`);
+    } else {
+      this.db[sid].push(data);
+      debug(`Added ${sid}++${key}`);
+    }
+  }
+
+
+  async remove(sid, key) {
+    if(!this.initialized) throw new Error('DB not initialized');
+    debug(`Remove ${sid}++${key}`);
+    await this.store.del(sid + '++' + key);
+    const i = this.get(sid).find(item => item.key === key);
+    this.get(sid).splice(i, 1);
+    if(this.get(sid).length === 0) {
+      delete this.db[sid];
+      await fsp.rmdir(path.resolve(this.uploadDir, sid));
+    }
+  }
+
+
+  get(sid) {
+    return this.db[sid];
+  }
+
+};

lib/endpoints.js

@@ -0,0 +1,145 @@
+'use strict';
+const config = require('../config');
+const tusboy = require('tusboy').default;
+const express = require('express');
+const morgan = require('morgan');
+const compression = require('compression');
+const Store = require('./store');
+const uuid = require('uuid/v4');
+const path = require('path');
+const fs = require("fs");
+const tusMeta = require('tus-metadata');
+const assert = require('assert');
+const AES = require("crypto-js/aes");
+const debug = require('debug')('psitransfer:main');
+
+const errorPage = fs.readFileSync(path.join(__dirname, '../public/html/error.html')).toString();
+const store = new Store(config.uploadDir);
+const Db = require('./db');
+const db = new Db(config.uploadDir, store);
+db.init();
+const app = express();
+
+app.disable('x-powered-by');
+app.use(compression());
+
+if(config.accessLog) {
+  app.use(morgan(config.accessLog));
+}
+
+// Static files
+app.use('/app', express.static(path.join(__dirname, '../public/app')));
+app.use('/assets', express.static(path.join(__dirname, '../public/assets')));
+
+
+// Upload App
+app.get('/', (req, res) => {
+  res.sendFile(path.join(__dirname, '../public/html/upload.html'));
+});
+
+// Config
+app.get('/config.json', (req, res) => {
+  res.json({
+    retentions: config.retentions,
+    defaultRetention: config.defaultRetention,
+    mailTemplate: config.mailTemplate
+  });
+});
+
+
+// List files / Download App
+app.get('/:sid', (req, res, next) => {
+  if(req.url.endsWith('.json')) {
+    const sid = req.params.sid.substr(0, req.params.sid.length-5);
+    if(!db.get(sid)) return res.status(404).end();
+
+    res.json(db.get(sid).map(data => {
+      const item = Object.assign(data, {url: `/files/${sid}++${data.key}`});
+      if(item.metadata.password) {
+        return AES.encrypt(JSON.stringify(data), item.metadata.password).toString();
+      } else {
+        return item;
+      }
+    }));
+  } else {
+    if(!db.get(req.params.sid)) return next();
+    res.sendFile(path.join(__dirname, '../public/html/download.html'));
+  }
+});
+
+
+// Download single file
+app.get('/files/:fid', async(req, res, next) => {
+  // let tusboy handle HEAD with Tus Header
+  if(req.method === 'HEAD' && req.get('Tus-Resumable')) return next();
+
+  debug(`Download ${req.params.fid}`);
+  try {
+    const info = await store.info(req.params.fid); // throws on 404
+    res.download(store.getFilename(req.params.fid), info.metadata.name);
+
+    // remove one-time files after download
+    if(info.metadata.retention === 'one-time') {
+      res.on('finish', async () => {
+        await db.remove(info.metadata.sid, info.metadata.key);
+      });
+    }
+  } catch(e) {
+    res.status(404).send(errorPage.replace('%%ERROR%%', e.message));
+  }
+});
+
+
+// Upload file
+app.use('/files',
+  function(req, res, next) {
+    if(req.method === 'GET') return res.status(405).end();
+
+    if(req.method === 'POST') {
+      // validate meta-data
+      // !! tusMeta.encode supports only strings !!
+      const meta = tusMeta.decode(req.get('Upload-Metadata'));
+
+      try {
+        assert(meta.name, 'tus meta prop missing: name');
+        assert(meta.sid, 'tus meta prop missing: sid');
+        assert(meta.retention, 'tus meta prop missing: retention');
+        assert(Object.keys(config.retentions).indexOf(meta.retention) >= 0,
+          `invalid tus meta prop retention. Value ${meta.retention} not in [${Object.keys(config.retentions).join(',')}]`);
+
+        meta.key = uuid();
+        meta.createdAt = Date.now().toString();
+
+        // store changed metadata for tusboy
+        req.headers['upload-metadata'] = tusMeta.encode(meta);
+        // for tusboy getKey()
+        req.FID = meta.sid + '++' + meta.key;
+
+        db.add(meta.sid, meta.key, {
+          "isPartial": true,
+          metadata: meta
+        });
+      }
+      catch(e) {
+        return res.status(400).end(e.message);
+      }
+    }
+
+    next();
+  },
+
+  // let tusboy handle the upload
+  tusboy(store, {
+    getKey: req => req.FID,
+    afterComplete: (req, upload, fid) => {
+      db.add(upload.metadata.sid, upload.metadata.key, upload);
+      debug(`Completed upload ${fid}, size=${upload.size} name=${upload.metadata.name}`);
+    },
+  })
+);
+
+app.use((req, res, next) => {
+  res.status(404).send(errorPage.replace('%%ERROR%%', 'Download bucket not found.'));
+});
+
+module.exports = app;

lib/store.js

@@ -0,0 +1,113 @@
+'use strict';
+const fsp = require('fs-promise');
+const path = require('path');
+const Transform = require('stream').Transform;
+const debug = require('debug')('psitransfer:store');
+const httpErrors = require('http-errors');
+
+class StreamLen extends Transform {
+  constructor(options) {
+    super(options);
+    this.bytes = 0;
+  }
+
+  _transform(chunk, encoding, cb) {
+    this.bytes += chunk.length;
+    this.push(chunk);
+    cb();
+  }
+}
+
+// TODO ???: make tus-store complaint: https://github.com/blockai/abstract-tus-store
+class Store {
+
+  constructor(targetDir) {
+    this.dir = path.normalize(targetDir);
+  }
+
+
+  getFilename(fid) {
+    let p = path.resolve(this.dir, fid.replace('++', '/'));
+    if(!p.startsWith(this.dir)) {
+      throw new Error('file name not in jail path. aborting');
+    }
+    return p;
+  }
+
+
+  async create(fid, opts = {}) {
+    debug(`New File ${this.getFilename(fid)}`);
+    await fsp.ensureDir(path.dirname(this.getFilename(fid)));
+    await fsp.writeJson(this.getFilename(fid) + '.json', Object.assign(opts, {
+      isPartial: true
+    }));
+    return {uploadId: fid};
+  }
+
+
+  async info(fid) {
+    try {
+      const info = await fsp.readJson(this.getFilename(fid) + '.json');
+      const stat = await fsp.stat(this.getFilename(fid));
+      info.size = stat.size;
+      info.offset = stat.size;
+      debug(`Fetched Fileinfo ${this.getFilename(fid)}`);
+      return info;
+    } catch(e) {
+      if(e.code === 'ENOENT') {
+        throw httpErrors.NotFound();
+      }
+      throw e;
+    }
+  }
+
+
+  async append(fid, readStream, offset) {
+    debug(`Append Data to ${this.getFilename(fid)}`);
+    const uploadSize = new StreamLen();
+    const ws = fsp.createWriteStream(this.getFilename(fid), {flags: 'a', start: offset});
+
+    const ret = new Promise((resolve, reject) => {
+      ws.on('finish', async() => {
+        const info = await this.info(fid);
+        if(info.size >= info.uploadLength) delete info.isPartial;
+        await fsp.writeJson(this.getFilename(fid) + '.json', info);
+        debug(`Finished appending Data to ${this.getFilename(fid)}`);
+        return resolve({ offset: info.offset, upload: info });
+      });
+      ws.on('error', reject);
+      // End writeStream on connection abort and wait for drain
+      readStream.on('close', () => {
+        ws.on('drain', () => {
+          ws.end();
+        });
+      });
+    });
+
+    readStream.pipe(uploadSize).pipe(ws);
+    return ret;
+  }
+
+
+  createReadStream(fid, start, end, cb) {
+    debug(`Create ReadStream for ${this.getFilename(fid)}`);
+    this.info(fid).then(info => {
+      let contentLength = info.size;
+      if(start > 0) {
+        if(!end) end = info.size - 1;
+        contentLength = end - start + 1
+      }
+      cb({ contentLength, metadata: info.metadata, info });
+    });
+    return fsp.createReadStream(this.getFilename(fid), {start, end});
+  }
+
+
+  async del(fid) {
+    debug(`Delete ${this.getFilename(fid)}`);
+    await fsp.unlink(this.getFilename(fid) + '.json');
+    await fsp.unlink(this.getFilename(fid));
+  }
+}
+
+module.exports = Store;