ethers.js/admin/cmds/upload-docs.js

"use strict";

const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const AWS = require('aws-sdk');

const config = require("../config");


const Bucket = "docs.ethers.io";


function _getKeys(s3, result, nextToken, callback) {
    const params = {
        Bucket: Bucket,
        MaxKeys: 1000,
        ContinuationToken: nextToken,
    };
    s3.listObjectsV2(params, function(error, data) {
        if (error) {
            console.log(error);
            callback(error);
            return;
        }
        data.Contents.forEach(function(item) {
            result[item.Key] = item.ETag.replace(/"/g,'');
        });
        callback(null, data.IsTruncated ? data.NextContinuationToken: null);
    });
}

function getKeys(s3) {
    const result = {};
    return new Promise(function(resolve, reject) {
        function handleBlock(error, nextToken) {
            if (error) {
                reject(error);
            } else if (nextToken) {
                nextBlock(nextToken);
            } else {
                resolve(result);
            }
        }
        function nextBlock(nextToken) {
            _getKeys(s3, result, nextToken, handleBlock);
        }
        nextBlock(undefined);
    });
}

function getMime(filename) {
    const comps = filename.split('.');
    const ext = comps[comps.length - 1];
    switch (ext.toLowerCase()) {
        case 'css':      return 'text/css';
        case 'doctree':  return 'application/x-doctree';
        case 'eot':      return 'application/vnd.ms-fontobject';
        case 'gif':      return 'image/gif';
        case 'html':     return 'text/html';
        case 'js':       return 'application/javascript';
        case 'jpg':      return 'image/jpeg';
        case 'jpeg':     return 'image/jpeg';
        case 'json':     return 'application/json';
        case 'md':       return 'text/markdown';
        case 'pickle':   return 'application/x-pickle';
        case 'png':      return 'image/png';
        case 'svg':      return 'image/svg+xml';
        case 'ttf':      return 'application/x-font-ttf';
        case 'txt':      return 'text/plain';
        case 'woff':     return 'application/font-woff';
    }
    console.log('NO MIME', filename);
    return undefined;
}

function putObject(s3, name, content) {
    return new Promise(function(resolve, reject) {
        s3.putObject({
            ACL: 'public-read',
            Body: content,
            Bucket: Bucket,
            ContentType: getMime(name),
            Key: name
        }, function(error, data) {
            if (error) {
                reject(error);
            } else {
                console.log("  Done.")
                resolve({
                    name: name,
                    hash: data.ETag.replace(/"/g, '')
                });
            }
        });
    });
}

function hash(filename) {
    const hasher = crypto.createHash('md5');
    hasher.update(fs.readFileSync(filename));
    return hasher.digest().toString('hex');
}

function _getFiles(result, root) {
    fs.readdirSync(root).forEach(function(filename) {

        // We don't need to upload junk
        if (filename === '.DS_Store') { return; }

        const fullFilename = path.join(root, filename)
        const stat = fs.statSync(fullFilename);
        if (stat.isDirectory()) {
            _getFiles(result, fullFilename);
        } else {
            result[fullFilename] = hash(fullFilename);
        }
    });
}

function getFiles(basedir) {
    // Make sure we have a trailing slash
    if (!basedir.match(/\/$/)) { basedir += "/"; }

    // Fetch all the file hashes
    const hashes = { };
    _getFiles(hashes, basedir);

    return Object.keys(hashes).reduce((accum, key) => {
        accum[key.substring(basedir.length)] = hashes[key];
        return accum;
    }, { });
}

(async function() {
    const awsAccessId = await config.get("aws-upload-docs-accesskey");
    const awsSecretKey = await config.get("aws-upload-docs-secretkey");

    const s3 = new AWS.S3({
        apiVersion: '2006-03-01',
        accessKeyId: awsAccessId,
        secretAccessKey: awsSecretKey
    });

    const added = [], removed = [], changed = [], upload = [];

    const basedir = path.resolve(__dirname, "../../docs");

    const local = await getFiles(basedir);
    const remote = await getKeys(s3);

    Object.keys(local).forEach((filename) => {
        if (!remote[filename]) {
            added.push(filename);
            upload.push(filename);
        } else if (remote[filename] != local[filename]) {
            changed.push(filename);
            upload.push(filename);
        }
    });

    Object.keys(remote).forEach((filename) => {
        if (!local[filename]) {
            removed.push(filename);
        } else if (!local[filename] && remote[filename] != local[filename]) {
            changed.push(filename);
            upload.push(filename);
        }
    });

    console.log('Added:    ', added.length);
    console.log('Removed:  ', removed.length);
    console.log('Changed:  ', changed.length);

    for (let i = 0; i < upload.length; i++) {
        const filename = upload[i];
        const content = fs.readFileSync(path.resolve(basedir, filename));
        console.log(`Uploading: ${ filename } (${ content.length } bytes)`);
        await putObject(s3, filename, content);
    }
})();