download-packages/lib/index.js

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
require("dotenv/config");
const crypto_1 = require("crypto");
const process_1 = __importDefault(require("process"));
const fs_1 = __importDefault(require("fs"));
const promises_1 = __importDefault(require("fs/promises"));
const GITEA_AUTH = process_1.default.env.GITEA_AUTH;
const GITEA_URL = 'https://git.tornado.ws';
const GITEA_ORG = 'tornado-packages';
const CONCURRENCY = 5;
if (!fs_1.default.existsSync('./data')) {
    fs_1.default.mkdirSync('./data', { recursive: true });
}
else {
    fs_1.default.rmSync('./data', { recursive: true, force: true });
    fs_1.default.mkdirSync('./data', { recursive: true });
}
const chunk = (arr, size) => [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
exports.chunk = chunk;
function sleep(ms) {
    return new Promise((resolve) => setTimeout(resolve, ms));
}
exports.sleep = sleep;
function bytesToBase64(bytes) {
    let binary = '';
    const len = bytes.byteLength;
    for (let i = 0; i < len; ++i) {
        binary += String.fromCharCode(bytes[i]);
    }
    return btoa(binary);
}
exports.bytesToBase64 = bytesToBase64;
async function download() {
    if (!GITEA_AUTH) {
        throw new Error('Auth token not set');
    }
    const packagesResults = [];
    let page = 1;
    let lastResult = 1;
    while (lastResult !== 0) {
        const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
            headers: {
                Authorization: `Bearer ${GITEA_AUTH}`
            }
        })).json();
        page += 1;
        lastResult = packageLists.length;
        packagesResults.push(...packageLists);
    }
    console.log(`Got list of ${packagesResults.length} packages`);
    const allPackages = packagesResults
        .filter(({ type }) => type === 'npm')
        .map(async ({ name, version }) => {
        const encodedName = encodeURIComponent(name);
        const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
            headers: {
                Authorization: `Bearer ${GITEA_AUTH}`
            }
        })).json();
        const file = files.find(({ name }) => name.includes(version));
        return {
            name,
            encodedName,
            version,
            file: file.name,
            sha512: file.sha512
        };
    });
    const packages = [];
    for (const pkg of (0, exports.chunk)(allPackages, CONCURRENCY)) {
        packages.push(...await Promise.all(pkg));
        await sleep(500);
    }
    const fetchPackages = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
        const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
        const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
        const digest = new Uint8Array(await crypto_1.webcrypto.subtle.digest('SHA-512', fetched));
        const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
        if (sha512 !== sha512Res) {
            const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
            throw new Error(errMsg);
        }
        await promises_1.default.writeFile(`./data/${file}`, fetched);
        return {
            name,
            encodedName,
            version,
            file,
            fileUrl,
            sha512,
            integrity: `sha512-${bytesToBase64(digest)}`
        };
    });
    const downloadedPackages = [];
    for (const pkg of (0, exports.chunk)(fetchPackages, CONCURRENCY)) {
        downloadedPackages.push(...await Promise.all(pkg));
        console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
        await sleep(500);
    }
    console.log(downloadedPackages);
    fs_1.default.writeFileSync('./data/packages.json', JSON.stringify({
        gitea: `${GITEA_URL}/${GITEA_ORG}`,
        timestamp: parseInt(`${Date.now() / 1000}`),
        packages: downloadedPackages
    }, null, 2));
}
exports.download = download;
download();