Add gitattributes

.gitattributes

@@ -0,0 +1 @@
+lib/* linguist-vendored

index.js

@@ -1,112 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
-const crypto_1 = require("crypto");
-const process_1 = __importDefault(require("process"));
-const fs_1 = __importDefault(require("fs"));
-const promises_1 = __importDefault(require("fs/promises"));
-;
-const GITEA_AUTH = process_1.default.env.GITEA_AUTH;
-const GITEA_URL = 'https://git.tornado.ws';
-const GITEA_ORG = 'tornado-packages';
-const CONCURRENCY = 5;
-if (!fs_1.default.existsSync('./data')) {
-    fs_1.default.mkdirSync('./data', { recursive: true });
-}
-const chunk = (arr, size) => [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
-exports.chunk = chunk;
-function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}
-exports.sleep = sleep;
-function bytesToBase64(bytes) {
-    let binary = '';
-    const len = bytes.byteLength;
-    for (let i = 0; i < len; ++i) {
-        binary += String.fromCharCode(bytes[i]);
-    }
-    return btoa(binary);
-}
-exports.bytesToBase64 = bytesToBase64;
-async function download() {
-    if (!GITEA_AUTH) {
-        throw new Error(`Auth token not set`);
-    }
-    const packagesResults = [];
-    let page = 1;
-    let lastResult = 1;
-    while (lastResult !== 0) {
-        const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
-            headers: {
-                Authorization: `Bearer ${GITEA_AUTH}`
-            }
-        })).json();
-        page += 1;
-        lastResult = packageLists.length;
-        packagesResults.push(...packageLists);
-    }
-    console.log(`Got list of ${packagesResults.length} packages`);
-    const allPackages = packagesResults
-        .filter(({ type }) => type === 'npm')
-        .map(async ({ name, version }) => {
-        const encodedName = encodeURIComponent(name);
-        const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
-            headers: {
-                Authorization: `Bearer ${GITEA_AUTH}`
-            }
-        })).json();
-        const file = files.find(({ name }) => name.includes(version));
-        return {
-            name,
-            encodedName,
-            version,
-            file: file.name,
-            sha512: file.sha512
-        };
-    });
-    const packages = [];
-    for (const pkg of (0, exports.chunk)(allPackages, CONCURRENCY)) {
-        packages.push(...await Promise.all(pkg));
-        await sleep(500);
-    }
-    const fetchPackages = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
-        const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
-        const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
-        const digest = new Uint8Array(await crypto_1.webcrypto.subtle.digest('SHA-512', fetched));
-        const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
-        if (sha512 !== sha512Res) {
-            const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
-            throw new Error(errMsg);
-        }
-        if (fs_1.default.existsSync(`./data/${file}`)) {
-            promises_1.default.rm(`./data/${file}`, { force: true });
-        }
-        await promises_1.default.writeFile(`./data/${file}`, fetched);
-        return {
-            name,
-            encodedName,
-            version,
-            file,
-            fileUrl,
-            sha512,
-            integrity: `sha512-${bytesToBase64(digest)}`
-        };
-    });
-    const downloadedPackages = [];
-    for (const pkg of (0, exports.chunk)(fetchPackages, CONCURRENCY)) {
-        downloadedPackages.push(...await Promise.all(pkg));
-        console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
-        await sleep(500);
-    }
-    console.log(downloadedPackages);
-    fs_1.default.writeFileSync('./data/packages.json', JSON.stringify({
-        gitea: `${GITEA_URL}/${GITEA_ORG}`,
-        timestamp: parseInt(`${Date.now() / 1000}`),
-        packages: downloadedPackages
-    }, null, 2));
-}
-exports.download = download;
-download();

lib/index.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
+require("dotenv/config");
 const crypto_1 = require("crypto");
 const process_1 = __importDefault(require("process"));
 const fs_1 = __importDefault(require("fs"));

package.json

@@ -1,7 +1,7 @@
 {
   "name": "download-packages",
   "version": "1.0.0",
-  "main": "index.js",
+  "main": "lib/index.js",
   "license": "MIT",
   "scripts": {
     "types": "tsc",