113 lines
4.3 KiB
JavaScript
113 lines
4.3 KiB
JavaScript
"use strict";
|
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
};
|
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
|
|
const crypto_1 = require("crypto");
|
|
const process_1 = __importDefault(require("process"));
|
|
const fs_1 = __importDefault(require("fs"));
|
|
const promises_1 = __importDefault(require("fs/promises"));
|
|
;
|
|
const GITEA_AUTH = process_1.default.env.GITEA_AUTH;
|
|
const GITEA_URL = 'https://git.tornado.ws';
|
|
const GITEA_ORG = 'tornado-packages';
|
|
const CONCURRENCY = 5;
|
|
if (!fs_1.default.existsSync('./data')) {
|
|
fs_1.default.mkdirSync('./data', { recursive: true });
|
|
}
|
|
const chunk = (arr, size) => [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
|
|
exports.chunk = chunk;
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
exports.sleep = sleep;
|
|
function bytesToBase64(bytes) {
|
|
let binary = '';
|
|
const len = bytes.byteLength;
|
|
for (let i = 0; i < len; ++i) {
|
|
binary += String.fromCharCode(bytes[i]);
|
|
}
|
|
return btoa(binary);
|
|
}
|
|
exports.bytesToBase64 = bytesToBase64;
|
|
async function download() {
|
|
if (!GITEA_AUTH) {
|
|
throw new Error(`Auth token not set`);
|
|
}
|
|
const packagesResults = [];
|
|
let page = 1;
|
|
let lastResult = 1;
|
|
while (lastResult !== 0) {
|
|
const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
|
|
headers: {
|
|
Authorization: `Bearer ${GITEA_AUTH}`
|
|
}
|
|
})).json();
|
|
page += 1;
|
|
lastResult = packageLists.length;
|
|
packagesResults.push(...packageLists);
|
|
}
|
|
console.log(`Got list of ${packagesResults.length} packages`);
|
|
const allPackages = packagesResults
|
|
.filter(({ type }) => type === 'npm')
|
|
.map(async ({ name, version }) => {
|
|
const encodedName = encodeURIComponent(name);
|
|
const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
|
|
headers: {
|
|
Authorization: `Bearer ${GITEA_AUTH}`
|
|
}
|
|
})).json();
|
|
const file = files.find(({ name }) => name.includes(version));
|
|
return {
|
|
name,
|
|
encodedName,
|
|
version,
|
|
file: file.name,
|
|
sha512: file.sha512
|
|
};
|
|
});
|
|
const packages = [];
|
|
for (const pkg of (0, exports.chunk)(allPackages, CONCURRENCY)) {
|
|
packages.push(...await Promise.all(pkg));
|
|
await sleep(500);
|
|
}
|
|
const fetchPackages = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
|
|
const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
|
|
const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
|
|
const digest = new Uint8Array(await crypto_1.webcrypto.subtle.digest('SHA-512', fetched));
|
|
const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
|
|
if (sha512 !== sha512Res) {
|
|
const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
|
|
throw new Error(errMsg);
|
|
}
|
|
if (fs_1.default.existsSync(`./data/${file}`)) {
|
|
promises_1.default.rm(`./data/${file}`, { force: true });
|
|
}
|
|
await promises_1.default.writeFile(`./data/${file}`, fetched);
|
|
return {
|
|
name,
|
|
encodedName,
|
|
version,
|
|
file,
|
|
fileUrl,
|
|
sha512,
|
|
integrity: `sha512-${bytesToBase64(digest)}`
|
|
};
|
|
});
|
|
const downloadedPackages = [];
|
|
for (const pkg of (0, exports.chunk)(fetchPackages, CONCURRENCY)) {
|
|
downloadedPackages.push(...await Promise.all(pkg));
|
|
console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
|
|
await sleep(500);
|
|
}
|
|
console.log(downloadedPackages);
|
|
fs_1.default.writeFileSync('./data/packages.json', JSON.stringify({
|
|
gitea: `${GITEA_URL}/${GITEA_ORG}`,
|
|
timestamp: parseInt(`${Date.now() / 1000}`),
|
|
packages: downloadedPackages
|
|
}, null, 2));
|
|
}
|
|
exports.download = download;
|
|
download();
|