download-packages/index.ts

import { webcrypto as crypto } from 'crypto';
import process from 'process';
import fs from 'fs';
import fsPromises from 'fs/promises';

const GITEA_AUTH = process.env.GITEA_AUTH;

const GITEA_URL = 'https://git.tornado.ws';

const GITEA_ORG = 'tornado-packages';

const CONCURRENCY = 5;

if (!fs.existsSync('./data')) {
  fs.mkdirSync('./data', { recursive: true });
} else {
  fs.rmSync('./data', { recursive: true, force: true });
  fs.mkdirSync('./data', { recursive: true });
}

export const chunk = <T>(arr: T[], size: number): T[][] =>
  [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));

export function sleep(ms: number) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

export function bytesToBase64(bytes: Uint8Array) {
  let binary = '';
  const len = bytes.byteLength;
  for (let i = 0; i < len; ++i) {
    binary += String.fromCharCode(bytes[i]);
  }
  return btoa(binary);
}

export interface pkgApi {
  type: string;
  name: string;
  version: string;
}

export interface filesApi {
  name: string;
  sha512: string;
}

export interface pkg {
  name: string;
  encodedName: string;
  version: string;
  file: string;
  sha512: string;
}

export type pkgResolved = pkg & {
  fileUrl: string;
  integrity: string;
}

export async function download() {
  if (!GITEA_AUTH) {
    throw new Error('Auth token not set');
  }

  const packagesResults = [];

  let page = 1;
  let lastResult = 1;

  while (lastResult !== 0) {
    const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
      headers: {
        Authorization: `Bearer ${GITEA_AUTH}`
      }
    })).json() as pkgApi[];

    page += 1;
    lastResult = packageLists.length;
    packagesResults.push(...packageLists);
  }

  console.log(`Got list of ${packagesResults.length} packages`);

  const allPackages: Promise<pkg>[] = packagesResults
    .filter(({ type }) => type === 'npm')
    .map(async ({ name, version }) => {
      const encodedName = encodeURIComponent(name);

      const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
        headers: {
          Authorization: `Bearer ${GITEA_AUTH}`
        }
      })).json() as filesApi[];

      const file = files.find(({ name }: { name: string }) => name.includes(version)) as filesApi;

      return {
        name,
        encodedName,
        version,
        file: file.name,
        sha512: file.sha512
      };
    });

  const packages: pkg[] = [];

  for (const pkg of chunk(allPackages, CONCURRENCY)) {
    packages.push(...await Promise.all(pkg));

    await sleep(500);
  }

  const fetchPackages: Promise<pkgResolved>[] = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
    const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;

    const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());

    const digest = new Uint8Array(await crypto.subtle.digest('SHA-512', fetched));

    const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');

    if (sha512 !== sha512Res) {
      const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
      throw new Error(errMsg);
    }

    await fsPromises.writeFile(`./data/${file}`, fetched);
    
    return {
      name,
      encodedName,
      version,
      file,
      fileUrl,
      sha512,
      integrity: `sha512-${bytesToBase64(digest)}`
    };
  });

  const downloadedPackages: pkgResolved[] = [];

  for (const pkg of chunk(fetchPackages, CONCURRENCY)) {
    downloadedPackages.push(...await Promise.all(pkg));

    console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);

    await sleep(500);
  }

  console.log(downloadedPackages);

  fs.writeFileSync('./data/packages.json', JSON.stringify({
    gitea: `${GITEA_URL}/${GITEA_ORG}`,
    timestamp: parseInt(`${Date.now() / 1000}`),
    packages: downloadedPackages
  }, null, 2));
}
download();