Initial commit

.eslintrc.js

@@ -0,0 +1,49 @@
+module.exports = {
+  "env": {
+    "es2021": true,
+    "node": true
+  },
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "overrides": [
+    {
+      "env": {
+        "node": true
+      },
+      "files": [
+        ".eslintrc.{js,cjs}"
+      ],
+      "parserOptions": {
+        "sourceType": "script"
+      }
+    }
+  ],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "rules": {
+    "indent": [
+      "error",
+      2
+    ],
+    "linebreak-style": [
+      "error",
+      "unix"
+    ],
+    "quotes": [
+      "error",
+      "single"
+    ],
+    "semi": [
+      "error",
+      "always"
+    ]
+  }
+}

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+data

README.md

@@ -0,0 +1,18 @@
+# Download NPM packages
+
+Download npm packages from gitea registry to create an additional backup of packages published on Gitea registry
+
+### Requirements
+
+- Node.js version supporting WebCrypto & Fetch API ( They are browser compatible and doesn't require additional deps )
+
+- Gitea token that has read permissions ( See https://docs.gitea.com/development/api-usage )
+
+### How to start
+
+```
+GITEA_AUTH=<bearer token here>
+yarn && yarn start
+```
+
+This will download packages under data directory

index.js

@@ -0,0 +1,112 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
+const crypto_1 = require("crypto");
+const process_1 = __importDefault(require("process"));
+const fs_1 = __importDefault(require("fs"));
+const promises_1 = __importDefault(require("fs/promises"));
+;
+const GITEA_AUTH = process_1.default.env.GITEA_AUTH;
+const GITEA_URL = 'https://git.tornado.ws';
+const GITEA_ORG = 'tornado-packages';
+const CONCURRENCY = 5;
+if (!fs_1.default.existsSync('./data')) {
+    fs_1.default.mkdirSync('./data', { recursive: true });
+}
+const chunk = (arr, size) => [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
+exports.chunk = chunk;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+exports.sleep = sleep;
+function bytesToBase64(bytes) {
+    let binary = '';
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; ++i) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+}
+exports.bytesToBase64 = bytesToBase64;
+async function download() {
+    if (!GITEA_AUTH) {
+        throw new Error(`Auth token not set`);
+    }
+    const packagesResults = [];
+    let page = 1;
+    let lastResult = 1;
+    while (lastResult !== 0) {
+        const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
+            headers: {
+                Authorization: `Bearer ${GITEA_AUTH}`
+            }
+        })).json();
+        page += 1;
+        lastResult = packageLists.length;
+        packagesResults.push(...packageLists);
+    }
+    console.log(`Got list of ${packagesResults.length} packages`);
+    const allPackages = packagesResults
+        .filter(({ type }) => type === 'npm')
+        .map(async ({ name, version }) => {
+        const encodedName = encodeURIComponent(name);
+        const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
+            headers: {
+                Authorization: `Bearer ${GITEA_AUTH}`
+            }
+        })).json();
+        const file = files.find(({ name }) => name.includes(version));
+        return {
+            name,
+            encodedName,
+            version,
+            file: file.name,
+            sha512: file.sha512
+        };
+    });
+    const packages = [];
+    for (const pkg of (0, exports.chunk)(allPackages, CONCURRENCY)) {
+        packages.push(...await Promise.all(pkg));
+        await sleep(500);
+    }
+    const fetchPackages = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
+        const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
+        const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
+        const digest = new Uint8Array(await crypto_1.webcrypto.subtle.digest('SHA-512', fetched));
+        const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
+        if (sha512 !== sha512Res) {
+            const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
+            throw new Error(errMsg);
+        }
+        if (fs_1.default.existsSync(`./data/${file}`)) {
+            promises_1.default.rm(`./data/${file}`, { force: true });
+        }
+        await promises_1.default.writeFile(`./data/${file}`, fetched);
+        return {
+            name,
+            encodedName,
+            version,
+            file,
+            fileUrl,
+            sha512,
+            integrity: `sha512-${bytesToBase64(digest)}`
+        };
+    });
+    const downloadedPackages = [];
+    for (const pkg of (0, exports.chunk)(fetchPackages, CONCURRENCY)) {
+        downloadedPackages.push(...await Promise.all(pkg));
+        console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
+        await sleep(500);
+    }
+    console.log(downloadedPackages);
+    fs_1.default.writeFileSync('./data/packages.json', JSON.stringify({
+        gitea: `${GITEA_URL}/${GITEA_ORG}`,
+        timestamp: parseInt(`${Date.now() / 1000}`),
+        packages: downloadedPackages
+    }, null, 2));
+}
+exports.download = download;
+download();

index.ts

@@ -0,0 +1,160 @@
+import { webcrypto as crypto } from 'crypto';
+import process from 'process';
+import fs from 'fs';
+import fsPromises from 'fs/promises';
+
+const GITEA_AUTH = process.env.GITEA_AUTH;
+
+const GITEA_URL = 'https://git.tornado.ws';
+
+const GITEA_ORG = 'tornado-packages';
+
+const CONCURRENCY = 5;
+
+if (!fs.existsSync('./data')) {
+  fs.mkdirSync('./data', { recursive: true });
+} else {
+  fs.rmSync('./data', { recursive: true, force: true });
+  fs.mkdirSync('./data', { recursive: true });
+}
+
+export const chunk = <T>(arr: T[], size: number): T[][] =>
+  [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
+
+export function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function bytesToBase64(bytes: Uint8Array) {
+  let binary = '';
+  const len = bytes.byteLength;
+  for (let i = 0; i < len; ++i) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+
+export interface pkgApi {
+  type: string;
+  name: string;
+  version: string;
+}
+
+export interface filesApi {
+  name: string;
+  sha512: string;
+}
+
+export interface pkg {
+  name: string;
+  encodedName: string;
+  version: string;
+  file: string;
+  sha512: string;
+}
+
+export type pkgResolved = pkg & {
+  fileUrl: string;
+  integrity: string;
+}
+
+export async function download() {
+  if (!GITEA_AUTH) {
+    throw new Error('Auth token not set');
+  }
+
+  const packagesResults = [];
+
+  let page = 1;
+  let lastResult = 1;
+
+  while (lastResult !== 0) {
+    const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
+      headers: {
+        Authorization: `Bearer ${GITEA_AUTH}`
+      }
+    })).json() as pkgApi[];
+
+    page += 1;
+    lastResult = packageLists.length;
+    packagesResults.push(...packageLists);
+  }
+
+  console.log(`Got list of ${packagesResults.length} packages`);
+
+  const allPackages: Promise<pkg>[] = packagesResults
+    .filter(({ type }) => type === 'npm')
+    .map(async ({ name, version }) => {
+      const encodedName = encodeURIComponent(name);
+
+      const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
+        headers: {
+          Authorization: `Bearer ${GITEA_AUTH}`
+        }
+      })).json() as filesApi[];
+
+      const file = files.find(({ name }: { name: string }) => name.includes(version)) as filesApi;
+
+      return {
+        name,
+        encodedName,
+        version,
+        file: file.name,
+        sha512: file.sha512
+      };
+    });
+
+  const packages: pkg[] = [];
+
+  for (const pkg of chunk(allPackages, CONCURRENCY)) {
+    packages.push(...await Promise.all(pkg));
+
+    await sleep(500);
+  }
+
+  const fetchPackages: Promise<pkgResolved>[] = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
+    const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
+
+    const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
+
+    const digest = new Uint8Array(await crypto.subtle.digest('SHA-512', fetched));
+
+    const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
+
+    if (sha512 !== sha512Res) {
+      const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
+      throw new Error(errMsg);
+    }
+
+    await fsPromises.writeFile(`./data/${file}`, fetched);
+    
+    return {
+      name,
+      encodedName,
+      version,
+      file,
+      fileUrl,
+      sha512,
+      integrity: `sha512-${bytesToBase64(digest)}`
+    };
+  });
+
+  const downloadedPackages: pkgResolved[] = [];
+
+  for (const pkg of chunk(fetchPackages, CONCURRENCY)) {
+    downloadedPackages.push(...await Promise.all(pkg));
+
+    console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
+
+    await sleep(500);
+  }
+
+  console.log(downloadedPackages);
+
+  fs.writeFileSync('./data/packages.json', JSON.stringify({
+    gitea: `${GITEA_URL}/${GITEA_ORG}`,
+    timestamp: parseInt(`${Date.now() / 1000}`),
+    packages: downloadedPackages
+  }, null, 2));
+}
+download();

lib/index.js

@@ -0,0 +1,112 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.download = exports.bytesToBase64 = exports.sleep = exports.chunk = void 0;
+const crypto_1 = require("crypto");
+const process_1 = __importDefault(require("process"));
+const fs_1 = __importDefault(require("fs"));
+const promises_1 = __importDefault(require("fs/promises"));
+const GITEA_AUTH = process_1.default.env.GITEA_AUTH;
+const GITEA_URL = 'https://git.tornado.ws';
+const GITEA_ORG = 'tornado-packages';
+const CONCURRENCY = 5;
+if (!fs_1.default.existsSync('./data')) {
+    fs_1.default.mkdirSync('./data', { recursive: true });
+}
+else {
+    fs_1.default.rmSync('./data', { recursive: true, force: true });
+    fs_1.default.mkdirSync('./data', { recursive: true });
+}
+const chunk = (arr, size) => [...Array(Math.ceil(arr.length / size))].map((_, i) => arr.slice(size * i, size + size * i));
+exports.chunk = chunk;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+exports.sleep = sleep;
+function bytesToBase64(bytes) {
+    let binary = '';
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; ++i) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    return btoa(binary);
+}
+exports.bytesToBase64 = bytesToBase64;
+async function download() {
+    if (!GITEA_AUTH) {
+        throw new Error('Auth token not set');
+    }
+    const packagesResults = [];
+    let page = 1;
+    let lastResult = 1;
+    while (lastResult !== 0) {
+        const packageLists = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}?limit=50&page=${page}`, {
+            headers: {
+                Authorization: `Bearer ${GITEA_AUTH}`
+            }
+        })).json();
+        page += 1;
+        lastResult = packageLists.length;
+        packagesResults.push(...packageLists);
+    }
+    console.log(`Got list of ${packagesResults.length} packages`);
+    const allPackages = packagesResults
+        .filter(({ type }) => type === 'npm')
+        .map(async ({ name, version }) => {
+        const encodedName = encodeURIComponent(name);
+        const files = await (await fetch(`${GITEA_URL}/api/v1/packages/${GITEA_ORG}/npm/${encodedName}/${version}/files`, {
+            headers: {
+                Authorization: `Bearer ${GITEA_AUTH}`
+            }
+        })).json();
+        const file = files.find(({ name }) => name.includes(version));
+        return {
+            name,
+            encodedName,
+            version,
+            file: file.name,
+            sha512: file.sha512
+        };
+    });
+    const packages = [];
+    for (const pkg of (0, exports.chunk)(allPackages, CONCURRENCY)) {
+        packages.push(...await Promise.all(pkg));
+        await sleep(500);
+    }
+    const fetchPackages = packages.map(async ({ name, encodedName, version, file, sha512 }) => {
+        const fileUrl = `${GITEA_URL}/api/packages/${GITEA_ORG}/npm/${encodedName}/-/${version}/${file}`;
+        const fetched = new Uint8Array(await (await fetch(fileUrl)).arrayBuffer());
+        const digest = new Uint8Array(await crypto_1.webcrypto.subtle.digest('SHA-512', fetched));
+        const sha512Res = Array.from(digest).map(b => b.toString(16).padStart(2, '0')).join('');
+        if (sha512 !== sha512Res) {
+            const errMsg = `Digest mismatch for ${fileUrl}, wants ${sha512} have ${sha512Res}`;
+            throw new Error(errMsg);
+        }
+        await promises_1.default.writeFile(`./data/${file}`, fetched);
+        return {
+            name,
+            encodedName,
+            version,
+            file,
+            fileUrl,
+            sha512,
+            integrity: `sha512-${bytesToBase64(digest)}`
+        };
+    });
+    const downloadedPackages = [];
+    for (const pkg of (0, exports.chunk)(fetchPackages, CONCURRENCY)) {
+        downloadedPackages.push(...await Promise.all(pkg));
+        console.log(`Downloaded ${downloadedPackages.length} packages of ${fetchPackages.length}`);
+        await sleep(500);
+    }
+    console.log(downloadedPackages);
+    fs_1.default.writeFileSync('./data/packages.json', JSON.stringify({
+        gitea: `${GITEA_URL}/${GITEA_ORG}`,
+        timestamp: parseInt(`${Date.now() / 1000}`),
+        packages: downloadedPackages
+    }, null, 2));
+}
+exports.download = download;
+download();

package.json

@@ -0,0 +1,29 @@
+{
+  "name": "download-packages",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "scripts": {
+    "types": "tsc",
+    "lint": "eslint *.ts --ext .ts",
+    "start": "ts-node index.ts"
+  },
+  "files": [
+    "lib",
+    ".eslintrc.js",
+    ".gitignore",
+    "index.js",
+    "index.ts",
+    "tsconfig.json",
+    "yarn.lock"
+  ],
+  "devDependencies": {
+    "@types/node": "^20.12.2",
+    "@typescript-eslint/eslint-plugin": "^7.4.0",
+    "@typescript-eslint/parser": "^7.4.0",
+    "dotenv": "^16.4.5",
+    "eslint": "^8.57.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.3"
+  }
+}

tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "typeRoots": [
+      "./node_modules/@types"
+    ],
+    "target": "es2020",
+    "module": "commonjs",
+    "lib": ["es5", "es6", "es2021", "esnext"],
+    "outDir": "lib",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  },
+  "include": ["*"],
+  "exclude": ["node_modules"]
+}