From 2e3993c0ee837004d6aafbba22c327247e7cb7e6 Mon Sep 17 00:00:00 2001 From: cupcakearmy Date: Thu, 9 Mar 2023 02:09:52 +0100 Subject: [PATCH] initial commit --- .dockerignore | 5 + .env.sample | 5 + .github/workflows/docker.yaml | 43 +++++ .gitignore | 7 + .nvmrc | 1 + Dockerfile | 19 +++ README.md | 5 + docker-compose.yaml | 17 ++ package.json | 21 +++ pnpm-lock.yaml | 293 ++++++++++++++++++++++++++++++++++ src/api/gitea.ts | 79 +++++++++ src/api/gitea.types.ts | 88 ++++++++++ src/api/github.ts | 39 +++++ src/api/github.types.ts | 113 +++++++++++++ src/config.ts | 49 ++++++ src/core.ts | 51 ++++++ src/index.ts | 10 ++ src/logger.ts | 12 ++ tsconfig.json | 12 ++ 19 files changed, 869 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.sample create mode 100644 .github/workflows/docker.yaml create mode 100644 .gitignore create mode 100644 .nvmrc create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yaml create mode 100644 package.json create mode 100644 pnpm-lock.yaml create mode 100644 src/api/gitea.ts create mode 100644 src/api/gitea.types.ts create mode 100644 src/api/github.ts create mode 100644 src/api/github.types.ts create mode 100644 src/config.ts create mode 100644 src/core.ts create mode 100644 src/index.ts create mode 100644 src/logger.ts create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..31e17c1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!package.json +!pnpm-lock.yaml +!src +!tsconfig.json diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..edb2560 --- /dev/null +++ b/.env.sample @@ -0,0 +1,5 @@ +GITHUB_TOKEN= +GITHUB_SCOPE= + +GITEA_HOST= +GITEA_TOKEN= diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..7c5f743 --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,43 @@ +name: Publish Docker image + +on: + release: + types: [published] + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + + - name: Docker Labels + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + # This assumes your repository is also github.com/foo/bar + # You could also use ghcr.io/foo/some-package as long as you are the user/org "foo" + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..79b68fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +node_modules + +dist +data +config + +.env diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..932b2b0 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +18.15 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dfa6950 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM node:18.15-alpine as base + +RUN npm -g install pnpm@7 +WORKDIR /app +COPY package.json pnpm-lock.yaml ./ + +FROM base as build +RUN pnpm install --frozen-lockfile +COPY . . +RUN pnpm run build + +FROM base as runner +RUN pnpm install --frozen-lockfile --prod +COPY --from=build /app/dist /app/dist +CMD ["pnpm", "run", "start"] + + + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..071bc2e --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Backup Github repos to Gitea + +## Known limitations + +- Issues, PR, etc. can be imported, but [not for a mirror](https://github.com/go-gitea/gitea/issues/18369) diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..9ea2876 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,17 @@ +version: "3.8" + +services: + job: + build: . + env_file: .env + + server: + image: gitea/gitea:1 + environment: + - USER_UID=1000 + - USER_GID=1000 + volumes: + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + ports: + - "3000:3000" diff --git a/package.json b/package.json new file mode 100644 index 0000000..e6a9bce --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "version": "1.0.0-rc.0", + "type": "module", + "scripts": { + "start": "node .", + "build": "tsc", + "dev": "tsc -w" + }, + "main": "./dist/index.js", + "dependencies": { + "axios": "^1.3.4", + "dotenv": "^16.0.3", + "node-cron": "^3.0.2", + "winston": "^3.8.2" + }, + "devDependencies": { + "@types/node": "18", + "@types/node-cron": "^3.0.7", + "typescript": "^4.9.5" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..32b4227 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,293 @@ +lockfileVersion: 5.4 + +specifiers: + '@types/node': '18' + '@types/node-cron': ^3.0.7 + axios: ^1.3.4 + dotenv: ^16.0.3 + node-cron: ^3.0.2 + typescript: ^4.9.5 + winston: ^3.8.2 + +dependencies: + axios: 1.3.4 + dotenv: 16.0.3 + node-cron: 3.0.2 + winston: 3.8.2 + +devDependencies: + '@types/node': 18.14.6 + '@types/node-cron': 3.0.7 + typescript: 4.9.5 + +packages: + + /@colors/colors/1.5.0: + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + dev: false + + /@dabh/diagnostics/2.0.3: + resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} + dependencies: + colorspace: 1.1.4 + enabled: 2.0.0 + kuler: 2.0.0 + dev: false + + /@types/node-cron/3.0.7: + resolution: {integrity: sha512-9PuLtBboc/+JJ7FshmJWv769gDonTpItN0Ol5TMwclpSQNjVyB2SRxSKBcTtbSysSL5R7Oea06kTTFNciCoYwA==} + dev: true + + /@types/node/18.14.6: + resolution: {integrity: sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA==} + dev: true + + /@types/triple-beam/1.3.2: + resolution: {integrity: sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==} + dev: false + + /async/3.2.4: + resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==} + dev: false + + /asynckit/0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: false + + /axios/1.3.4: + resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} + dependencies: + follow-redirects: 1.15.2 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + + /color-convert/1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + dependencies: + color-name: 1.1.3 + dev: false + + /color-name/1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + dev: false + + /color-name/1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: false + + /color-string/1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.2 + dev: false + + /color/3.2.1: + resolution: {integrity: sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==} + dependencies: + color-convert: 1.9.3 + color-string: 1.9.1 + dev: false + + /colorspace/1.1.4: + resolution: {integrity: sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==} + dependencies: + color: 3.2.1 + text-hex: 1.0.0 + dev: false + + /combined-stream/1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: false + + /delayed-stream/1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: false + + /dotenv/16.0.3: + resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} + engines: {node: '>=12'} + dev: false + + /enabled/2.0.0: + resolution: {integrity: sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==} + dev: false + + /fecha/4.2.3: + resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} + dev: false + + /fn.name/1.1.0: + resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} + dev: false + + /follow-redirects/1.15.2: + resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + + /form-data/4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + + /inherits/2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /is-arrayish/0.3.2: + resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} + dev: false + + /is-stream/2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + dev: false + + /kuler/2.0.0: + resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} + dev: false + + /logform/2.5.1: + resolution: {integrity: sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==} + dependencies: + '@colors/colors': 1.5.0 + '@types/triple-beam': 1.3.2 + fecha: 4.2.3 + ms: 2.1.3 + safe-stable-stringify: 2.4.2 + triple-beam: 1.3.0 + dev: false + + /mime-db/1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: false + + /mime-types/2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false + + /ms/2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + dev: false + + /node-cron/3.0.2: + resolution: {integrity: sha512-iP8l0yGlNpE0e6q1o185yOApANRe47UPbLf4YxfbiNHt/RU5eBcGB/e0oudruheSf+LQeDMezqC5BVAb5wwRcQ==} + engines: {node: '>=6.0.0'} + dependencies: + uuid: 8.3.2 + dev: false + + /one-time/1.0.0: + resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==} + dependencies: + fn.name: 1.1.0 + dev: false + + /proxy-from-env/1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: false + + /readable-stream/3.6.1: + resolution: {integrity: sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==} + engines: {node: '>= 6'} + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + dev: false + + /safe-buffer/5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + dev: false + + /safe-stable-stringify/2.4.2: + resolution: {integrity: sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==} + engines: {node: '>=10'} + dev: false + + /simple-swizzle/0.2.2: + resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} + dependencies: + is-arrayish: 0.3.2 + dev: false + + /stack-trace/0.0.10: + resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} + dev: false + + /string_decoder/1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /text-hex/1.0.0: + resolution: {integrity: sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==} + dev: false + + /triple-beam/1.3.0: + resolution: {integrity: sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==} + dev: false + + /typescript/4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + + /util-deprecate/1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /uuid/8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + + /winston-transport/4.5.0: + resolution: {integrity: sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==} + engines: {node: '>= 6.4.0'} + dependencies: + logform: 2.5.1 + readable-stream: 3.6.1 + triple-beam: 1.3.0 + dev: false + + /winston/3.8.2: + resolution: {integrity: sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==} + engines: {node: '>= 12.0.0'} + dependencies: + '@colors/colors': 1.5.0 + '@dabh/diagnostics': 2.0.3 + async: 3.2.4 + is-stream: 2.0.1 + logform: 2.5.1 + one-time: 1.0.0 + readable-stream: 3.6.1 + safe-stable-stringify: 2.4.2 + stack-trace: 0.0.10 + triple-beam: 1.3.0 + winston-transport: 4.5.0 + dev: false diff --git a/src/api/gitea.ts b/src/api/gitea.ts new file mode 100644 index 0000000..b76f55c --- /dev/null +++ b/src/api/gitea.ts @@ -0,0 +1,79 @@ +import axios, { AxiosError } from 'axios' + +import { Config } from '../config.js' +import { logger } from '../logger.js' +import { ListRepositoriesResponse } from './gitea.types.js' + +const Base = axios.create({ + baseURL: new URL('/api/v1', Config.gitea.host).href, + headers: { + Authorization: `token ${Config.gitea.token}`, + }, +}) + +export type MirrorOptions = { + private: boolean + auth_token: string + clone_addr: string + repo_name: string +} +export async function mirror(options: MirrorOptions) { + try { + logger.debug('Mirroring repository', options) + const response = await Base({ + url: '/repos/migrate', + method: 'POST', + data: { + ...options, + + lfs: true, + mirror: true, + service: 'github', + wiki: true, + + // These don't work for now on mirrored repos. + // https://github.com/go-gitea/gitea/issues/18369 + issues: true, + labels: true, + milestones: true, + pull_requests: true, + releases: true, + }, + }) + logger.debug('Mirrored repository', { data: response.data }) + return response.data + } catch (e) { + if (e instanceof AxiosError) { + logger.error('Error mirroring repository', e.response?.data) + } else { + logger.error('Unknown error', e) + } + } +} + +export async function listRepositories(page: number, limit: number) { + const response = await Base({ + url: '/user/repos', + method: 'GET', + params: { + page, + limit, + }, + }) + return response.data +} + +export async function listAllRepositories() { + logger.debug('Listing all repositories in Gitea') + const limit = 50 + const repos: ListRepositoriesResponse = [] + let page = 1 + while (true) { + const response = await listRepositories(page, limit) + repos.push(...response) + if (response.length < limit) break + page++ + } + logger.debug('Listed all repositories in Gitea', { data: repos }) + return repos +} diff --git a/src/api/gitea.types.ts b/src/api/gitea.types.ts new file mode 100644 index 0000000..60e0a8c --- /dev/null +++ b/src/api/gitea.types.ts @@ -0,0 +1,88 @@ +export type ListRepositoriesResponse = Repository[] + +export interface Repository { + id: number + owner: Owner + name: string + full_name: string + description: string + empty: boolean + private: boolean + fork: boolean + template: boolean + parent: null + mirror: boolean + size: number + language: string + languages_url: string + html_url: string + ssh_url: string + clone_url: string + original_url: string + website: string + stars_count: number + forks_count: number + watchers_count: number + open_issues_count: number + open_pr_counter: number + release_counter: number + default_branch: string + archived: boolean + created_at: Date + updated_at: Date + permissions: Permissions + has_issues: boolean + internal_tracker: InternalTracker + has_wiki: boolean + has_pull_requests: boolean + has_projects: boolean + ignore_whitespace_conflicts: boolean + allow_merge_commits: boolean + allow_rebase: boolean + allow_rebase_explicit: boolean + allow_squash_merge: boolean + allow_rebase_update: boolean + default_delete_branch_after_merge: boolean + default_merge_style: string + avatar_url: string + internal: boolean + mirror_interval: string + mirror_updated: Date + repo_transfer: null +} + +export interface InternalTracker { + enable_time_tracker: boolean + allow_only_contributors_to_track_time: boolean + enable_issue_dependencies: boolean +} + +export interface Owner { + id: number + login: string + login_name: string + full_name: string + email: string + avatar_url: string + language: string + is_admin: boolean + last_login: Date + created: Date + restricted: boolean + active: boolean + prohibit_login: boolean + location: string + website: string + description: string + visibility: string + followers_count: number + following_count: number + starred_repos_count: number + username: string +} + +export interface Permissions { + admin: boolean + push: boolean + pull: boolean +} diff --git a/src/api/github.ts b/src/api/github.ts new file mode 100644 index 0000000..2fb6812 --- /dev/null +++ b/src/api/github.ts @@ -0,0 +1,39 @@ +import axios from 'axios' + +import { Config } from '../config.js' +import type { ListRepositoriesResponse } from './github.types.js' + +const Base = axios.create({ + baseURL: 'https://api.github.com', + headers: { + Authorization: `Bearer ${Config.github.token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, +}) + +async function listRepos(page: number, limit: number) { + const response = await Base({ + url: `user/repos`, + method: 'GET', + params: { + page, + per_page: limit, + affiliation: 'owner', + }, + }) + return response.data +} + +export async function listAllRepositories() { + const limit = 100 + const repos: ListRepositoriesResponse = [] + let page = 1 + while (true) { + const response = await listRepos(page, limit) + repos.push(...response) + if (response.length < limit) break + page++ + } + return repos +} diff --git a/src/api/github.types.ts b/src/api/github.types.ts new file mode 100644 index 0000000..9866649 --- /dev/null +++ b/src/api/github.types.ts @@ -0,0 +1,113 @@ +export type ListRepositoriesResponse = Repository[] + +export interface Repository { + id: number + node_id: string + name: string + full_name: string + private: boolean + owner: Owner + html_url: string + description: string + fork: boolean + url: string + forks_url: string + keys_url: string + collaborators_url: string + teams_url: string + hooks_url: string + issue_events_url: string + events_url: string + assignees_url: string + branches_url: string + tags_url: string + blobs_url: string + git_tags_url: string + git_refs_url: string + trees_url: string + statuses_url: string + languages_url: string + stargazers_url: string + contributors_url: string + subscribers_url: string + subscription_url: string + commits_url: string + git_commits_url: string + comments_url: string + issue_comment_url: string + contents_url: string + compare_url: string + merges_url: string + archive_url: string + downloads_url: string + issues_url: string + pulls_url: string + milestones_url: string + notifications_url: string + labels_url: string + releases_url: string + deployments_url: string + created_at: Date + updated_at: Date + pushed_at: Date + git_url: string + ssh_url: string + clone_url: string + svn_url: string + homepage: string + size: number + stargazers_count: number + watchers_count: number + language: string + has_issues: boolean + has_projects: boolean + has_downloads: boolean + has_wiki: boolean + has_pages: boolean + has_discussions: boolean + forks_count: number + mirror_url: null + archived: boolean + disabled: boolean + open_issues_count: number + license: null + allow_forking: boolean + is_template: boolean + web_commit_signoff_required: boolean + topics: string[] + visibility: string + forks: number + open_issues: number + watchers: number + default_branch: string + permissions: Permissions +} + +export interface Owner { + login: string + id: number + node_id: string + avatar_url: string + gravatar_id: string + url: string + html_url: string + followers_url: string + following_url: string + gists_url: string + starred_url: string + subscriptions_url: string + organizations_url: string + repos_url: string + events_url: string + received_events_url: string + type: string + site_admin: boolean +} + +export interface Permissions { + admin: boolean + maintain: boolean + push: boolean + triage: boolean + pull: boolean +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..3796927 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,49 @@ +import { config } from 'dotenv' + +config() + +function getEnv(key: string, fallback: string, parse?: undefined, validator?: (s: string) => boolean): string +function getEnv(key: string, fallback: T, parse: (value: string) => T, validator?: (T: string) => boolean): T +function getEnv( + key: string, + fallback: T, + parse?: (value: string) => T, + validator?: (s: string | T) => boolean +): T | string { + const value = process.env[key] + const parsed = value === undefined ? fallback : parse ? parse(value) : value + if (validator && !validator(parsed)) { + console.error(`Invalid or missing value for ${key}: ${value}`) + process.exit(1) + } + return parsed +} + +function parseBoolean(value: string): boolean { + value = value.toLowerCase() + const truthy = ['true', 'yes', '1', 'y'] + return truthy.includes(value) +} + +function isPresent(s: string): boolean { + return s.length > 0 +} + +function simple(key: string) { + return getEnv(key, '', undefined, isPresent) +} + +export const Config = { + logging: { + level: getEnv('LOG_LEVEL', 'info'), + }, + github: { + scope: simple('GITHUB_SCOPE'), + token: simple('GITHUB_TOKEN'), + }, + gitea: { + host: simple('GITEA_HOST'), + token: simple('GITEA_TOKEN'), + }, + version: getEnv('npm_package_version', 'unknown'), +} diff --git a/src/core.ts b/src/core.ts new file mode 100644 index 0000000..b48562b --- /dev/null +++ b/src/core.ts @@ -0,0 +1,51 @@ +import { listAllRepositories as giteaRepos, mirror, MirrorOptions } from './api/gitea.js' +import { listAllRepositories as githubRepos } from './api/github.js' +import { Config } from './config.js' +import { logger } from './logger.js' + +let running = false + +export async function sync() { + if (running) { + logger.info('Already running, skipping') + return + } + try { + logger.info('Starting sync') + const syncedRepos = await giteaRepos() + const toSync = await githubRepos() + logger.debug('Loaded repos', { remote: toSync.length, local: syncedRepos.length }) + + for (const repo of toSync) { + const sameName = syncedRepos.find((r) => r.name === repo.name || r.original_url === repo.clone_url) + if (sameName) { + if (sameName.original_url === repo.clone_url) { + logger.info('Already synced, skipping', { name: repo.name }) + } else { + logger.error('Repo with same name but different url', { + name: repo.name, + url: repo.clone_url, + original_url: sameName.original_url, + }) + } + continue + } + + const options: MirrorOptions = { + repo_name: repo.name, + clone_addr: repo.clone_url, + private: repo.private, + auth_token: Config.github.token, + } + logger.info('Mirroring repository', options) + await mirror(options) + logger.info('Mirrored repository', { name: repo.name }) + } + logger.info('Finished sync') + } catch (e) { + logger.error(e) + logger.error('Failed to sync') + } finally { + running = false + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..0978311 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,10 @@ +import cron from 'node-cron' + +import { Config } from './config.js' +import { sync } from './core.js' +import { logger } from './logger.js' + +logger.info(`Mirror manager - ${Config.version}`, { version: Config.version }) + +await sync() +cron.schedule('0/5 * * * *', sync) diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..5e4b3b2 --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,12 @@ +import winston from 'winston' + +import { Config } from './config.js' + +export const logger = winston.createLogger({ + level: Config.logging.level, + transports: [ + new winston.transports.Console({ + format: winston.format.combine(winston.format.timestamp(), winston.format.colorize(), winston.format.simple()), + }), + ], +}) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0d109f8 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2022" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, + "module": "ES2022" /* Specify what module code is generated. */, + "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, + "outDir": "./dist" /* Specify an output folder for all emitted files. */, + "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, + "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, + "strict": true /* Enable all strict type-checking options. */, + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +}