diff --git a/.env.example b/.env.example index b54a472..ee7845c 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -CLOUDFLARE_API_KEY="" -CLOUDFLARE_ACCOUNT_ID="" -CLOUDFLARE_ACCOUNT_EMAIL="" -CLOUDFLARE_LIST_ITEM_LIMIT="300000" \ No newline at end of file +CLOUDFLARE_API_KEY= +CLOUDFLARE_ACCOUNT_ID= +CLOUDFLARE_ACCOUNT_EMAIL= +CLOUDFLARE_LIST_ITEM_LIMIT=300000 diff --git a/cf_list_create.js b/cf_list_create.js index 8d45c37..87d86db 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -1,11 +1,6 @@ -import 'dotenv/config'; -import fetch from 'node-fetch'; import fs from 'fs'; - -const API_TOKEN = process.env.CLOUDFLARE_API_KEY; -const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; -const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; -const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000; +import { DRY_RUN, LIST_ITEM_LIMIT } from './lib/constants.js'; +import { createZeroTrustLists } from './lib/helpers.js'; if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`); @@ -113,79 +108,11 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // If we are dry-running, stop here because we don't want to create lists // TODO: we should probably continue, just without making any real requests to Cloudflare - if (process.env.DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); + if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.'); - // Separate domains into chunks of 1000 (Cloudflare list cap) - const chunks = chunkArray(domains, 1000); - - // Create Cloudflare Zero Trust lists - for (const [index, chunk] of chunks.entries()) { - const listName = `CGPS List - Chunk ${index}`; - - let properList = []; - - chunk.forEach(domain => { - properList.push({ "value": domain }) - }); - - try { - await createZeroTrustList(listName, properList, (index+1), listsToCreate); - await sleep(350); // Sleep for 350ms between list additions - } catch (error) { - console.error(`Error creating list `, process.env.CI ? "(redacted on CI)" : `"${listName}": ${error.response.data}`); - } - } + await createZeroTrustLists(domains) }); function trimArray(arr, size) { return arr.slice(0, size); } - -// Function to check if a domain is valid -function isValidDomain(domain) { - const regex = /^((?!-)[A-Za-z0-9-]{1,63}(? setTimeout(resolve, ms)); -} diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..9b528f3 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,25 @@ +import { isDev } from "./utils.js"; + +if (isDev()) { + const dotenv = await import("dotenv"); + + dotenv.config(); +} + +export const API_TOKEN = process.env.CLOUDFLARE_API_KEY; + +export const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID; + +export const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL; + +export const LIST_ITEM_LIMIT = isNaN(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) + ? 300000 + : parseInt(process.env.CLOUDFLARE_LIST_ITEM_LIMIT, 10); + +export const LIST_ITEM_SIZE = 1000; + +export const API_HOST = "https://api.cloudflare.com/client/v4"; + +export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10); + +export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10); diff --git a/lib/helpers.js b/lib/helpers.js new file mode 100644 index 0000000..bd8b2b6 --- /dev/null +++ b/lib/helpers.js @@ -0,0 +1,89 @@ +import { + ACCOUNT_EMAIL, + ACCOUNT_ID, + API_HOST, + API_TOKEN, + FAST_MODE, + LIST_ITEM_SIZE, +} from "./constants.js"; +import { sleep } from "./utils.js"; + +const request = async (url, options) => { + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${API_TOKEN}`, + "Content-Type": "application/json", + "X-Auth-Email": ACCOUNT_EMAIL, + "X-Auth-Key": API_TOKEN, + }, + ...options, + }); + + return response.json(); +}; + +const createZeroTrustList = (name, items) => { + // https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list + + return request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway/lists`, { + method: "POST", + body: JSON.stringify({ + name, + type: "DOMAIN", + items, + }), + }); +}; + +const createZeroTrustListsOneByOne = async (items) => { + let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + try { + const { + result: { id }, + } = await createZeroTrustList(listName, chunk); + + totalListNumber--; + listNumber++; + console.log( + `Created ${listName} with ID ${id} - ${totalListNumber} left` + ); + await sleep(); + } catch (err) { + console.error(`Could not create ${listName} - ${err.toString()}`); + } + } +}; + +const createZeroTrustListsAtOnce = async (items) => { + let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE); + const requests = []; + + for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) { + const chunk = items + .slice(i, i + LIST_ITEM_SIZE) + .map((item) => ({ value: item })); + const listName = `CGPS List - Chunk ${listNumber}`; + + requests.push(createZeroTrustList(listName, chunk)); + listNumber++; + } + + await Promise.all(requests); + console.log(`Created ${totalListNumber} lists`); +}; + +export const createZeroTrustLists = (items) => { + if (FAST_MODE) { + return createZeroTrustListsAtOnce(items); + } + + return createZeroTrustListsOneByOne(items); +}; diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..f717822 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,4 @@ +export const isDev = () => process.env.NODE_ENV !== "production"; + +export const sleep = (ms = 350) => + new Promise((resolve) => setTimeout(resolve, ms));